Skip to content

Commit a484172

Browse files
authored
SOLR-18202 Reduce Memory Usage of testSequentialVsParallelFingerprint (#4310)
testSequentialVsParallelFingerprint creates a bunch of segments by design. This creates enormous memory overhead, partly because a new searcher is opened with each commit. I tried setting openSearcher to false but the RealtimeGetHandler searcher still gets opened and incurs similar memory overhead per commit. I landed on just writing via IW which seems ok for this test. For reference by doing this we reduce the total memory consumption of this test by about 10-20X according to my profiler.
1 parent db36284 commit a484172

1 file changed

Lines changed: 23 additions & 8 deletions

File tree

solr/core/src/test/org/apache/solr/update/SolrIndexFingerprintTest.java

Lines changed: 23 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -17,11 +17,16 @@
1717
package org.apache.solr.update;
1818

1919
import java.io.IOException;
20-
import java.util.stream.IntStream;
20+
import org.apache.lucene.document.Document;
21+
import org.apache.lucene.document.Field;
22+
import org.apache.lucene.document.NumericDocValuesField;
23+
import org.apache.lucene.document.StringField;
2124
import org.apache.lucene.index.FilterLeafReader;
25+
import org.apache.lucene.index.IndexWriter;
2226
import org.apache.lucene.index.LeafReader;
2327
import org.apache.solr.SolrTestCaseJ4;
2428
import org.apache.solr.core.SolrCore;
29+
import org.apache.solr.util.RefCounted;
2530
import org.junit.BeforeClass;
2631
import org.junit.Test;
2732

@@ -38,13 +43,23 @@ public void testSequentialVsParallelFingerprint() throws Exception {
3843
SolrCore core = h.getCore();
3944

4045
int numDocs = RANDOM_MULTIPLIER == 1 ? 3 : 500;
41-
// Create a set of many segments (to catch race conditions, i.e. SOLR-17863)
42-
IntStream.range(0, numDocs)
43-
.forEach(
44-
i -> {
45-
assertU(adoc("id", "" + i));
46-
assertU(commit());
47-
});
46+
// Create a set of many segments (to catch race conditions, i.e. SOLR-17863).
47+
// Write directly to the IndexWriter and flush after each doc to create one segment per doc,
48+
// avoiding the overhead of opening a new searcher on every Solr commit.
49+
RefCounted<IndexWriter> iwRef = core.getSolrCoreState().getIndexWriter(core);
50+
try {
51+
IndexWriter writer = iwRef.get();
52+
for (int i = 0; i < numDocs; i++) {
53+
Document doc = new Document();
54+
doc.add(new StringField("id", Integer.toString(i), Field.Store.YES));
55+
doc.add(new NumericDocValuesField("_version_", i + 1));
56+
writer.addDocument(doc);
57+
writer.flush();
58+
}
59+
} finally {
60+
iwRef.decref();
61+
}
62+
assertU(commit("openSearcher", "true"));
4863

4964
try (var searcher = core.getSearcher().get()) {
5065
// Compute fingerprint sequentially to compare with parallel computation

0 commit comments

Comments
 (0)