Skip to content

Commit 95a6203

Browse files
clockardclockard
authored
Make solr field/schema resolvers respect the tokenized field of attributes (#7003)
1 parent 6206f19 commit 95a6203

12 files changed

Lines changed: 366 additions & 87 deletions

File tree

‎catalog/core/catalog-core-api-impl/src/main/java/ddf/catalog/data/impl/types/CoreAttributes.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -206,7 +206,7 @@ public class CoreAttributes implements Core, MetacardType {
206206
METACARD_TAGS,
207207
true /* indexed */,
208208
true /* stored */,
209-
false /* tokenized */,
209+
true /* tokenized */,
210210
true /* multivalued */,
211211
BasicTypes.STRING_TYPE));
212212
descriptors.add(

‎catalog/solr/catalog-solr-core/src/main/java/ddf/catalog/source/solr/DynamicSchemaResolver.java‎

Lines changed: 39 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -178,9 +178,6 @@ public DynamicSchemaResolver(List<String> additionalFields) {
178178
textSortCharacterLimit = 127;
179179
}
180180
fieldsCache.add(Metacard.ID + SchemaFields.TEXT_SUFFIX);
181-
fieldsCache.add(Metacard.ID + SchemaFields.TEXT_SUFFIX + SchemaFields.TOKENIZED);
182-
fieldsCache.add(
183-
Metacard.ID + SchemaFields.TEXT_SUFFIX + SchemaFields.TOKENIZED + SchemaFields.HAS_CASE);
184181

185182
fieldsCache.add(SchemaFields.METACARD_TYPE_FIELD_NAME);
186183
fieldsCache.add(SchemaFields.METACARD_TYPE_OBJECT_FIELD_NAME);
@@ -273,10 +270,10 @@ void addFields(Metacard metacard, SolrInputDocument solrInputDocument)
273270
if (AttributeFormat.XML.equals(format)
274271
&& solrInputDocument.getFieldValue(
275272
formatIndexName + getSpecialIndexSuffix(AttributeFormat.STRING))
276-
== null) {
273+
== null
274+
&& ad.isTokenized()) {
277275
List<String> parsedTexts = parseTextFrom(attributeValues);
278-
279-
// parsedTexts => *_txt_tokenized
276+
// parsedTexts => *_txt_tokenized - only if attribute is tokenized
280277
String specialStringIndexName =
281278
ad.getName()
282279
+ getFieldSuffix(AttributeFormat.STRING)
@@ -294,12 +291,14 @@ void addFields(Metacard metacard, SolrInputDocument solrInputDocument)
294291
solrInputDocument.addField(
295292
ad.getName() + getFieldSuffix(AttributeFormat.STRING), truncatedValues);
296293

297-
// *_txt_tokenized
298-
solrInputDocument.addField(
299-
ad.getName()
300-
+ getFieldSuffix(AttributeFormat.STRING)
301-
+ getSpecialIndexSuffix(AttributeFormat.STRING),
302-
attributeValues);
294+
// *_txt_tokenized - only if attribute is tokenized
295+
if (ad.isTokenized()) {
296+
solrInputDocument.addField(
297+
ad.getName()
298+
+ getFieldSuffix(AttributeFormat.STRING)
299+
+ getSpecialIndexSuffix(AttributeFormat.STRING),
300+
attributeValues);
301+
}
303302
} else if (AttributeFormat.OBJECT.equals(format)) {
304303
ByteArrayOutputStream byteArrayOS = new ByteArrayOutputStream();
305304
List<Serializable> byteArrays = new ArrayList<>();
@@ -594,13 +593,12 @@ String getField(
594593
Map<String, Serializable> enabledFeatures) {
595594

596595
final String fieldSuffix = getFieldSuffix(format);
597-
String fieldName =
598-
propertyName
599-
+ fieldSuffix
600-
+ (isSearchedAsExactValue ? "" : getSpecialIndexSuffix(format, enabledFeatures));
596+
String baseName = propertyName + fieldSuffix;
597+
String extendedName =
598+
baseName + (isSearchedAsExactValue ? "" : getSpecialIndexSuffix(format, enabledFeatures));
601599

602-
if (fieldsCache.contains(fieldName)) {
603-
return fieldName;
600+
if (fieldsCache.contains(extendedName)) {
601+
return extendedName;
604602
}
605603

606604
switch (format) {
@@ -613,13 +611,14 @@ String getField(
613611
default:
614612
break;
615613
}
616-
614+
// if we have the base name in the cache but not the extended name assume the extended doesn't
615+
// exist otherwise fallback to the old convention
616+
String bestGuess = fieldsCache.contains(baseName) ? baseName : extendedName;
617617
LOGGER.debug(
618618
"Could not find exact schema field name for [{}], attempting to search with [{}]",
619619
propertyName,
620-
fieldName);
621-
622-
return fieldName;
620+
bestGuess);
621+
return bestGuess;
623622
}
624623

625624
private String getFieldSuffix(AttributeFormat format) {
@@ -658,8 +657,11 @@ String getCaseSensitiveField(
658657
&& mappedPropertyName.endsWith(SchemaFields.PHONETICS)) {
659658
return mappedPropertyName;
660659
}
661-
// TODO We can check if this field really does exist
662-
return mappedPropertyName + SchemaFields.HAS_CASE;
660+
String potentialField = mappedPropertyName + SchemaFields.HAS_CASE;
661+
if (fieldsCache.contains(potentialField)) {
662+
return potentialField;
663+
}
664+
return fieldsCache.contains(mappedPropertyName) ? mappedPropertyName : potentialField;
663665
}
664666

665667
private String getSpecialIndexSuffix(AttributeFormat format) {
@@ -706,6 +708,18 @@ private void addToFieldsCache(AttributeDescriptor descriptor) {
706708

707709
fieldsCache.add(descriptor.getName() + schemaFields.getFieldSuffix(format));
708710

711+
if (format.equals(AttributeFormat.GEOMETRY)) {
712+
fieldsCache.add(
713+
descriptor.getName()
714+
+ schemaFields.getFieldSuffix(format)
715+
+ getSpecialIndexSuffix(format));
716+
return;
717+
}
718+
719+
if (!descriptor.isTokenized()) {
720+
return;
721+
}
722+
709723
if (!getSpecialIndexSuffix(format).equals("")) {
710724
fieldsCache.add(
711725
descriptor.getName()
@@ -892,9 +906,7 @@ private Set<String> getAnyTextFields() {
892906
.collect(Collectors.toList());
893907
}
894908

895-
return fields.stream()
896-
.map(field -> field + SchemaFields.TEXT_SUFFIX)
897-
.collect(Collectors.toSet());
909+
return new HashSet<>(fields);
898910
}
899911

900912
private Set<String> getAnyGeoFields() {

‎catalog/solr/catalog-solr-core/src/main/java/ddf/catalog/source/solr/SolrFilterDelegate.java‎

Lines changed: 4 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -313,19 +313,14 @@ public SolrQuery propertyIsEqualTo(String propertyName, String literal, boolean
313313
private String wildcardSolrQuery(
314314
String searchPhrase, String propertyName, boolean isCaseSensitive, boolean isExact) {
315315
String solrQuery;
316-
String tokenized = resolver.getSpecialIndexSuffix(AttributeFormat.STRING, enabledFeatures);
317316
if (Metacard.ANY_TEXT.equals(propertyName)) {
318317
solrQuery =
319318
resolver
320319
.anyTextFields()
321320
.map(
322-
field -> {
323-
if (!isExact) {
324-
return field + tokenized;
325-
} else {
326-
return field;
327-
}
328-
})
321+
field ->
322+
resolver.getField(
323+
field, AttributeFormat.STRING, isExact, Collections.emptyMap()))
329324
.map(
330325
textField -> {
331326
if (isCaseSensitive && !isExact) {
@@ -337,10 +332,7 @@ private String wildcardSolrQuery(
337332
.map(field -> field + ":" + searchPhrase)
338333
.collect(Collectors.joining(" "));
339334
} else {
340-
String field = getMappedPropertyName(propertyName, AttributeFormat.STRING, true);
341-
if (!isExact) {
342-
field += tokenized;
343-
}
335+
String field = getMappedPropertyName(propertyName, AttributeFormat.STRING, isExact);
344336
if (isCaseSensitive && !isExact) {
345337
field = resolver.getCaseSensitiveField(field, enabledFeatures);
346338
}

‎catalog/solr/catalog-solr-core/src/test/java/ddf/catalog/source/solr/DynamicSchemaResolverTest.java‎

Lines changed: 201 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
import static org.hamcrest.Matchers.hasItem;
2020
import static org.hamcrest.Matchers.hasSize;
2121
import static org.hamcrest.Matchers.is;
22+
import static org.hamcrest.Matchers.notNullValue;
2223
import static org.hamcrest.Matchers.nullValue;
2324
import static org.mockito.ArgumentMatchers.eq;
2425
import static org.mockito.Mockito.RETURNS_DEEP_STUBS;
@@ -54,7 +55,7 @@
5455

5556
public class DynamicSchemaResolverTest {
5657

57-
private static final int INITIAL_FIELDS_CACHE_COUNT = 5;
58+
private static final int INITIAL_FIELDS_CACHE_COUNT = 3;
5859

5960
private static final ObjectMapper METACARD_TYPE_MAPPER =
6061
MetacardTypeMapperFactory.newObjectMapper();
@@ -180,7 +181,7 @@ public void testAnyTextFieldPropertyParsing() throws Exception {
180181
// Perform Test
181182
List<String> fields = resolver.anyTextFields().collect(Collectors.toList());
182183
Truth.assertThat(fields)
183-
.containsExactly("metadata_txt", "title_txt", "description_txt", "ext.extracted.text_txt");
184+
.containsExactly("metadata", "title", "description", "ext.extracted.text");
184185
}
185186

186187
@Test
@@ -301,4 +302,202 @@ public void getFieldExactValue() {
301302
dynamicSchemaResolver.getField("unknown", AttributeFormat.STRING, true, enabledFeatures),
302303
is("unknown_txt"));
303304
}
305+
306+
@Test
307+
public void testAddFieldsWithXmlAttributeNotTokenized() throws Exception {
308+
// Setup - XML attribute that is NOT tokenized
309+
String metacardTypeName = "test";
310+
Set<AttributeDescriptor> attributeDescriptors = new HashSet<>(1);
311+
String attributeName = "metadata";
312+
boolean indexed = true;
313+
boolean stored = true;
314+
boolean tokenized = false;
315+
boolean multiValued = false;
316+
317+
attributeDescriptors.add(
318+
new TestAttributeDescriptorImpl(
319+
attributeName,
320+
attributeName,
321+
indexed,
322+
stored,
323+
tokenized,
324+
multiValued,
325+
BasicTypes.XML_TYPE));
326+
327+
Attribute mockAttribute = mock(Attribute.class);
328+
when(mockAttribute.getValue()).thenReturn("<root>test</root>");
329+
when(mockAttribute.getValues()).thenReturn(Collections.singletonList("<root>test</root>"));
330+
331+
Metacard mockMetacard = mock(Metacard.class, RETURNS_DEEP_STUBS);
332+
when(mockMetacard.getMetacardType().getName()).thenReturn(metacardTypeName);
333+
when(mockMetacard.getMetacardType().getAttributeDescriptors()).thenReturn(attributeDescriptors);
334+
when(mockMetacard.getAttribute(attributeName)).thenReturn(mockAttribute);
335+
336+
SolrInputDocument solrInputDocument = new SolrInputDocument();
337+
DynamicSchemaResolver resolver = new DynamicSchemaResolver();
338+
339+
// Perform Test
340+
resolver.addFields(mockMetacard, solrInputDocument);
341+
342+
// Verify - tokenized=false means the special string index should NOT be populated
343+
assertThat(solrInputDocument.getFieldValue("metadata_txt_tokenized"), is(nullValue()));
344+
}
345+
346+
@Test
347+
public void testAddFieldsWithXmlAttributeTokenized() throws Exception {
348+
// Setup - XML attribute that IS tokenized
349+
String metacardTypeName = "test";
350+
Set<AttributeDescriptor> attributeDescriptors = new HashSet<>(1);
351+
String attributeName = "metadata";
352+
boolean indexed = true;
353+
boolean stored = true;
354+
boolean tokenized = true;
355+
boolean multiValued = false;
356+
357+
attributeDescriptors.add(
358+
new TestAttributeDescriptorImpl(
359+
attributeName,
360+
attributeName,
361+
indexed,
362+
stored,
363+
tokenized,
364+
multiValued,
365+
BasicTypes.XML_TYPE));
366+
367+
Attribute mockAttribute = mock(Attribute.class);
368+
when(mockAttribute.getValue()).thenReturn("<root>test</root>");
369+
when(mockAttribute.getValues()).thenReturn(Collections.singletonList("<root>test</root>"));
370+
371+
Metacard mockMetacard = mock(Metacard.class, RETURNS_DEEP_STUBS);
372+
when(mockMetacard.getMetacardType().getName()).thenReturn(metacardTypeName);
373+
when(mockMetacard.getMetacardType().getAttributeDescriptors()).thenReturn(attributeDescriptors);
374+
when(mockMetacard.getAttribute(attributeName)).thenReturn(mockAttribute);
375+
376+
SolrInputDocument solrInputDocument = new SolrInputDocument();
377+
DynamicSchemaResolver resolver = new DynamicSchemaResolver();
378+
379+
// Perform Test
380+
resolver.addFields(mockMetacard, solrInputDocument);
381+
382+
// Verify - tokenized=true means the special string index SHOULD be populated
383+
assertThat(solrInputDocument.getFieldValue("metadata_txt_tokenized"), notNullValue());
384+
}
385+
386+
@Test
387+
public void testAddFieldsWithStringAttributeNotTokenizedAddsTxtNotTokenizedField()
388+
throws Exception {
389+
// Setup - STRING attribute that is NOT tokenized
390+
String metacardTypeName = "test";
391+
Set<AttributeDescriptor> attributeDescriptors = new HashSet<>(1);
392+
String attributeName = "title";
393+
boolean indexed = true;
394+
boolean stored = true;
395+
boolean tokenized = false;
396+
boolean multiValued = false;
397+
398+
attributeDescriptors.add(
399+
new TestAttributeDescriptorImpl(
400+
attributeName,
401+
attributeName,
402+
indexed,
403+
stored,
404+
tokenized,
405+
multiValued,
406+
BasicTypes.STRING_TYPE));
407+
408+
Attribute mockAttribute = mock(Attribute.class);
409+
when(mockAttribute.getValue()).thenReturn("test value");
410+
when(mockAttribute.getValues()).thenReturn(Collections.singletonList("test value"));
411+
412+
Metacard mockMetacard = mock(Metacard.class, RETURNS_DEEP_STUBS);
413+
when(mockMetacard.getMetacardType().getName()).thenReturn(metacardTypeName);
414+
when(mockMetacard.getMetacardType().getAttributeDescriptors()).thenReturn(attributeDescriptors);
415+
when(mockMetacard.getAttribute(attributeName)).thenReturn(mockAttribute);
416+
417+
SolrInputDocument solrInputDocument = new SolrInputDocument();
418+
DynamicSchemaResolver resolver = new DynamicSchemaResolver();
419+
420+
// Perform Test
421+
resolver.addFields(mockMetacard, solrInputDocument);
422+
423+
// Verify - tokenized=false means the tokenized field should NOT be populated
424+
assertThat(solrInputDocument.getFieldValue("title_txt"), is("test value"));
425+
assertThat(solrInputDocument.getFieldValue("title_txt_tokenized"), is(nullValue()));
426+
}
427+
428+
@Test
429+
public void testAddFieldsWithStringAttributeTokenizedAddsBothTxtAndTokenizedField()
430+
throws Exception {
431+
// Setup - STRING attribute that IS tokenized
432+
String metacardTypeName = "test";
433+
Set<AttributeDescriptor> attributeDescriptors = new HashSet<>(1);
434+
String attributeName = "title";
435+
boolean indexed = true;
436+
boolean stored = true;
437+
boolean tokenized = true;
438+
boolean multiValued = false;
439+
440+
attributeDescriptors.add(
441+
new TestAttributeDescriptorImpl(
442+
attributeName,
443+
attributeName,
444+
indexed,
445+
stored,
446+
tokenized,
447+
multiValued,
448+
BasicTypes.STRING_TYPE));
449+
450+
Attribute mockAttribute = mock(Attribute.class);
451+
when(mockAttribute.getValue()).thenReturn("test value");
452+
when(mockAttribute.getValues()).thenReturn(Collections.singletonList("test value"));
453+
454+
Metacard mockMetacard = mock(Metacard.class, RETURNS_DEEP_STUBS);
455+
when(mockMetacard.getMetacardType().getName()).thenReturn(metacardTypeName);
456+
when(mockMetacard.getMetacardType().getAttributeDescriptors()).thenReturn(attributeDescriptors);
457+
when(mockMetacard.getAttribute(attributeName)).thenReturn(mockAttribute);
458+
459+
SolrInputDocument solrInputDocument = new SolrInputDocument();
460+
DynamicSchemaResolver resolver = new DynamicSchemaResolver();
461+
462+
// Perform Test
463+
resolver.addFields(mockMetacard, solrInputDocument);
464+
465+
// Verify - tokenized=true means both fields SHOULD be populated
466+
assertThat(solrInputDocument.getFieldValue("title_txt"), is("test value"));
467+
assertThat(solrInputDocument.getFieldValue("title_txt_tokenized"), is("test value"));
468+
}
469+
470+
@Test
471+
public void testGetFieldNumericalFallsBackToRequestedSuffix() {
472+
// When no numerical field exists in cache, should fall back to the requested suffix
473+
assertThat(
474+
dynamicSchemaResolver.getField(
475+
"unknown", AttributeFormat.INTEGER, false, Collections.emptyMap()),
476+
is("unknown_int"));
477+
assertThat(
478+
dynamicSchemaResolver.getField(
479+
"unknown", AttributeFormat.LONG, false, Collections.emptyMap()),
480+
is("unknown_lng"));
481+
assertThat(
482+
dynamicSchemaResolver.getField(
483+
"unknown", AttributeFormat.DOUBLE, false, Collections.emptyMap()),
484+
is("unknown_dbl"));
485+
assertThat(
486+
dynamicSchemaResolver.getField(
487+
"unknown", AttributeFormat.FLOAT, false, Collections.emptyMap()),
488+
is("unknown_flt"));
489+
assertThat(
490+
dynamicSchemaResolver.getField(
491+
"unknown", AttributeFormat.SHORT, false, Collections.emptyMap()),
492+
is("unknown_shr"));
493+
}
494+
495+
@Test
496+
public void testGetFieldXmlReturnsTextPathSuffix() {
497+
// XML format should use TEXT_PATH (_tpt) suffix
498+
assertThat(
499+
dynamicSchemaResolver.getField(
500+
"metadata", AttributeFormat.XML, false, Collections.emptyMap()),
501+
is("metadata_xml_tpt"));
502+
}
304503
}

0 commit comments

Comments
 (0)