diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/CQLFields.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/CQLFields.java index 239a8381..84520908 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/CQLFields.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/CQLFields.java @@ -145,17 +145,27 @@ public enum CQLFields implements CQLFieldsInterface { " .getAsLong()" + " }")) .order(order))), + // Phrase match on the field and its synonyms sub-field (search-time acronym expansion), fuzziness is not allowed + // for type phrase. Also used for quoted keywords, see fuzzy_title for why best of the two fields is scored. title( StacBasicField.Title.searchField, StacBasicField.Title.displayField, - null, + (literal) -> MultiMatchQuery.of(m -> m + .type(TextQueryType.Phrase) + .fields(StacBasicField.Title.searchField + "^2", StacBasicField.Title.searchField + ".synonyms^2") + .tieBreaker(0.1) + .query(literal))._toQuery(), null, (order) -> new SortOptions.Builder() .field(f -> f.field(StacBasicField.Title.sortField).order(order))), description( StacBasicField.Description.searchField, StacBasicField.Description.displayField, - null, + (literal) -> MultiMatchQuery.of(m -> m + .type(TextQueryType.Phrase) + .fields(StacBasicField.Description.searchField, StacBasicField.Description.searchField + ".synonyms") + .tieBreaker(0.1) + .query(literal))._toQuery(), null, null), providers( @@ -266,15 +276,18 @@ public enum CQLFields implements CQLFieldsInterface { null, (order) -> new SortOptions.Builder() .field(f -> f.field(StacSummeries.Score.sortField).order(order))), + // Fuzzy match on the field and its synonyms sub-field (search-time acronym expansion, e.g. "SOOP" -> "ships of opportunity"). + // best_fields scores the better field plus tie_breaker times the other. Plain words pass the synonym analyzer + // unchanged and would otherwise be scored twice. Title fields are boosted 2, description fields 1, as before. fuzzy_title( null, StacBasicField.Title.displayField, - (literal) -> MatchQuery.of(m -> m + (literal) -> MultiMatchQuery.of(m -> m + .type(TextQueryType.BestFields) + .fields(StacBasicField.Title.searchField + "^2", StacBasicField.Title.searchField + ".synonyms^2") + .tieBreaker(0.1) .fuzziness("AUTO") - .field(StacBasicField.Title.searchField) .prefixLength(4)// Use 4 to deal with NRMN short form may match NRM records - // Increase the relevance of matches in title - .boost(2.0F) .operator(Operator.And)// ensure all terms are matched with fuzziness .query(literal))._toQuery(), null, @@ -282,34 +295,16 @@ public enum CQLFields implements CQLFieldsInterface { fuzzy_desc( null, StacBasicField.Description.displayField, - (literal) -> MatchQuery.of(m -> m + (literal) -> MultiMatchQuery.of(m -> m + .type(TextQueryType.BestFields) + .fields(StacBasicField.Description.searchField, StacBasicField.Description.searchField + ".synonyms") + .tieBreaker(0.1) .fuzziness("AUTO") - .field(StacBasicField.Description.searchField) .prefixLength(4)// Use 4 to deal with NRMN short form may match NRM records .operator(Operator.And)// ensure all terms are matched with fuzziness .query(literal))._toQuery(), null, null), - // Acronym match on the synonyms sub-fields (search-time expansion), e.g. "SOOP" -> "ships of opportunity". - acronym_title( - StacBasicField.Title.searchField + ".synonyms", - StacBasicField.Title.displayField, - (literal) -> MatchQuery.of(m -> m - .field(StacBasicField.Title.searchField + ".synonyms") - .operator(Operator.And)// all expanded terms must match - .boost(2.0F)// align with fuzzy_title weighting - .query(literal))._toQuery(), - null, - null), - acronym_desc( - StacBasicField.Description.searchField + ".synonyms", - StacBasicField.Description.displayField, - (literal) -> MatchQuery.of(m -> m - .field(StacBasicField.Description.searchField + ".synonyms") - .operator(Operator.And) - .query(literal))._toQuery(), - null, - null), // Contains cloud-optimized data assets_summary( StacBasicField.AssetsSummary.searchField, diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java index 868b477e..7a704e03 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java @@ -448,9 +448,6 @@ private static List createKeywordShouldClauses(List keywords) { should.add(CQLFields.organisation_vocabs.getPropertyEqualToQuery(term)); should.add(CQLFields.platform_vocabs.getPropertyEqualToQuery(term)); should.add(CQLFields.id.getPropertyEqualToQuery(term)); - // Acronym match on the *.synonyms sub-fields, e.g. "SOOP" -> "ships of opportunity". - should.add(CQLFields.acronym_title.getPropertyEqualToQuery(term)); - should.add(CQLFields.acronym_desc.getPropertyEqualToQuery(term)); // credit_contains uses match query by default, exact match is not applied here should.add(CQLFields.credit_contains.getPropertyEqualToQuery(term)); } diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/util/ExplainSimplifier.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/util/ExplainSimplifier.java index 6a6b6ec3..15350dec 100644 --- a/server/src/main/java/au/org/aodn/ogcapi/server/core/util/ExplainSimplifier.java +++ b/server/src/main/java/au/org/aodn/ogcapi/server/core/util/ExplainSimplifier.java @@ -38,6 +38,12 @@ public class ExplainSimplifier { protected static final String SYNONYM_PREFIX = "Synonym("; + // A dis_max (disjunction max) explanation includes every match, but it is scored as its winning child plus + // tie_breaker times the others, e.g. the best_fields multi_match on title and title.synonyms (see CQLFields.fuzzy_title). + // Lucene writes "max of:" for tie_breaker 0, otherwise "max plus 0.1 times others of:". + protected static final String MAX_OF_PREFIX = "max of:"; + protected static final String MAX_PLUS_PREFIX = "max plus "; + protected static final String RELEVANCE_DESCRIPTION_PREFIX = "_score:"; protected static final Comparator BY_SCORE_DESC = @@ -214,6 +220,19 @@ protected static void collectScoreParts(List details, for (ExplanationDetail detail : details) { String description = detail.description(); + // report only the winner, the tie_breaker share of the others is small and would list the same word twice + if (description != null + && (description.startsWith(MAX_OF_PREFIX) || description.startsWith(MAX_PLUS_PREFIX))) { + if (detail.details() != null) { + // the highest child, not one equal to the parent, so float rounding cannot lose the winner; a tie keeps the first + detail.details().stream() + .max(Comparator.comparingDouble(ExplanationDetail::value)) + .ifPresent(winner -> collectScoreParts( + List.of(winner), terms, filters)); + } + continue; + } + // most nodes are idf/tf breakdowns, skip the regex for them if (description != null && description.startsWith(WEIGHT_PREFIX)) { Matcher matcher = WEIGHT_PATTERN.matcher(description); diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/core/util/ExplainSimplifierTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/core/util/ExplainSimplifierTest.java index 43db66ee..5510da71 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/core/util/ExplainSimplifierTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/core/util/ExplainSimplifierTest.java @@ -5,10 +5,12 @@ import co.elastic.clients.elasticsearch._types.ScriptSortType; import co.elastic.clients.elasticsearch._types.SortOptions; import co.elastic.clients.elasticsearch._types.SortOrder; +import co.elastic.clients.elasticsearch.core.explain.ExplanationDetail; import co.elastic.clients.elasticsearch.core.search.Hit; import com.fasterxml.jackson.databind.node.ObjectNode; import org.junit.jupiter.api.Test; +import java.util.ArrayList; import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -86,6 +88,68 @@ public void sortValuesAreLeftOutWhenTheHitCarriesNone() { assertNull(simplified.getSortValues()); } + @Test + public void disMaxReportsOnlyOneWinnerWhenPlainAndSynonymScoresTie() { + List terms = collectDisMax("max of:", 12.1f, + detail(12.1f, "weight(title:wave in 1) [BM25], result of:"), + detail(12.1f, "weight(title.synonyms:wave in 1) [BM25], result of:")); + + assertEquals(1, terms.size()); + assertEquals("title", terms.get(0).getField()); + assertEquals(12.1, terms.get(0).getScore(), 0.0001); + } + + @Test + public void disMaxReportsSynonymWinnerAsOneEntry() { + List terms = collectDisMax("max of:", 12.1f, + detail(4.2f, "weight(title:soop in 1) [BM25], result of:"), + detail(12.1f, "weight(Synonym(title.synonyms:soop title.synonyms:ships " + + "title.synonyms:of title.synonyms:opportunity) in 1) [BM25], result of:")); + + assertEquals(1, terms.size()); + assertEquals("title.synonyms", terms.get(0).getField()); + assertEquals("soop ships of opportunity", terms.get(0).getTerm()); + assertEquals(12.1, terms.get(0).getScore(), 0.0001); + } + + @Test + public void disMaxReportsHighestChildWhenNoChildEqualsParent() { + List terms = collectDisMax("max of:", 12.1f, + detail(12.0999f, "weight(title:wave in 1) [BM25], result of:"), + detail(4.0f, "weight(title.synonyms:wave in 1) [BM25], result of:")); + + assertEquals(1, terms.size()); + assertEquals("title", terms.get(0).getField()); + } + + @Test + public void disMaxWithTieBreakerReportsOnlyTheWinner() { + // best_fields multi_match with tie_breaker 0.1: 36.3 + 0.1 * 12.1 + List terms = collectDisMax("max plus 0.1 times others of:", 37.51f, + detail(36.3f, "weight(title:wave in 1) [BM25], result of:"), + detail(12.1f, "weight(title.synonyms:wave in 1) [BM25], result of:")); + + assertEquals(1, terms.size()); + assertEquals("title", terms.get(0).getField()); + assertEquals(36.3, terms.get(0).getScore(), 0.0001); + } + + private static List collectDisMax(String description, float value, + ExplanationDetail... children) { + List terms = new ArrayList<>(); + ExplainSimplifier.collectScoreParts( + List.of(detail(value, description, children)), terms, new ArrayList<>()); + return terms; + } + + private static ExplanationDetail detail(float value, String description, + ExplanationDetail... children) { + return ExplanationDetail.of(d -> d + .value(value) + .description(description) + .details(List.of(children))); + } + private static ExplainSimplifiedResponse.SortValue sortValue(String field, Object value) { return ExplainSimplifiedResponse.SortValue.builder() .field(field) diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/service/ElasticSearchTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/service/ElasticSearchTest.java index f5a262b7..c7c6dc40 100644 --- a/server/src/test/java/au/org/aodn/ogcapi/server/service/ElasticSearchTest.java +++ b/server/src/test/java/au/org/aodn/ogcapi/server/service/ElasticSearchTest.java @@ -5,7 +5,10 @@ import co.elastic.clients.elasticsearch.ElasticsearchClient; import co.elastic.clients.elasticsearch._types.FieldValue; import co.elastic.clients.elasticsearch._types.SortOptions; +import co.elastic.clients.elasticsearch._types.query_dsl.MultiMatchQuery; +import co.elastic.clients.elasticsearch._types.query_dsl.Operator; import co.elastic.clients.elasticsearch._types.query_dsl.Query; +import co.elastic.clients.elasticsearch._types.query_dsl.TextQueryType; import co.elastic.clients.elasticsearch.core.SearchRequest; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; @@ -37,10 +40,22 @@ public void searchByParametersWithDoubleQuote() throws Exception { "-score,-rank", CQLCrsType.EPSG4326); - assertEquals(9, capturingSearch.should.size(), - "Exact match should produce 9 queries (title + description + other fields, no dataset_group)"); - assertTrue(capturingSearch.should.get(0).isMatchPhrase(), "Title query should be MatchPhraseQuery"); - assertTrue(capturingSearch.should.get(1).isMatchPhrase(), "Description query should be MatchPhraseQuery"); + assertEquals(7, capturingSearch.should.size(), + "Exact match should produce 7 queries (title + description + other fields, no dataset_group)"); + + MultiMatchQuery title = capturingSearch.should.get(0).multiMatch(); + assertEquals(TextQueryType.Phrase, title.type(), "Title query should be a phrase multi_match"); + assertEquals(List.of("title^2", "title.synonyms^2"), title.fields()); + assertEquals(0.1, title.tieBreaker()); + assertEquals("ocean temperature", title.query(), "quotes are removed"); + assertNull(title.fuzziness(), "fuzziness is not allowed for a phrase"); + + MultiMatchQuery description = capturingSearch.should.get(1).multiMatch(); + assertEquals(TextQueryType.Phrase, description.type(), "Description query should be a phrase multi_match"); + assertEquals(List.of("description", "description.synonyms"), description.fields()); + assertEquals(0.1, description.tieBreaker()); + + assertSynonymsOnlyInsideMultiMatch(capturingSearch.should); assertTrue(capturingSearch.arguments.sortOptions().get(0).isScript(), "dataset_group priority sort should be the first sort key"); } @@ -56,12 +71,40 @@ public void searchByParametersWithoutDoubleQuote() throws Exception { "-score,-rank", CQLCrsType.EPSG4326); - assertEquals(9, capturingSearch.should.size(), "Fuzzy match should produce 9 queries"); - assertTrue(capturingSearch.should.get(0).isMatch(), "fuzzy_title should be MatchQuery"); + assertEquals(7, capturingSearch.should.size(), "Fuzzy match should produce 7 queries"); + + MultiMatchQuery title = capturingSearch.should.get(0).multiMatch(); + assertEquals(TextQueryType.BestFields, title.type(), "fuzzy_title should be a best_fields multi_match"); + assertEquals(List.of("title^2", "title.synonyms^2"), title.fields()); + assertEquals(0.1, title.tieBreaker()); + assertEquals("AUTO", title.fuzziness()); + assertEquals(4, title.prefixLength()); + assertEquals(Operator.And, title.operator()); + + MultiMatchQuery description = capturingSearch.should.get(1).multiMatch(); + assertEquals(TextQueryType.BestFields, description.type(), "fuzzy_desc should be a best_fields multi_match"); + assertEquals(List.of("description", "description.synonyms"), description.fields()); + assertEquals(0.1, description.tieBreaker()); + assertEquals("AUTO", description.fuzziness()); + assertEquals(4, description.prefixLength()); + assertEquals(Operator.And, description.operator()); + + assertSynonymsOnlyInsideMultiMatch(capturingSearch.should); assertTrue(capturingSearch.arguments.sortOptions().get(0).isScript(), "dataset_group priority sort should be the first sort key"); } + /** + * The synonyms sub-fields must only be scored inside the title/description multi_match, a separate clause on them + * would score plain words twice + */ + private static void assertSynonymsOnlyInsideMultiMatch(List should) { + should.stream() + .filter(q -> !q.isMultiMatch()) + .forEach(q -> assertFalse(q.toString().contains(".synonyms"), + "unexpected clause on a synonyms field: " + q)); + } + // The portal SEO pipeline requests these in bulk to build Dataset JSON-LD @Test public void seoPropertiesMapToStacFields() throws Exception {