Feature/9450 improve search relevance calculation

server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/CQLFields.java

@@ -145,17 +145,27 @@ public enum CQLFields implements CQLFieldsInterface {
                                                                 "           .getAsLong()" +
                                                                 "     }"))
                                 .order(order))),
+        // Phrase match on the field and its synonyms sub-field (search-time acronym expansion), fuzziness is not allowed
+        // for type phrase. Also used for quoted keywords, see fuzzy_title for why best of the two fields is scored.
         title(
                 StacBasicField.Title.searchField,
                 StacBasicField.Title.displayField,
-                null,
+                (literal) -> MultiMatchQuery.of(m -> m
+                                .type(TextQueryType.Phrase)
+                                .fields(StacBasicField.Title.searchField + "^2", StacBasicField.Title.searchField + ".synonyms^2")
+                                .tieBreaker(0.1)
+                                .query(literal))._toQuery(),
                 null,
                 (order) -> new SortOptions.Builder()
                                 .field(f -> f.field(StacBasicField.Title.sortField).order(order))),
         description(
                 StacBasicField.Description.searchField,
                 StacBasicField.Description.displayField,
-                null,
+                (literal) -> MultiMatchQuery.of(m -> m
+                                .type(TextQueryType.Phrase)
+                                .fields(StacBasicField.Description.searchField, StacBasicField.Description.searchField + ".synonyms")
+                                .tieBreaker(0.1)
+                                .query(literal))._toQuery(),
                 null,
                 null),
         providers(
@@ -266,50 +276,35 @@ public enum CQLFields implements CQLFieldsInterface {
                 null,
                 (order) -> new SortOptions.Builder()
                                 .field(f -> f.field(StacSummeries.Score.sortField).order(order))),
+        // Fuzzy match on the field and its synonyms sub-field (search-time acronym expansion, e.g. "SOOP" -> "ships of opportunity").
+        // best_fields scores the better field plus tie_breaker times the other. Plain words pass the synonym analyzer
+        // unchanged and would otherwise be scored twice. Title fields are boosted 2, description fields 1, as before.
         fuzzy_title(
                 null,
                 StacBasicField.Title.displayField,
-                (literal) -> MatchQuery.of(m -> m
+                (literal) -> MultiMatchQuery.of(m -> m
+                                .type(TextQueryType.BestFields)
+                                .fields(StacBasicField.Title.searchField + "^2", StacBasicField.Title.searchField + ".synonyms^2")
+                                .tieBreaker(0.1)
                                 .fuzziness("AUTO")
-                                .field(StacBasicField.Title.searchField)
                                 .prefixLength(4)// Use 4 to deal with NRMN short form may match NRM records
-                                // Increase the relevance of matches in title
-                                .boost(2.0F)
                                 .operator(Operator.And)// ensure all terms are matched with fuzziness
                                 .query(literal))._toQuery(),
                 null,
                 null),
         fuzzy_desc(
                 null,
                 StacBasicField.Description.displayField,
-                (literal) -> MatchQuery.of(m -> m
+                (literal) -> MultiMatchQuery.of(m -> m
+                                .type(TextQueryType.BestFields)
+                                .fields(StacBasicField.Description.searchField, StacBasicField.Description.searchField + ".synonyms")
+                                .tieBreaker(0.1)
                                 .fuzziness("AUTO")
-                                .field(StacBasicField.Description.searchField)
                                 .prefixLength(4)// Use 4 to deal with NRMN short form may match NRM records
                                 .operator(Operator.And)// ensure all terms are matched with fuzziness
                                 .query(literal))._toQuery(),
                 null,
                 null),
-        // Acronym match on the synonyms sub-fields (search-time expansion), e.g. "SOOP" -> "ships of opportunity".
-        acronym_title(
-                StacBasicField.Title.searchField + ".synonyms",
-                StacBasicField.Title.displayField,
-                (literal) -> MatchQuery.of(m -> m
-                                .field(StacBasicField.Title.searchField + ".synonyms")
-                                .operator(Operator.And)// all expanded terms must match
-                                .boost(2.0F)// align with fuzzy_title weighting
-                                .query(literal))._toQuery(),
-                null,
-                null),
-        acronym_desc(
-                StacBasicField.Description.searchField + ".synonyms",
-                StacBasicField.Description.displayField,
-                (literal) -> MatchQuery.of(m -> m
-                                .field(StacBasicField.Description.searchField + ".synonyms")
-                                .operator(Operator.And)
-                                .query(literal))._toQuery(),
-                null,
-                null),
         // Contains cloud-optimized data
         assets_summary(
                 StacBasicField.AssetsSummary.searchField,

server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java

@@ -448,9 +448,6 @@ private static List<Query> createKeywordShouldClauses(List<String> keywords) {
             should.add(CQLFields.organisation_vocabs.getPropertyEqualToQuery(term));
             should.add(CQLFields.platform_vocabs.getPropertyEqualToQuery(term));
             should.add(CQLFields.id.getPropertyEqualToQuery(term));
-            // Acronym match on the *.synonyms sub-fields, e.g. "SOOP" -> "ships of opportunity".
-            should.add(CQLFields.acronym_title.getPropertyEqualToQuery(term));
-            should.add(CQLFields.acronym_desc.getPropertyEqualToQuery(term));
             // credit_contains uses match query by default, exact match is not applied here
             should.add(CQLFields.credit_contains.getPropertyEqualToQuery(term));
         }

server/src/main/java/au/org/aodn/ogcapi/server/core/util/ExplainSimplifier.java

@@ -38,6 +38,12 @@ public class ExplainSimplifier {
 
     protected static final String SYNONYM_PREFIX = "Synonym(";
 
+    // A dis_max (disjunction max) explanation includes every match, but it is scored as its winning child plus
+    // tie_breaker times the others, e.g. the best_fields multi_match on title and title.synonyms (see CQLFields.fuzzy_title).
+    // Lucene writes "max of:" for tie_breaker 0, otherwise "max plus 0.1 times others of:".
+    protected static final String MAX_OF_PREFIX = "max of:";
+    protected static final String MAX_PLUS_PREFIX = "max plus ";
+
     protected static final String RELEVANCE_DESCRIPTION_PREFIX = "_score:";
 
     protected static final Comparator<ExplainSimplifiedResponse.MatchedTerm> BY_SCORE_DESC =
@@ -214,6 +220,19 @@ protected static void collectScoreParts(List<ExplanationDetail> details,
         for (ExplanationDetail detail : details) {
             String description = detail.description();
 
+            // report only the winner, the tie_breaker share of the others is small and would list the same word twice
+            if (description != null
+                    && (description.startsWith(MAX_OF_PREFIX) || description.startsWith(MAX_PLUS_PREFIX))) {
+                if (detail.details() != null) {
+                    // the highest child, not one equal to the parent, so float rounding cannot lose the winner; a tie keeps the first
+                    detail.details().stream()
+                            .max(Comparator.comparingDouble(ExplanationDetail::value))
+                            .ifPresent(winner -> collectScoreParts(
+                                    List.of(winner), terms, filters));
+                }
+                continue;
+            }
+
             // most nodes are idf/tf breakdowns, skip the regex for them
             if (description != null && description.startsWith(WEIGHT_PREFIX)) {
                 Matcher matcher = WEIGHT_PATTERN.matcher(description);

server/src/test/java/au/org/aodn/ogcapi/server/core/util/ExplainSimplifierTest.java

@@ -5,10 +5,12 @@
 import co.elastic.clients.elasticsearch._types.ScriptSortType;
 import co.elastic.clients.elasticsearch._types.SortOptions;
 import co.elastic.clients.elasticsearch._types.SortOrder;
+import co.elastic.clients.elasticsearch.core.explain.ExplanationDetail;
 import co.elastic.clients.elasticsearch.core.search.Hit;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import org.junit.jupiter.api.Test;
 
+import java.util.ArrayList;
 import java.util.List;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -86,6 +88,68 @@ public void sortValuesAreLeftOutWhenTheHitCarriesNone() {
         assertNull(simplified.getSortValues());
     }
 
+    @Test
+    public void disMaxReportsOnlyOneWinnerWhenPlainAndSynonymScoresTie() {
+        List<ExplainSimplifiedResponse.MatchedTerm> terms = collectDisMax("max of:", 12.1f,
+                detail(12.1f, "weight(title:wave in 1) [BM25], result of:"),
+                detail(12.1f, "weight(title.synonyms:wave in 1) [BM25], result of:"));
+
+        assertEquals(1, terms.size());
+        assertEquals("title", terms.get(0).getField());
+        assertEquals(12.1, terms.get(0).getScore(), 0.0001);
+    }
+
+    @Test
+    public void disMaxReportsSynonymWinnerAsOneEntry() {
+        List<ExplainSimplifiedResponse.MatchedTerm> terms = collectDisMax("max of:", 12.1f,
+                detail(4.2f, "weight(title:soop in 1) [BM25], result of:"),
+                detail(12.1f, "weight(Synonym(title.synonyms:soop title.synonyms:ships "
+                        + "title.synonyms:of title.synonyms:opportunity) in 1) [BM25], result of:"));
+
+        assertEquals(1, terms.size());
+        assertEquals("title.synonyms", terms.get(0).getField());
+        assertEquals("soop ships of opportunity", terms.get(0).getTerm());
+        assertEquals(12.1, terms.get(0).getScore(), 0.0001);
+    }
+
+    @Test
+    public void disMaxReportsHighestChildWhenNoChildEqualsParent() {
+        List<ExplainSimplifiedResponse.MatchedTerm> terms = collectDisMax("max of:", 12.1f,
+                detail(12.0999f, "weight(title:wave in 1) [BM25], result of:"),
+                detail(4.0f, "weight(title.synonyms:wave in 1) [BM25], result of:"));
+
+        assertEquals(1, terms.size());
+        assertEquals("title", terms.get(0).getField());
+    }
+
+    @Test
+    public void disMaxWithTieBreakerReportsOnlyTheWinner() {
+        // best_fields multi_match with tie_breaker 0.1: 36.3 + 0.1 * 12.1
+        List<ExplainSimplifiedResponse.MatchedTerm> terms = collectDisMax("max plus 0.1 times others of:", 37.51f,
+                detail(36.3f, "weight(title:wave in 1) [BM25], result of:"),
+                detail(12.1f, "weight(title.synonyms:wave in 1) [BM25], result of:"));
+
+        assertEquals(1, terms.size());
+        assertEquals("title", terms.get(0).getField());
+        assertEquals(36.3, terms.get(0).getScore(), 0.0001);
+    }
+
+    private static List<ExplainSimplifiedResponse.MatchedTerm> collectDisMax(String description, float value,
+                                                                            ExplanationDetail... children) {
+        List<ExplainSimplifiedResponse.MatchedTerm> terms = new ArrayList<>();
+        ExplainSimplifier.collectScoreParts(
+                List.of(detail(value, description, children)), terms, new ArrayList<>());
+        return terms;
+    }
+
+    private static ExplanationDetail detail(float value, String description,
+                                            ExplanationDetail... children) {
+        return ExplanationDetail.of(d -> d
+                .value(value)
+                .description(description)
+                .details(List.of(children)));
+    }
+
     private static ExplainSimplifiedResponse.SortValue sortValue(String field, Object value) {
         return ExplainSimplifiedResponse.SortValue.builder()
                 .field(field)

server/src/test/java/au/org/aodn/ogcapi/server/service/ElasticSearchTest.java

@@ -5,7 +5,10 @@
 import co.elastic.clients.elasticsearch.ElasticsearchClient;
 import co.elastic.clients.elasticsearch._types.FieldValue;
 import co.elastic.clients.elasticsearch._types.SortOptions;
+import co.elastic.clients.elasticsearch._types.query_dsl.MultiMatchQuery;
+import co.elastic.clients.elasticsearch._types.query_dsl.Operator;
 import co.elastic.clients.elasticsearch._types.query_dsl.Query;
+import co.elastic.clients.elasticsearch._types.query_dsl.TextQueryType;
 import co.elastic.clients.elasticsearch.core.SearchRequest;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
@@ -37,10 +40,22 @@ public void searchByParametersWithDoubleQuote() throws Exception {
                 "-score,-rank",
                 CQLCrsType.EPSG4326);
 
-        assertEquals(9, capturingSearch.should.size(),
-                "Exact match should produce 9 queries (title + description + other fields, no dataset_group)");
-        assertTrue(capturingSearch.should.get(0).isMatchPhrase(), "Title query should be MatchPhraseQuery");
-        assertTrue(capturingSearch.should.get(1).isMatchPhrase(), "Description query should be MatchPhraseQuery");
+        assertEquals(7, capturingSearch.should.size(),
+                "Exact match should produce 7 queries (title + description + other fields, no dataset_group)");
+
+        MultiMatchQuery title = capturingSearch.should.get(0).multiMatch();
+        assertEquals(TextQueryType.Phrase, title.type(), "Title query should be a phrase multi_match");
+        assertEquals(List.of("title^2", "title.synonyms^2"), title.fields());
+        assertEquals(0.1, title.tieBreaker());
+        assertEquals("ocean temperature", title.query(), "quotes are removed");
+        assertNull(title.fuzziness(), "fuzziness is not allowed for a phrase");
+
+        MultiMatchQuery description = capturingSearch.should.get(1).multiMatch();
+        assertEquals(TextQueryType.Phrase, description.type(), "Description query should be a phrase multi_match");
+        assertEquals(List.of("description", "description.synonyms"), description.fields());
+        assertEquals(0.1, description.tieBreaker());
+
+        assertSynonymsOnlyInsideMultiMatch(capturingSearch.should);
         assertTrue(capturingSearch.arguments.sortOptions().get(0).isScript(),
                 "dataset_group priority sort should be the first sort key");
     }
@@ -56,12 +71,40 @@ public void searchByParametersWithoutDoubleQuote() throws Exception {
                 "-score,-rank",
                 CQLCrsType.EPSG4326);
 
-        assertEquals(9, capturingSearch.should.size(), "Fuzzy match should produce 9 queries");
-        assertTrue(capturingSearch.should.get(0).isMatch(), "fuzzy_title should be MatchQuery");
+        assertEquals(7, capturingSearch.should.size(), "Fuzzy match should produce 7 queries");
+
+        MultiMatchQuery title = capturingSearch.should.get(0).multiMatch();
+        assertEquals(TextQueryType.BestFields, title.type(), "fuzzy_title should be a best_fields multi_match");
+        assertEquals(List.of("title^2", "title.synonyms^2"), title.fields());
+        assertEquals(0.1, title.tieBreaker());
+        assertEquals("AUTO", title.fuzziness());
+        assertEquals(4, title.prefixLength());
+        assertEquals(Operator.And, title.operator());
+
+        MultiMatchQuery description = capturingSearch.should.get(1).multiMatch();
+        assertEquals(TextQueryType.BestFields, description.type(), "fuzzy_desc should be a best_fields multi_match");
+        assertEquals(List.of("description", "description.synonyms"), description.fields());
+        assertEquals(0.1, description.tieBreaker());
+        assertEquals("AUTO", description.fuzziness());
+        assertEquals(4, description.prefixLength());
+        assertEquals(Operator.And, description.operator());
+
+        assertSynonymsOnlyInsideMultiMatch(capturingSearch.should);
         assertTrue(capturingSearch.arguments.sortOptions().get(0).isScript(),
                 "dataset_group priority sort should be the first sort key");
     }
 
+    /**
+     * The synonyms sub-fields must only be scored inside the title/description multi_match, a separate clause on them
+     * would score plain words twice
+     */
+    private static void assertSynonymsOnlyInsideMultiMatch(List<Query> should) {
+        should.stream()
+                .filter(q -> !q.isMultiMatch())
+                .forEach(q -> assertFalse(q.toString().contains(".synonyms"),
+                        "unexpected clause on a synonyms field: " + q));
+    }
+
     // The portal SEO pipeline requests these in bulk to build Dataset JSON-LD
     @Test
     public void seoPropertiesMapToStacFields() throws Exception {