feat(search): highlight the found line by meaning, without the query's words

SeforimApp/addons/semantic/enabled/kotlin/io/github/kdroidfilter/seforimapp/features/search/semantic/HybridSearchEngine.kt

@@ -102,7 +102,12 @@ internal class HybridSearchEngine(
     override fun rangeSnippet(
         text: String,
         ranges: List<IntRange>,
-    ): String = lexical.rangeSnippet(text, ranges)
+        lineStart: Int,
+        lineEnd: Int,
+    ): String = lexical.rangeSnippet(text, ranges, lineStart, lineEnd)
+
+    // A quoted phrase asks for those exact words: they alone are highlighted, as they alone are found
+    override fun highlightsByMeaning(query: String): Boolean = semanticConfigured && !SearchQueryParser.parse(query).hasExactPhrases
 
     override suspend fun passagesByMeaning(
         query: String,
@@ -244,22 +249,29 @@ internal class HybridSearchEngine(
             val withSnippets = if (snippets) lexical.attachSnippets(lexicalHits, query, near) else lexicalHits
             val lexicalById = withSnippets.associateBy { it.lineId }
             return entries.mapNotNull { entry ->
-                // Dense-only: a lexical snippet first, refined to the passage closest in meaning (refinedSnippets)
-                val hit = lexicalById[entry.lineId] ?: resolveLine(entry.lineId, query)
-                hit?.copy(score = entry.score)
+                val hit = lexicalById[entry.lineId] ?: denseHit(entry.lineId) ?: return@mapNotNull null
+                // No word in bold: the passage closest in meaning is highlighted once ready (refinedSnippets)
+                val snippet = if (snippets) lexical.rangeSnippet(hit.rawText, emptyList(), hit.lineStart, hit.lineEnd) else hit.snippet
+                hit.copy(score = entry.score, snippet = snippet)
             }
         }
 
+        // A dense-only line, its text cleaned as a lexical snippet source
+        private suspend fun denseHit(lineId: Long): LineHit? {
+            val hit = resolveLine(lineId, query) ?: return null
+            val text = SnippetSources.clean(hit.rawText)
+            return hit.copy(rawText = text, lineStart = 0, lineEnd = text.length)
+        }
+
         override suspend fun refinedSnippets(hits: List<LineHit>): Map<Long, String> {
-            val lexicalIds = fused.orEmpty().mapNotNullTo(HashSet()) { entry -> entry.lexical?.lineId }
-            val denseOnly = hits.filter { it.lineId !in lexicalIds && it.rawText.isNotBlank() }
-            if (denseOnly.isEmpty()) return emptyMap()
-            // Snippet text, as the lexical snippets: a line the query's words show in keeps its lexical snippet
-            val texts = denseOnly.map { SnippetSources.clean(it.rawText) }
-            val highlights = highlights(texts, query)
+            val shown = hits.filter { it.rawText.isNotBlank() && it.lineEnd > it.lineStart }
+            if (shown.isEmpty()) return emptyMap()
+            // The found line alone, not the neighbors its snippet shows around it
+            val highlights = highlights(shown.map { it.rawText.substring(it.lineStart, it.lineEnd) }, query)
             return buildMap {
-                denseOnly.forEachIndexed { i, hit ->
-                    if (highlights[i].byMeaning) put(hit.lineId, lexical.rangeSnippet(texts[i], highlights[i].ranges))
+                shown.forEachIndexed { i, hit ->
+                    val ranges = highlights[i].ranges.map { (it.first + hit.lineStart)..(it.last + hit.lineStart) }
+                    if (ranges.isNotEmpty()) put(hit.lineId, lexical.rangeSnippet(hit.rawText, ranges, hit.lineStart, hit.lineEnd))
                 }
             }
         }

SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/bookcontent/ui/panels/bookcontent/views/BookContentView.kt

@@ -1523,11 +1523,8 @@ private fun LineItem(
             smartQuery?.let { query ->
                 runSuspendCatching {
                     searchEngine
-                        .highlights(
-                            listOf(annotated.text),
-                            query,
-                            alwaysByMeaning = true,
-                        ).single()
+                        .highlights(listOf(annotated.text), query)
+                        .single()
                         .ranges
                 }.getOrNull()
             }

SeforimApp/src/jvmMain/kotlin/io/github/kdroidfilter/seforimapp/features/search/SearchResultScreen.kt

@@ -874,13 +874,14 @@ private fun PassagePreview(
                     // The whole width, as the book's text
                     modifier = Modifier.fillMaxWidth().padding(horizontal = 24.dp, vertical = 18.dp),
                 ) {
+                    val byMeaning = remember(query) { searchEngine.highlightsByMeaning(query) }
                     lines.forEach { line ->
                         val found = line.id == hit.lineId
                         val annotated = remember(line.id, textSize) { buildAnnotatedFromHtml(line.content, textSize) }
-                        // The search's own highlighting: the query's words, and in the found line, when none is there,
-                        // the passage closest in meaning (as its snippet)
+                        // The search's own highlighting (SearchEngine.highlights): by meaning, the passage closest in
+                        // meaning in the found line alone; else the query's words in every line
                         val ranges by produceState(
-                            searchEngine.highlightRanges(annotated.text, query),
+                            if (byMeaning) emptyList() else searchEngine.highlightRanges(annotated.text, query),
                             annotated,
                             query,
                             found,

SeforimApp/src/jvmTest/kotlin/io/github/kdroidfilter/seforimapp/features/search/SearchBench.kt

@@ -40,6 +40,8 @@ fun main(args: Array<String>) {
             engine.warmUp()
             println("warmUp ${ms(w)} ms")
         }
+        var refineNanos = 0L
+        var refinedHits = 0
         repeat(rounds) { round ->
             println("--- round $round")
             for (q in System.getenv("BENCH_QUERY")?.let { listOf(it) } ?: QUERIES) {
@@ -58,6 +60,10 @@ fun main(args: Array<String>) {
                 val s3 = System.nanoTime()
                 val refined = page?.let { session.refinedSnippets(it.hits) }.orEmpty()
                 val tRefine = ms(s3)
+                if (round > 0) {
+                    refineNanos += System.nanoTime() - s3
+                    refinedHits += page?.hits?.size ?: 0
+                }
                 session?.close()
                 println(
                     "%-34s total=%6d facets=%6d page=%6d refine=%5d(%2d) hits=%8d first=%s".format(
@@ -73,10 +79,34 @@ fun main(args: Array<String>) {
                 )
             }
         }
+        if (refinedHits > 0) println("refine %.2f ms/result over %d results".format(refineNanos / 1e6 / refinedHits, refinedHits))
     }
+    println(memory())
     engine.close()
 }
 
+/**
+ * Resident memory: all of it (with the mapped index pages, which the page cache lends and takes back), the process's own
+ * (anonymous: heap, native ONNX arenas), and the live heap after a GC.
+ */
+private fun memory(): String {
+    repeat(3) { System.gc() }
+    val heap =
+        java.lang.management.ManagementFactory
+            .getMemoryMXBean()
+            .heapMemoryUsage.used / 1_000_000
+    // Linux only: elsewhere, the heap alone
+    val status = runCatching { java.io.File("/proc/self/status").readLines() }.getOrDefault(emptyList())
+
+    fun kb(key: String) =
+        status
+            .firstOrNull { it.startsWith(key) }
+            ?.filter(Char::isDigit)
+            ?.toLong()
+            ?.div(1000)
+    return "memory: rss=${kb("VmRSS:")} MB anon=${kb("RssAnon:")} MB heap=$heap MB"
+}
+
 private fun ms(start: Long) = (System.nanoTime() - start) / 1_000_000
 
 private fun buildEngine(

gradle/libs.versions.toml

@@ -39,7 +39,7 @@ sqlDelight = "2.3.2"
 adaptiveNavigation = "1.2.0"
 kosherkotlin = "3.0.5"
 seforimSiddur = "0.2.2"
-seforimSemantic = "0.4.2"
+seforimSemantic = "0.5.0"
 zstdJni = "1.5.7-10"
 metro = "1.4.5"
 lucene = "10.5.2"