From 1cdd4b7d5e04a45b0aec4642db17f7fa56342f9c Mon Sep 17 00:00:00 2001 From: Casey West Date: Sat, 12 Sep 2026 02:54:28 -0400 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20feat(lint):=20add=20tag-hygiene=20b?= =?UTF-8?q?eside=20type-hygiene=20on=20a=20shared=20fold?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit lint warned that Concept and Concepts are one type spelled twice but had no opinion about Wine/wine or runbook/runbooks/run-book — even though tags is the other §4.1 field every consumer routes and filters on. The --tag scope filter and the /api/v1/search tag parameter match exactly and case-sensitively, so one tag spelled three ways is three disjoint result sets, and analyze's cluster detector counted the variants separately and missed the cluster. Extract the shared base fold (case, trim, single trailing 's') as canonFold; canonType is a thin wrapper over it, so type-hygiene stays byte-identical. The tag path gets its own fold: canonTag computes a separator-insensitive base key (-, _, space, and a letter-flanked / or . per UAX #29, so node.js folds with nodejs while .net stays apart from net), and newTagFolder adds a vocabulary-aware plural merge. The plural rules adapt KStem's plural() (Lucene KStemmer.java): the -is/-us/-ss and length guards, -ies/-es/-s handling, and Porter2's invariant list (news, atlas, cosmos, bias). A plural folds onto its stem only when that stem is itself an attested tag in the bundle, so a nonsense stem can never surface in a finding. This keeps new apart from news and op apart from ops, and merges patch/patches, policy/policies and process/processes, which the bare trailing-'s' rule missed. Add lintTagHygiene as a sibling of lintTypeHygiene: bundle-level, advisory, "tag-hygiene" in --json, --strict treats it like any finding; the message lists each variant with its per-node count, ordered most-common-first so the dominant spelling to normalize toward leads. analyze adopts the same bundle-scoped fold so it and lint agree on when two spellings are one tag, labels a case-only cluster with the lower-cased spelling (matching prior output), and dedups the node set per cluster, fixing a double-count where a tag was repeated within one node's list. The check never rejects a value (PRD §7.4 anti-taxonomy): it flags only spellings that fold to one value. The fold is a heuristic, so the help text tells a curator to review a finding before normalizing. Closes #166 --- cmd/lint.go | 7 + docs/commands/README.md | 2 + internal/okf/analyze.go | 57 +++- internal/okf/analyze_clusters_test.go | 90 ++++++ internal/okf/lint.go | 263 ++++++++++++++++- internal/okf/lint_taghygiene_test.go | 395 ++++++++++++++++++++++++++ 6 files changed, 797 insertions(+), 17 deletions(-) create mode 100644 internal/okf/analyze_clusters_test.go create mode 100644 internal/okf/lint_taghygiene_test.go diff --git a/cmd/lint.go b/cmd/lint.go index 86bfed0..887102c 100644 --- a/cmd/lint.go +++ b/cmd/lint.go @@ -80,6 +80,13 @@ func newLintCmd() *cobra.Command { "node with the same basename exists elsewhere — a moved or mistyped path (a defect), " + "distinct from a genuinely unwritten concept (a coverage gap, which analyze reports " + "advisorily and lint stays quiet on).\n\n" + + "tag-hygiene reports two or more `tags` spellings that fold to one canonical form. " + + "The fold is a heuristic: case, trimmed, separator-insensitive (so run-book, run_book " + + "and runbook are one value), plus a vocabulary-aware plural merge (a plural folds onto " + + "its singular only when that singular is itself a tag in the bundle). Like type-hygiene, " + + "it never rejects a value: it only reports spellings that fold together, so a curator " + + "can pick one. Because the fold is heuristic, review a finding before normalizing — it " + + "groups spellings that look like one tag, not a proof that they are.\n\n" + "--semantic adds similarity-driven checks (similar-but-unlinked pairs, nodes with " + "no semantic neighbors) by reading the index built by 'okfctl-search index build'. " + "Core only reads that index, so no embedding model is needed to lint.", diff --git a/docs/commands/README.md b/docs/commands/README.md index e21ad70..d2a0104 100644 --- a/docs/commands/README.md +++ b/docs/commands/README.md @@ -408,6 +408,8 @@ lint surfaces judgment-worthy curation findings, not spec-floor violations (use A broken-link finding reports an internal .md link that resolves to no node when a node with the same basename exists elsewhere — a moved or mistyped path (a defect), distinct from a genuinely unwritten concept (a coverage gap, which analyze reports advisorily and lint stays quiet on). +tag-hygiene reports two or more `tags` spellings that fold to one canonical form. The fold is a heuristic: case, trimmed, separator-insensitive (so run-book, run_book and runbook are one value), plus a vocabulary-aware plural merge (a plural folds onto its singular only when that singular is itself a tag in the bundle). Like type-hygiene, it never rejects a value: it only reports spellings that fold together, so a curator can pick one. Because the fold is heuristic, review a finding before normalizing — it groups spellings that look like one tag, not a proof that they are. + --semantic adds similarity-driven checks (similar-but-unlinked pairs, nodes with no semantic neighbors) by reading the index built by 'okfctl-search index build'. Core only reads that index, so no embedding model is needed to lint. ``` diff --git a/internal/okf/analyze.go b/internal/okf/analyze.go index 4d60b34..40e7eaa 100644 --- a/internal/okf/analyze.go +++ b/internal/okf/analyze.go @@ -604,28 +604,59 @@ func analyzeConnectivity(b *Bundle) ConnectivityReport { } func analyzeClusters(b *Bundle, opts AnalyzeOptions) []ClusterFinding { - byTag := map[string][]string{} + // Adopt lint's shared, bundle-scoped tag fold (newTagFolder: case- and + // separator-insensitive plus a vocabulary-aware plural merge) so analyze and + // lint agree on when two spellings are one tag. Group by canonical key; + // within a key, keep the DISTINCT node set (a node tagged both `wine` and + // `Wine` counts once) and the lower-cased variants so the reported label is a + // real spelling and matches main's output for a case-only cluster (wine, not + // Wine — uppercase would otherwise sort first and change text + --json). + fold := newTagFolder(b) + type cluster struct { + nodes map[string]bool + variants map[string]bool + } + byKey := map[string]*cluster{} for _, p := range sortedNodePaths(b) { for _, tag := range nodeTags(b.Nodes[p]) { - key := strings.ToLower(strings.TrimSpace(tag)) + key := fold(tag) if key == "" { continue } - byTag[key] = append(byTag[key], p) + c := byKey[key] + if c == nil { + c = &cluster{nodes: map[string]bool{}, variants: map[string]bool{}} + byKey[key] = c + } + c.nodes[p] = true + c.variants[strings.ToLower(strings.TrimSpace(tag))] = true } } var out []ClusterFinding - tags := make([]string, 0, len(byTag)) - for t := range byTag { - tags = append(tags, t) - } - sort.Strings(tags) - for _, t := range tags { - members := byTag[t] - if len(members) >= opts.ClusterMin { - sort.Strings(members) - out = append(out, ClusterFinding{Tag: t, Nodes: members}) + keys := make([]string, 0, len(byKey)) + for k := range byKey { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + c := byKey[k] + if len(c.nodes) < opts.ClusterMin { + continue + } + members := make([]string, 0, len(c.nodes)) + for p := range c.nodes { + members = append(members, p) + } + sort.Strings(members) + // Report the lexicographically-first lower-cased variant as the cluster + // tag: a real spelling from the corpus, deterministic across runs, and + // matching main's lower-cased label for a case-only cluster. + variants := make([]string, 0, len(c.variants)) + for v := range c.variants { + variants = append(variants, v) } + sort.Strings(variants) + out = append(out, ClusterFinding{Tag: variants[0], Nodes: members}) } return out } diff --git a/internal/okf/analyze_clusters_test.go b/internal/okf/analyze_clusters_test.go new file mode 100644 index 0000000..f5040b3 --- /dev/null +++ b/internal/okf/analyze_clusters_test.go @@ -0,0 +1,90 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package okf + +import "testing" + +// analyze adopts the shared tag fold, so three spellings of one tag over three +// nodes form a single cluster that reaches the default --cluster-min 3 — where +// today's case-only fold leaves them as three 1-node tags, all invisible. +func TestAnalyzeClusters_FoldedVariantsFormOneCluster(t *testing.T) { + b := mkLintBundle(t, map[string]string{ + "index.md": "---\ntype: Index\ntitle: Index\n---\n\n# Index\n\n- [A](a.md)\n- [B](b.md)\n- [C](c.md)\n", + "a.md": lintDocTags("Concept", "A", []string{"runbook"}, "Body."), + "b.md": lintDocTags("Concept", "B", []string{"runbooks"}, "Body."), + "c.md": lintDocTags("Concept", "C", []string{"run-book"}, "Body."), + }) + clusters := analyzeClusters(b, DefaultAnalyzeOptions()) + found := false + var nodeCount int + for _, c := range clusters { + // The label is the lexicographically-first lower-cased variant; + // run-book sorts before runbook/runbooks. + if c.Tag == "run-book" { + found = true + nodeCount = len(c.Nodes) + break + } + } + if !found { + t.Fatalf("expected a folded runbook cluster labelled run-book at cluster-min 3, got clusters=%+v", clusters) + } + if nodeCount != 3 { + t.Fatalf("folded runbook cluster should span all 3 nodes, got %d", nodeCount) + } +} + +// Case folding already merged Wine/wine before this change; that behavior must +// survive the fold swap, AND the label must stay the lower-cased `wine` that +// main emits — not an upper-cased variant that sorts first. Pins the exact Tag, +// which is the regression guard thread 2 asked for. +func TestAnalyzeClusters_CaseVariantsStillMerge(t *testing.T) { + b := mkLintBundle(t, map[string]string{ + "index.md": "---\ntype: Index\ntitle: Index\n---\n\n# Index\n\n- [A](a.md)\n- [B](b.md)\n- [C](c.md)\n", + "a.md": lintDocTags("Concept", "A", []string{"Wine"}, "Body."), + "b.md": lintDocTags("Concept", "B", []string{"wine"}, "Body."), + "c.md": lintDocTags("Concept", "C", []string{"WINE"}, "Body."), + }) + clusters := analyzeClusters(b, DefaultAnalyzeOptions()) + found := false + for _, c := range clusters { + if c.Tag == "wine" && len(c.Nodes) == 3 { + found = true + } + // Guard against the upper-cased label regression: no case-variant label. + if c.Tag == "Wine" || c.Tag == "WINE" { + t.Fatalf("case-only cluster must be labelled lower-cased (wine), got %q", c.Tag) + } + } + if !found { + t.Fatalf("Wine/wine/WINE must still merge into one 3-node cluster labelled %q, got %+v", "wine", clusters) + } +} + +// Distinct tags must not be merged by the fold (negative control for analyze). +func TestAnalyzeClusters_DistinctTagsNotMerged(t *testing.T) { + b := mkLintBundle(t, map[string]string{ + "index.md": "---\ntype: Index\ntitle: Index\n---\n\n# Index\n\n- [A](a.md)\n- [B](b.md)\n- [C](c.md)\n", + "a.md": lintDocTags("Concept", "A", []string{"oauth1"}, "Body."), + "b.md": lintDocTags("Concept", "B", []string{"oauth2"}, "Body."), + "c.md": lintDocTags("Concept", "C", []string{"incident"}, "Body."), + }) + clusters := analyzeClusters(b, DefaultAnalyzeOptions()) + for _, c := range clusters { + if len(c.Nodes) >= 3 { + t.Fatalf("distinct tags must not be folded into a cluster, got %+v", c) + } + } +} diff --git a/internal/okf/lint.go b/internal/okf/lint.go index 0080a1e..c16a872 100644 --- a/internal/okf/lint.go +++ b/internal/okf/lint.go @@ -27,7 +27,7 @@ import ( // validate Finding (a spec-floor violation), a lint finding is curation // guidance — never a format failure. type LintFinding struct { - Check string `json:"check"` // "orphan" | "missing-xref" | "coverage-gap" | "type-hygiene" | "broken-link" | "status-lifecycle" | "spec-version" + Check string `json:"check"` // "orphan" | "missing-xref" | "coverage-gap" | "type-hygiene" | "tag-hygiene" | "broken-link" | "status-lifecycle" | "spec-version" Path string `json:"path"` // node path the finding is about ("" for bundle-level findings) Message string `json:"message"` } @@ -56,6 +56,7 @@ func Lint(b *Bundle, opts LintOptions) []LintFinding { findings = append(findings, lintBrokenLinks(b)...) findings = append(findings, lintCoverageGaps(b, threshold)...) findings = append(findings, lintTypeHygiene(b)...) + findings = append(findings, lintTagHygiene(b)...) findings = append(findings, lintStatusLifecycle(b)...) findings = append(findings, lintSpecVersion(b)...) @@ -655,12 +656,266 @@ func lintTypeHygiene(b *Bundle) []LintFinding { return out } -// canonType folds a type value to case-insensitive + singular (drop a single -// trailing 's') for near-duplicate grouping. -func canonType(s string) string { +// lintTagHygiene warns when two or more distinct `tags` values fold to the same +// canonical form, which usually signals accidental drift on the other §4.1 +// cross-cutting field every consumer routes and filters on. The `--tag` scope +// filter and the /api/v1/search `tag` parameter match exactly and +// case-sensitively, so one tag spelled three ways is three disjoint result sets; +// this surfaces that drift the same way type-hygiene does for `type`. +// +// The fold is the bundle-scoped tagFolder: case- and separator-insensitive, plus +// a vocabulary-aware plural merge (see newTagFolder). Anti-taxonomy stands (PRD +// §7.4): it flags only spellings that fold to ONE value, never two genuinely +// distinct values, and it is a warning class — never a validate rejection. The +// message lists each variant with its per-node count, variants ordered +// most-common-first (the dominant spelling to normalize toward leads), ties +// broken alphabetically. +func lintTagHygiene(b *Bundle) []LintFinding { + fold := newTagFolder(b) + + // canonical -> raw tag value -> set of node paths carrying it. + groups := map[string]map[string]map[string]bool{} + for _, n := range b.Nodes { + seen := map[string]bool{} // dedupe a repeated tag within one node + for _, raw := range n.Tags() { + raw = strings.TrimSpace(raw) + if raw == "" || seen[raw] { + continue + } + seen[raw] = true + c := fold(raw) + if c == "" { + continue + } + if groups[c] == nil { + groups[c] = map[string]map[string]bool{} + } + if groups[c][raw] == nil { + groups[c][raw] = map[string]bool{} + } + groups[c][raw][n.Path] = true + } + } + + var out []LintFinding + canons := make([]string, 0, len(groups)) + for c := range groups { + canons = append(canons, c) + } + sort.Strings(canons) + for _, c := range canons { + if len(groups[c]) < 2 { + continue + } + variants := make([]string, 0, len(groups[c])) + for v := range groups[c] { + variants = append(variants, v) + } + // Order variants most-common-first so the dominant spelling to + // normalize toward leads the message; ties broken alphabetically for a + // deterministic order. + sort.Slice(variants, func(i, j int) bool { + ni, nj := len(groups[c][variants[i]]), len(groups[c][variants[j]]) + if ni != nj { + return ni > nj + } + return variants[i] < variants[j] + }) + parts := make([]string, 0, len(variants)) + for _, v := range variants { + n := len(groups[c][v]) + parts = append(parts, fmt.Sprintf("%s (%d %s)", v, n, pluralNodes(n))) + } + out = append(out, LintFinding{ + Check: "tag-hygiene", + Path: "", + Message: fmt.Sprintf("tag-hygiene: near-duplicate tag values likely refer to one tag: %s", strings.Join(parts, ", ")), + }) + } + return out +} + +// pluralNodes returns "node" for a count of 1, "nodes" otherwise. +func pluralNodes(n int) string { + if n == 1 { + return "node" + } + return "nodes" +} + +// canonFold is the shared base fold: case-insensitive, trimmed, and singular +// (drop a single trailing 's'). It is the one place `type` and `tags` +// near-duplicate grouping agree on, so the tool can never apply two different +// folds to the same kind of §4.1 value. It is separator-SENSITIVE by design — +// separator-insensitivity is a tag-only layer (see canonTag) and must not leak +// into the type path. +func canonFold(s string) string { c := strings.ToLower(strings.TrimSpace(s)) if len(c) > 1 && strings.HasSuffix(c, "s") { c = c[:len(c)-1] } return c } + +// canonType folds a `type` value for near-duplicate grouping. It is exactly the +// shared base fold (case + trim + single trailing 's'); it deliberately does NOT +// add tag's separator or plural rules, so type-hygiene behavior is unchanged by +// the shared extraction. +func canonType(s string) string { return canonFold(s) } + +// canonTag computes a tag's base key: lower-cased, trimmed, and +// separator-insensitive. It does NOT strip plurals — pluralization is a +// vocabulary-aware, bundle-scoped decision made by newTagFolder, because a +// plural should only fold onto a stem the corpus actually attests. Two tags that +// differ only in case or in how a compound is joined share a base key: +// +// Run Book / run-book / run_book / runbook -> runbook +// +// Separators are '-', '_', space, and — per Unicode UAX #29 — a '/' or '.' with +// a letter on BOTH sides (node.js == nodejs, ci/cd == cicd), so a leading '.' +// stays word-internal-free (.net does not fold onto net). +func canonTag(s string) string { + c := strings.ToLower(strings.TrimSpace(s)) + c = tagSeparators.Replace(c) + c = dropLetterFlankedSeparators(c, '/', '.') + return c +} + +// tagSeparators are the always-intra-tag separators (§4.1 tags are free-form): a +// hyphen, underscore, or space. Two tags that differ only in how a compound is +// joined (home-lab vs homelab, run book vs runbook) are one value spelled two +// ways. +var tagSeparators = strings.NewReplacer("-", "", "_", "", " ", "") + +// dropLetterFlankedSeparators removes each listed rune only when a letter sits on +// both sides of it (UAX #29 word boundaries: a letter-flanked '.' or '/' is +// word-internal). This folds node.js->nodejs and ci/cd->cicd while leaving a +// leading/trailing or non-letter-flanked separator in place (.net stays apart +// from net). +func dropLetterFlankedSeparators(s string, seps ...rune) string { + isSep := func(r rune) bool { + for _, x := range seps { + if r == x { + return true + } + } + return false + } + rs := []rune(s) + var out []rune + for i, r := range rs { + if isSep(r) && i > 0 && i < len(rs)-1 && + isLetter(rs[i-1]) && isLetter(rs[i+1]) { + continue + } + out = append(out, r) + } + return string(out) +} + +func isLetter(r rune) bool { + return (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') +} + +// pluralInvariants are forms that LOOK like plurals but are not — Porter2's +// exceptional set (the Snowball English stemmer's invariant forms). A tag in this +// set is never plural-stripped, so `news` stays `news` (not `new`) and `bias` +// stays `bias` (not `bia`). See https://snowballstem.org/algorithms/english/. +var pluralInvariants = map[string]bool{ + "news": true, "atlas": true, "cosmos": true, "bias": true, + "caress": true, "gas": true, +} + +// kstemPlural returns the de-pluralized candidate stems for a base-key tag, most +// specific first, per the plural() rules of KStem (the UMass CIIR stemmer shipped +// as Lucene's KStemmer.java, plural()). It is a STEM GENERATOR, not a decision: +// the caller folds onto a candidate only if the bundle attests it (see +// newTagFolder), so a nonsense stem is never reported. Guards: +// +// - letters-only words only (k8s, v1, oauth2 are left alone); +// - length > 2; +// - a plain final 's' is stripped only when length > 3 and the word does not +// end in 'ss' or 'ous' (keeps aws, ops, ios, dns, class, famous); +// - '-ies' -> try '-ie' then '-y'; '-es' -> try dropping 's' then 'es', +// skipping the '-e' candidate after a double 's' (crosses, processes); +// - Porter2 invariants (news, atlas, cosmos, bias, …) are never stripped. +// +// This is an adaptation of KStem's plural(): KStem decides between candidates +// with an English dictionary; here the bundle's own tag vocabulary is the +// dictionary, which is exactly what a near-duplicate finding needs (two attested +// spellings) and means no out-of-vocabulary stem can ever surface. +func kstemPlural(w string) []string { + if len(w) <= 2 || !lettersOnly(w) || pluralInvariants[w] { + return nil + } + if !strings.HasSuffix(w, "s") { + return nil + } + switch { + case strings.HasSuffix(w, "ies"): + // calories -> calorie (try) then category: policies -> policy. + stem := w[:len(w)-3] + return []string{stem + "ie", stem + "y"} + case strings.HasSuffix(w, "es"): + stem := w[:len(w)-2] // drop "es" + var cands []string + // Try dropping just the 's' (…e) UNLESS the char before "es" is itself + // 's' (crosses -> cross, not crosse; processes -> process, not processe). + doubleS := len(stem) > 0 && stem[len(stem)-1] == 's' + if !doubleS { + cands = append(cands, w[:len(w)-1]) // drop 's' -> …e + } + cands = append(cands, stem) // drop 'es' + return cands + default: + // plain trailing 's': strip only when len>3, not 'ss', not 'ous'. + if len(w) > 3 && !strings.HasSuffix(w, "ss") && !strings.HasSuffix(w, "ous") { + return []string{w[:len(w)-1]} + } + return nil + } +} + +func lettersOnly(s string) bool { + if s == "" { + return false + } + for _, r := range s { + if !isLetter(r) { + return false + } + } + return true +} + +// newTagFolder returns a bundle-scoped fold that maps a raw tag to its canonical +// form. The fold is two-pass and VOCABULARY-AWARE, which is why it takes the +// bundle: a plural is folded onto its stem only when that stem is itself an +// attested base key in this bundle. Pass 1 collects every attested base key +// (canonTag of every tag on every node). Pass 2 resolves a tag by walking its +// KStem plural candidates and folding onto the first one the bundle attests; +// absent any attested candidate the base key is canonical as-is. This keeps the +// anti-taxonomy posture (nonsense stems can never be reported) and makes lint and +// analyze agree on when two spellings are one tag. +func newTagFolder(b *Bundle) func(string) string { + attested := map[string]bool{} + for _, n := range b.Nodes { + for _, raw := range n.Tags() { + if base := canonTag(raw); base != "" { + attested[base] = true + } + } + } + return func(raw string) string { + base := canonTag(raw) + if base == "" { + return "" + } + for _, cand := range kstemPlural(base) { + if attested[cand] { + return cand + } + } + return base + } +} diff --git a/internal/okf/lint_taghygiene_test.go b/internal/okf/lint_taghygiene_test.go new file mode 100644 index 0000000..b93e38e --- /dev/null +++ b/internal/okf/lint_taghygiene_test.go @@ -0,0 +1,395 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package okf + +import ( + "fmt" + "strings" + "testing" +) + +// lintDocTags builds a node with a type, title, and a YAML tags list, so tests +// can exercise the tag-hygiene fold the way the real corpus declares tags. +func lintDocTags(typ, title string, tags []string, body string) string { + tl := "[" + strings.Join(tags, ", ") + "]" + return "---\ntype: " + typ + "\ntitle: " + title + "\ntags: " + tl + "\n---\n\n# " + title + "\n\n" + body + "\n" +} + +// The base fold is the contract. canonFold is the shared base fold used by +// type-hygiene (case, trim, single trailing 's'); it is UNCHANGED by the tag +// rework, which lives entirely on the tag path. +func TestCanonFold_SharedBaseFold(t *testing.T) { + cases := []struct{ in, want string }{ + {"Concept", "concept"}, + {"concept", "concept"}, + {"Concepts", "concept"}, // single trailing 's' dropped + {" Wine ", "wine"}, // trim + {"WINE", "wine"}, // case + {"runbooks", "runbook"}, // trailing 's' + {"run-book", "run-book"}, // base fold does NOT strip separators + {"s", "s"}, // len==1 guard: don't strip the only char + {"gas", "ga"}, // trailing 's' dropped even mid-word (existing type behavior) + {"", ""}, + } + for _, c := range cases { + if got := canonFold(c.in); got != c.want { + t.Errorf("canonFold(%q) = %q, want %q", c.in, got, c.want) + } + } +} + +// canonTag is the tag BASE KEY: lower, trim, separator-insensitive. It does NOT +// strip plurals — that is a vocabulary-aware, bundle-scoped decision (see +// TestTagFolder_PluralIsVocabularyAware). Separators are '-', '_', space, and a +// letter-flanked '/' or '.' (UAX #29). A leading '.' is NOT a separator. +func TestCanonTag_BaseKey(t *testing.T) { + cases := []struct{ in, want string }{ + {"Wine", "wine"}, + {"wine", "wine"}, + {"runbook", "runbook"}, + {"run-book", "runbook"}, // hyphen stripped + {"run_book", "runbook"}, // underscore stripped + {"Run Book", "runbook"}, // space stripped + case + {"Play Book", "playbook"}, // space stripped + case + {"home-lab", "homelab"}, // separator-only collision + {"homelab", "homelab"}, + {"node.js", "nodejs"}, // letter-flanked '.' is a separator + {"nodejs", "nodejs"}, + {"ci/cd", "cicd"}, // letter-flanked '/' is a separator + {"cicd", "cicd"}, + {".net", ".net"}, // leading '.' is NOT letter-flanked: stays apart from net + {"net", "net"}, + // Base key leaves plurals alone (no bundle context here). + {"runbooks", "runbooks"}, + {"design-patterns", "designpatterns"}, + {"hormesis", "hormesis"}, // the -is/-us/-ss class is untouched + {"css", "css"}, + {"gpt-oss", "gptoss"}, + } + for _, c := range cases { + if got := canonTag(c.in); got != c.want { + t.Errorf("canonTag(%q) = %q, want %q", c.in, got, c.want) + } + } +} + +// The plural merge is an adaptation of KStem's plural() (Lucene KStemmer.java): +// a plural folds onto its stem ONLY when that stem is itself attested in the +// bundle. The attested set is supplied, newTagFolder is built from it, and the +// fold is exercised through it. +func TestTagFolder_PluralIsVocabularyAware(t *testing.T) { + attested := []string{ + "patch", "patches", // -es, drop "es" -> patch + "policy", "policies", // -ies -> policy + "process", "processes", // -es after double-s -> process (not processe) + "anti-pattern", "anti-patterns", // plain 's' + separator + "runbook", "runbooks", // plain 's' + "category", "categories", // -ies -> category + // Negatives that must NOT fold even though they look plural-ish: + "new", "news", // Porter2 invariant: news never strips -> stays apart from new + "op", "ops", // len <= 3: ops never strips -> stays apart from op + } + files := map[string]string{} + idx := "---\ntype: Index\ntitle: Index\n---\n\n# Index\n\n" + for i, tag := range attested { + name := fmt.Sprintf("n%02d.md", i) + files[name] = lintDocTags("Concept", name, []string{tag}, "Body.") + idx += fmt.Sprintf("- [%s](%s)\n", name, name) + } + files["index.md"] = idx + b := mkLintBundle(t, files) + fold := newTagFolder(b) + + same := func(a, c string) { + t.Helper() + if fold(a) != fold(c) { + t.Errorf("expected %q and %q to fold together, got %q vs %q", a, c, fold(a), fold(c)) + } + } + diff := func(a, c string) { + t.Helper() + if fold(a) == fold(c) { + t.Errorf("expected %q and %q to stay distinct, both folded to %q", a, c, fold(a)) + } + } + + // Plurals fold onto an attested stem. + same("patch", "patches") + same("policy", "policies") + same("process", "processes") + same("runbook", "runbooks") + same("anti-pattern", "anti-patterns") + same("category", "categories") + + // Distinct tags do NOT merge. + diff("new", "news") // Porter2 invariant + diff("op", "ops") // length guard +} + +// kstemPlural is the pure stem generator (vocabulary decision is separate). +func TestKstemPlural_Guards(t *testing.T) { + cases := []struct { + in string + want []string + }{ + {"runbooks", []string{"runbook"}}, // plain 's' + {"patches", []string{"patche", "patch"}}, // -es, not double-s: try "patche" then "patch" + {"policies", []string{"policie", "policy"}}, + {"processes", []string{"process"}}, // double-s -> skip the "processe" candidate + {"aws", nil}, // len 3: not > 3 + {"ops", nil}, // len 3 + {"ios", nil}, // len 3 + {"dns", nil}, // len 3 + {"class", nil}, // double-s (penult 's') + {"famous", nil}, // -ous + {"news", nil}, // Porter2 invariant + {"k8s", nil}, // not letters-only + {"runbook", nil}, // no trailing 's' + {"", nil}, + } + for _, c := range cases { + got := kstemPlural(c.in) + if len(got) != len(c.want) { + t.Errorf("kstemPlural(%q) = %v, want %v", c.in, got, c.want) + continue + } + // "patches" generates ["patche","patch"]; only assert "patch" is present + // for the simple cases to keep the table readable. + for _, w := range c.want { + found := false + for _, g := range got { + if g == w { + found = true + } + } + if !found { + t.Errorf("kstemPlural(%q) = %v, want it to contain %q", c.in, got, w) + } + } + } +} + +// canonTag must NOT change the type path. Types still fold with canonType +// (== canonFold), which is separator- AND plural-insensitive only in the base +// sense. A hyphenated and an unhyphenated type do NOT collapse under the type +// fold, and the tag's plural vocabulary rule never touches types. +func TestCanonType_UnchangedByTagRules(t *testing.T) { + if canonType("run-book") == canonType("runbook") { + t.Fatalf("type fold must stay separator-sensitive: run-book and runbook must NOT collapse") + } + if canonType("Concepts") != canonType("concept") { + t.Fatalf("type fold must still be case+plural: Concepts and concept must collapse") + } +} + +// POSITIVE controls: each fires exactly one tag-hygiene finding. Covers case, +// plain plural, -es/-ies plurals, hyphen/space separators, and letter-flanked +// '/' and '.' separators. +func TestLint_TagHygiene_Positive(t *testing.T) { + cases := []struct { + name string + tagA string + tagB string + }{ + {"case", "Wine", "wine"}, + {"plural", "runbook", "runbooks"}, + {"plural-es", "patch", "patches"}, + {"plural-ies", "policy", "policies"}, + {"plural-double-s", "process", "processes"}, + {"separator-hyphen", "run-book", "runbook"}, + {"separator-space", "Play Book", "playbook"}, + {"separator-slash", "ci/cd", "cicd"}, + {"separator-dot", "node.js", "nodejs"}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + b := mkLintBundle(t, map[string]string{ + "index.md": "---\ntype: Index\ntitle: Index\n---\n\n# Index\n\n- [A](a.md)\n- [B](b.md)\n", + "a.md": lintDocTags("Concept", "A", []string{c.tagA}, "Body."), + "b.md": lintDocTags("Concept", "B", []string{c.tagB}, "Body."), + }) + th := findingsFor(Lint(b, LintOptions{}), "tag-hygiene") + if len(th) != 1 { + t.Fatalf("%s: expected one tag-hygiene finding for %q/%q, got %+v", c.name, c.tagA, c.tagB, th) + } + if th[0].Path != "" { + t.Fatalf("%s: tag-hygiene must be bundle-level (Path==\"\"), got %q", c.name, th[0].Path) + } + m := lc(th[0].Message) + if !strings.Contains(m, lc(c.tagA)) || !strings.Contains(m, lc(c.tagB)) { + t.Fatalf("%s: message must name both variants: %q", c.name, th[0].Message) + } + }) + } +} + +// The message pins the FULL string: variants ordered most-common-first (the +// dominant spelling to normalize toward leads), ties broken alphabetically, with +// per-node counts. runbook x2 (a, c) dominates runbooks x1 (b) and run-book x1 +// (d); the two one-node variants tie and sort alphabetically (run-book < runbook). +func TestLint_TagHygiene_MessageFullStringMostCommonFirst(t *testing.T) { + b := mkLintBundle(t, map[string]string{ + "index.md": "---\ntype: Index\ntitle: Index\n---\n\n# Index\n\n- [A](a.md)\n- [B](b.md)\n- [C](c.md)\n- [D](d.md)\n", + "a.md": lintDocTags("Concept", "A", []string{"runbook"}, "Body."), + "b.md": lintDocTags("Concept", "B", []string{"runbooks"}, "Body."), + "c.md": lintDocTags("Concept", "C", []string{"runbook"}, "Body."), + "d.md": lintDocTags("Concept", "D", []string{"run-book"}, "Body."), + }) + th := findingsFor(Lint(b, LintOptions{}), "tag-hygiene") + if len(th) != 1 { + t.Fatalf("expected one folded tag-hygiene finding, got %+v", th) + } + const want = "tag-hygiene: near-duplicate tag values likely refer to one tag: runbook (2 nodes), run-book (1 node), runbooks (1 node)" + if th[0].Message != want { + t.Fatalf("message mismatch:\n got: %q\nwant: %q", th[0].Message, want) + } +} + +// NEGATIVE controls: legitimately similar-looking tags stay silent. The +// new/news and op/ops pairs are LOAD-BEARING — they collide under a bare +// trailing-'s' rule and only stay silent because of the Porter2 invariant and +// the length guard. See TestLint_TagHygiene_NegativeControlsAreLoadBearing. +func TestLint_TagHygiene_Negative(t *testing.T) { + cases := []struct { + name string + tagA string + tagB string + }{ + {"version-digit", "v1", "v2"}, + {"oauth-digit", "oauth1", "oauth2"}, + {"distinct-words", "oncall", "incident"}, + {"porter2-invariant", "new", "news"}, // news never strips -> distinct from new + {"length-guard", "op", "ops"}, // ops len 3 -> never strips + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + b := mkLintBundle(t, map[string]string{ + "index.md": "---\ntype: Index\ntitle: Index\n---\n\n# Index\n\n- [A](a.md)\n- [B](b.md)\n", + "a.md": lintDocTags("Concept", "A", []string{c.tagA}, "Body."), + "b.md": lintDocTags("Concept", "B", []string{c.tagB}, "Body."), + }) + if n := len(findingsFor(Lint(b, LintOptions{}), "tag-hygiene")); n != 0 { + t.Fatalf("%s: %q/%q are distinct; expected 0 tag-hygiene findings, got %d", c.name, c.tagA, c.tagB, n) + } + }) + } +} + +// Proof the new/news and op/ops negative controls are LOAD-BEARING: under the +// OLD bare trailing-'s' fold (lower+trim+drop one 's'), both pairs collide, so a +// test that passes them only means something because the new fold deliberately +// keeps them apart. This reconstructs the old fold inline and asserts it WOULD +// have merged them — so a regression back to the bare rule turns the negative +// control red. +func TestLint_TagHygiene_NegativeControlsAreLoadBearing(t *testing.T) { + oldBareFold := func(s string) string { + c := strings.ToLower(strings.TrimSpace(s)) + if len(c) > 1 && strings.HasSuffix(c, "s") { + c = c[:len(c)-1] + } + return c + } + for _, p := range [][2]string{{"new", "news"}, {"op", "ops"}} { + if oldBareFold(p[0]) != oldBareFold(p[1]) { + t.Fatalf("negative control %q/%q is NOT load-bearing: the old bare fold did not merge them (%q vs %q)", + p[0], p[1], oldBareFold(p[0]), oldBareFold(p[1])) + } + } +} + +// A single tag spread over many nodes is not drift — it must stay silent. +func TestLint_TagHygiene_SingleTagManyNodesNoFinding(t *testing.T) { + b := mkLintBundle(t, map[string]string{ + "index.md": "---\ntype: Index\ntitle: Index\n---\n\n# Index\n\n- [A](a.md)\n- [B](b.md)\n- [C](c.md)\n", + "a.md": lintDocTags("Concept", "A", []string{"wine"}, "Body."), + "b.md": lintDocTags("Concept", "B", []string{"wine"}, "Body."), + "c.md": lintDocTags("Concept", "C", []string{"wine"}, "Body."), + }) + if n := len(findingsFor(Lint(b, LintOptions{}), "tag-hygiene")); n != 0 { + t.Fatalf("one tag on many nodes is not drift; expected 0 tag-hygiene findings, got %d", n) + } +} + +// The -is/-us/-ss digraph class (real-corpus measurement: 65 such tags, zero +// collisions) must each be silent. Two mechanisms keep them silent: 'ss' and +// 'ous' are never stripped at all (css, gpt-oss), and an '-is'/'-us' word only +// folds if its stripped stem is ATTESTED in the bundle — which it is not here, +// nor in the real corpus. Each is paired with an unrelated sibling so the only +// way a finding could fire is an over-eager fold; the vocabulary guard prevents +// it without a stop-list. +func TestLint_TagHygiene_DigraphClassSilent(t *testing.T) { + digraphs := []string{"hormesis", "hysteresis", "css", "freshness", "blast-radius", "gpt-oss"} + for _, d := range digraphs { + t.Run(d, func(t *testing.T) { + b := mkLintBundle(t, map[string]string{ + "index.md": "---\ntype: Index\ntitle: Index\n---\n\n# Index\n\n- [A](a.md)\n- [B](b.md)\n", + "a.md": lintDocTags("Concept", "A", []string{d}, "Body."), + "b.md": lintDocTags("Concept", "B", []string{"incident"}, "Body."), + }) + if n := len(findingsFor(Lint(b, LintOptions{}), "tag-hygiene")); n != 0 { + t.Fatalf("digraph tag %q must not collide with an unrelated sibling; got %d findings", d, n) + } + }) + } +} + +// The vocabulary guard is what keeps the '-is'/'-us' class quiet: hormesis WOULD +// strip to hormesi under the plain-'s' rule, but only folds if hormesi is an +// attested tag. This proves the stem generator does produce the candidate (so +// the guard is doing real work) AND that a bundle without the stem never folds. +func TestLint_TagHygiene_IsUsClassGuardedByVocabulary(t *testing.T) { + // Generator DOES offer the stripped candidate (plain 's', len>3, not ss/ous). + if got := kstemPlural("hormesis"); len(got) != 1 || got[0] != "hormesi" { + t.Fatalf("kstemPlural(hormesis) = %v, want [hormesi]", got) + } + // But with no attested "hormesi" in the bundle, the two stay distinct. + b := mkLintBundle(t, map[string]string{ + "index.md": "---\ntype: Index\ntitle: Index\n---\n\n# Index\n\n- [A](a.md)\n- [B](b.md)\n", + "a.md": lintDocTags("Concept", "A", []string{"hormesis"}, "Body."), + "b.md": lintDocTags("Concept", "B", []string{"incident"}, "Body."), + }) + if n := len(findingsFor(Lint(b, LintOptions{}), "tag-hygiene")); n != 0 { + t.Fatalf("hormesis must not fold without an attested hormesi stem; got %d findings", n) + } +} + +// The shared-fold extraction must leave type-hygiene byte-identical. This pins +// the exact message string the check emitted before the refactor, proving the +// type path did not move when canonType became a thin canonFold wrapper. +func TestLint_TypeHygiene_MessageByteIdenticalAfterFoldExtraction(t *testing.T) { + b := mkLintBundle(t, map[string]string{ + "index.md": "---\ntype: Index\ntitle: Index\n---\n\n# Index\n\n- [A](a.md)\n- [B](b.md)\n", + "a.md": lintDoc("Concept", "A", "Body."), + "b.md": lintDoc("Concepts", "B", "Body."), + }) + th := findingsFor(Lint(b, LintOptions{}), "type-hygiene") + if len(th) != 1 { + t.Fatalf("expected one type-hygiene finding, got %+v", th) + } + const want = "type-hygiene: near-duplicate type values likely refer to one type: Concept, Concepts" + if th[0].Message != want { + t.Fatalf("type-hygiene message changed by fold extraction:\n got: %q\nwant: %q", th[0].Message, want) + } + // And a separator-only type pair must NOT collapse (types stay + // separator-sensitive; the tag-only rules must not leak into the type path). + b2 := mkLintBundle(t, map[string]string{ + "index.md": "---\ntype: Index\ntitle: Index\n---\n\n# Index\n\n- [A](a.md)\n- [B](b.md)\n", + "a.md": lintDoc("run-book", "A", "Body."), + "b.md": lintDoc("runbook", "B", "Body."), + }) + if n := len(findingsFor(Lint(b2, LintOptions{}), "type-hygiene")); n != 0 { + t.Fatalf("type-hygiene must stay separator-sensitive: run-book/runbook types must NOT fold, got %d findings", n) + } +}