diff --git a/cmd/lint.go b/cmd/lint.go index 86bfed0..887102c 100644 --- a/cmd/lint.go +++ b/cmd/lint.go @@ -80,6 +80,13 @@ func newLintCmd() *cobra.Command { "node with the same basename exists elsewhere — a moved or mistyped path (a defect), " + "distinct from a genuinely unwritten concept (a coverage gap, which analyze reports " + "advisorily and lint stays quiet on).\n\n" + + "tag-hygiene reports two or more `tags` spellings that fold to one canonical form. " + + "The fold is a heuristic: case, trimmed, separator-insensitive (so run-book, run_book " + + "and runbook are one value), plus a vocabulary-aware plural merge (a plural folds onto " + + "its singular only when that singular is itself a tag in the bundle). Like type-hygiene, " + + "it never rejects a value: it only reports spellings that fold together, so a curator " + + "can pick one. Because the fold is heuristic, review a finding before normalizing — it " + + "groups spellings that look like one tag, not a proof that they are.\n\n" + "--semantic adds similarity-driven checks (similar-but-unlinked pairs, nodes with " + "no semantic neighbors) by reading the index built by 'okfctl-search index build'. " + "Core only reads that index, so no embedding model is needed to lint.", diff --git a/docs/commands/README.md b/docs/commands/README.md index e21ad70..d2a0104 100644 --- a/docs/commands/README.md +++ b/docs/commands/README.md @@ -408,6 +408,8 @@ lint surfaces judgment-worthy curation findings, not spec-floor violations (use A broken-link finding reports an internal .md link that resolves to no node when a node with the same basename exists elsewhere — a moved or mistyped path (a defect), distinct from a genuinely unwritten concept (a coverage gap, which analyze reports advisorily and lint stays quiet on). +tag-hygiene reports two or more `tags` spellings that fold to one canonical form. The fold is a heuristic: case, trimmed, separator-insensitive (so run-book, run_book and runbook are one value), plus a vocabulary-aware plural merge (a plural folds onto its singular only when that singular is itself a tag in the bundle). Like type-hygiene, it never rejects a value: it only reports spellings that fold together, so a curator can pick one. Because the fold is heuristic, review a finding before normalizing — it groups spellings that look like one tag, not a proof that they are. + --semantic adds similarity-driven checks (similar-but-unlinked pairs, nodes with no semantic neighbors) by reading the index built by 'okfctl-search index build'. Core only reads that index, so no embedding model is needed to lint. ``` diff --git a/internal/okf/analyze.go b/internal/okf/analyze.go index 4d60b34..40e7eaa 100644 --- a/internal/okf/analyze.go +++ b/internal/okf/analyze.go @@ -604,28 +604,59 @@ func analyzeConnectivity(b *Bundle) ConnectivityReport { } func analyzeClusters(b *Bundle, opts AnalyzeOptions) []ClusterFinding { - byTag := map[string][]string{} + // Adopt lint's shared, bundle-scoped tag fold (newTagFolder: case- and + // separator-insensitive plus a vocabulary-aware plural merge) so analyze and + // lint agree on when two spellings are one tag. Group by canonical key; + // within a key, keep the DISTINCT node set (a node tagged both `wine` and + // `Wine` counts once) and the lower-cased variants so the reported label is a + // real spelling and matches main's output for a case-only cluster (wine, not + // Wine — uppercase would otherwise sort first and change text + --json). + fold := newTagFolder(b) + type cluster struct { + nodes map[string]bool + variants map[string]bool + } + byKey := map[string]*cluster{} for _, p := range sortedNodePaths(b) { for _, tag := range nodeTags(b.Nodes[p]) { - key := strings.ToLower(strings.TrimSpace(tag)) + key := fold(tag) if key == "" { continue } - byTag[key] = append(byTag[key], p) + c := byKey[key] + if c == nil { + c = &cluster{nodes: map[string]bool{}, variants: map[string]bool{}} + byKey[key] = c + } + c.nodes[p] = true + c.variants[strings.ToLower(strings.TrimSpace(tag))] = true } } var out []ClusterFinding - tags := make([]string, 0, len(byTag)) - for t := range byTag { - tags = append(tags, t) - } - sort.Strings(tags) - for _, t := range tags { - members := byTag[t] - if len(members) >= opts.ClusterMin { - sort.Strings(members) - out = append(out, ClusterFinding{Tag: t, Nodes: members}) + keys := make([]string, 0, len(byKey)) + for k := range byKey { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + c := byKey[k] + if len(c.nodes) < opts.ClusterMin { + continue + } + members := make([]string, 0, len(c.nodes)) + for p := range c.nodes { + members = append(members, p) + } + sort.Strings(members) + // Report the lexicographically-first lower-cased variant as the cluster + // tag: a real spelling from the corpus, deterministic across runs, and + // matching main's lower-cased label for a case-only cluster. + variants := make([]string, 0, len(c.variants)) + for v := range c.variants { + variants = append(variants, v) } + sort.Strings(variants) + out = append(out, ClusterFinding{Tag: variants[0], Nodes: members}) } return out } diff --git a/internal/okf/analyze_clusters_test.go b/internal/okf/analyze_clusters_test.go new file mode 100644 index 0000000..f5040b3 --- /dev/null +++ b/internal/okf/analyze_clusters_test.go @@ -0,0 +1,90 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package okf + +import "testing" + +// analyze adopts the shared tag fold, so three spellings of one tag over three +// nodes form a single cluster that reaches the default --cluster-min 3 — where +// today's case-only fold leaves them as three 1-node tags, all invisible. +func TestAnalyzeClusters_FoldedVariantsFormOneCluster(t *testing.T) { + b := mkLintBundle(t, map[string]string{ + "index.md": "---\ntype: Index\ntitle: Index\n---\n\n# Index\n\n- [A](a.md)\n- [B](b.md)\n- [C](c.md)\n", + "a.md": lintDocTags("Concept", "A", []string{"runbook"}, "Body."), + "b.md": lintDocTags("Concept", "B", []string{"runbooks"}, "Body."), + "c.md": lintDocTags("Concept", "C", []string{"run-book"}, "Body."), + }) + clusters := analyzeClusters(b, DefaultAnalyzeOptions()) + found := false + var nodeCount int + for _, c := range clusters { + // The label is the lexicographically-first lower-cased variant; + // run-book sorts before runbook/runbooks. + if c.Tag == "run-book" { + found = true + nodeCount = len(c.Nodes) + break + } + } + if !found { + t.Fatalf("expected a folded runbook cluster labelled run-book at cluster-min 3, got clusters=%+v", clusters) + } + if nodeCount != 3 { + t.Fatalf("folded runbook cluster should span all 3 nodes, got %d", nodeCount) + } +} + +// Case folding already merged Wine/wine before this change; that behavior must +// survive the fold swap, AND the label must stay the lower-cased `wine` that +// main emits — not an upper-cased variant that sorts first. Pins the exact Tag, +// which is the regression guard thread 2 asked for. +func TestAnalyzeClusters_CaseVariantsStillMerge(t *testing.T) { + b := mkLintBundle(t, map[string]string{ + "index.md": "---\ntype: Index\ntitle: Index\n---\n\n# Index\n\n- [A](a.md)\n- [B](b.md)\n- [C](c.md)\n", + "a.md": lintDocTags("Concept", "A", []string{"Wine"}, "Body."), + "b.md": lintDocTags("Concept", "B", []string{"wine"}, "Body."), + "c.md": lintDocTags("Concept", "C", []string{"WINE"}, "Body."), + }) + clusters := analyzeClusters(b, DefaultAnalyzeOptions()) + found := false + for _, c := range clusters { + if c.Tag == "wine" && len(c.Nodes) == 3 { + found = true + } + // Guard against the upper-cased label regression: no case-variant label. + if c.Tag == "Wine" || c.Tag == "WINE" { + t.Fatalf("case-only cluster must be labelled lower-cased (wine), got %q", c.Tag) + } + } + if !found { + t.Fatalf("Wine/wine/WINE must still merge into one 3-node cluster labelled %q, got %+v", "wine", clusters) + } +} + +// Distinct tags must not be merged by the fold (negative control for analyze). +func TestAnalyzeClusters_DistinctTagsNotMerged(t *testing.T) { + b := mkLintBundle(t, map[string]string{ + "index.md": "---\ntype: Index\ntitle: Index\n---\n\n# Index\n\n- [A](a.md)\n- [B](b.md)\n- [C](c.md)\n", + "a.md": lintDocTags("Concept", "A", []string{"oauth1"}, "Body."), + "b.md": lintDocTags("Concept", "B", []string{"oauth2"}, "Body."), + "c.md": lintDocTags("Concept", "C", []string{"incident"}, "Body."), + }) + clusters := analyzeClusters(b, DefaultAnalyzeOptions()) + for _, c := range clusters { + if len(c.Nodes) >= 3 { + t.Fatalf("distinct tags must not be folded into a cluster, got %+v", c) + } + } +} diff --git a/internal/okf/lint.go b/internal/okf/lint.go index 0080a1e..c16a872 100644 --- a/internal/okf/lint.go +++ b/internal/okf/lint.go @@ -27,7 +27,7 @@ import ( // validate Finding (a spec-floor violation), a lint finding is curation // guidance — never a format failure. type LintFinding struct { - Check string `json:"check"` // "orphan" | "missing-xref" | "coverage-gap" | "type-hygiene" | "broken-link" | "status-lifecycle" | "spec-version" + Check string `json:"check"` // "orphan" | "missing-xref" | "coverage-gap" | "type-hygiene" | "tag-hygiene" | "broken-link" | "status-lifecycle" | "spec-version" Path string `json:"path"` // node path the finding is about ("" for bundle-level findings) Message string `json:"message"` } @@ -56,6 +56,7 @@ func Lint(b *Bundle, opts LintOptions) []LintFinding { findings = append(findings, lintBrokenLinks(b)...) findings = append(findings, lintCoverageGaps(b, threshold)...) findings = append(findings, lintTypeHygiene(b)...) + findings = append(findings, lintTagHygiene(b)...) findings = append(findings, lintStatusLifecycle(b)...) findings = append(findings, lintSpecVersion(b)...) @@ -655,12 +656,266 @@ func lintTypeHygiene(b *Bundle) []LintFinding { return out } -// canonType folds a type value to case-insensitive + singular (drop a single -// trailing 's') for near-duplicate grouping. -func canonType(s string) string { +// lintTagHygiene warns when two or more distinct `tags` values fold to the same +// canonical form, which usually signals accidental drift on the other §4.1 +// cross-cutting field every consumer routes and filters on. The `--tag` scope +// filter and the /api/v1/search `tag` parameter match exactly and +// case-sensitively, so one tag spelled three ways is three disjoint result sets; +// this surfaces that drift the same way type-hygiene does for `type`. +// +// The fold is the bundle-scoped tagFolder: case- and separator-insensitive, plus +// a vocabulary-aware plural merge (see newTagFolder). Anti-taxonomy stands (PRD +// §7.4): it flags only spellings that fold to ONE value, never two genuinely +// distinct values, and it is a warning class — never a validate rejection. The +// message lists each variant with its per-node count, variants ordered +// most-common-first (the dominant spelling to normalize toward leads), ties +// broken alphabetically. +func lintTagHygiene(b *Bundle) []LintFinding { + fold := newTagFolder(b) + + // canonical -> raw tag value -> set of node paths carrying it. + groups := map[string]map[string]map[string]bool{} + for _, n := range b.Nodes { + seen := map[string]bool{} // dedupe a repeated tag within one node + for _, raw := range n.Tags() { + raw = strings.TrimSpace(raw) + if raw == "" || seen[raw] { + continue + } + seen[raw] = true + c := fold(raw) + if c == "" { + continue + } + if groups[c] == nil { + groups[c] = map[string]map[string]bool{} + } + if groups[c][raw] == nil { + groups[c][raw] = map[string]bool{} + } + groups[c][raw][n.Path] = true + } + } + + var out []LintFinding + canons := make([]string, 0, len(groups)) + for c := range groups { + canons = append(canons, c) + } + sort.Strings(canons) + for _, c := range canons { + if len(groups[c]) < 2 { + continue + } + variants := make([]string, 0, len(groups[c])) + for v := range groups[c] { + variants = append(variants, v) + } + // Order variants most-common-first so the dominant spelling to + // normalize toward leads the message; ties broken alphabetically for a + // deterministic order. + sort.Slice(variants, func(i, j int) bool { + ni, nj := len(groups[c][variants[i]]), len(groups[c][variants[j]]) + if ni != nj { + return ni > nj + } + return variants[i] < variants[j] + }) + parts := make([]string, 0, len(variants)) + for _, v := range variants { + n := len(groups[c][v]) + parts = append(parts, fmt.Sprintf("%s (%d %s)", v, n, pluralNodes(n))) + } + out = append(out, LintFinding{ + Check: "tag-hygiene", + Path: "", + Message: fmt.Sprintf("tag-hygiene: near-duplicate tag values likely refer to one tag: %s", strings.Join(parts, ", ")), + }) + } + return out +} + +// pluralNodes returns "node" for a count of 1, "nodes" otherwise. +func pluralNodes(n int) string { + if n == 1 { + return "node" + } + return "nodes" +} + +// canonFold is the shared base fold: case-insensitive, trimmed, and singular +// (drop a single trailing 's'). It is the one place `type` and `tags` +// near-duplicate grouping agree on, so the tool can never apply two different +// folds to the same kind of §4.1 value. It is separator-SENSITIVE by design — +// separator-insensitivity is a tag-only layer (see canonTag) and must not leak +// into the type path. +func canonFold(s string) string { c := strings.ToLower(strings.TrimSpace(s)) if len(c) > 1 && strings.HasSuffix(c, "s") { c = c[:len(c)-1] } return c } + +// canonType folds a `type` value for near-duplicate grouping. It is exactly the +// shared base fold (case + trim + single trailing 's'); it deliberately does NOT +// add tag's separator or plural rules, so type-hygiene behavior is unchanged by +// the shared extraction. +func canonType(s string) string { return canonFold(s) } + +// canonTag computes a tag's base key: lower-cased, trimmed, and +// separator-insensitive. It does NOT strip plurals — pluralization is a +// vocabulary-aware, bundle-scoped decision made by newTagFolder, because a +// plural should only fold onto a stem the corpus actually attests. Two tags that +// differ only in case or in how a compound is joined share a base key: +// +// Run Book / run-book / run_book / runbook -> runbook +// +// Separators are '-', '_', space, and — per Unicode UAX #29 — a '/' or '.' with +// a letter on BOTH sides (node.js == nodejs, ci/cd == cicd), so a leading '.' +// stays word-internal-free (.net does not fold onto net). +func canonTag(s string) string { + c := strings.ToLower(strings.TrimSpace(s)) + c = tagSeparators.Replace(c) + c = dropLetterFlankedSeparators(c, '/', '.') + return c +} + +// tagSeparators are the always-intra-tag separators (§4.1 tags are free-form): a +// hyphen, underscore, or space. Two tags that differ only in how a compound is +// joined (home-lab vs homelab, run book vs runbook) are one value spelled two +// ways. +var tagSeparators = strings.NewReplacer("-", "", "_", "", " ", "") + +// dropLetterFlankedSeparators removes each listed rune only when a letter sits on +// both sides of it (UAX #29 word boundaries: a letter-flanked '.' or '/' is +// word-internal). This folds node.js->nodejs and ci/cd->cicd while leaving a +// leading/trailing or non-letter-flanked separator in place (.net stays apart +// from net). +func dropLetterFlankedSeparators(s string, seps ...rune) string { + isSep := func(r rune) bool { + for _, x := range seps { + if r == x { + return true + } + } + return false + } + rs := []rune(s) + var out []rune + for i, r := range rs { + if isSep(r) && i > 0 && i < len(rs)-1 && + isLetter(rs[i-1]) && isLetter(rs[i+1]) { + continue + } + out = append(out, r) + } + return string(out) +} + +func isLetter(r rune) bool { + return (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') +} + +// pluralInvariants are forms that LOOK like plurals but are not — Porter2's +// exceptional set (the Snowball English stemmer's invariant forms). A tag in this +// set is never plural-stripped, so `news` stays `news` (not `new`) and `bias` +// stays `bias` (not `bia`). See https://snowballstem.org/algorithms/english/. +var pluralInvariants = map[string]bool{ + "news": true, "atlas": true, "cosmos": true, "bias": true, + "caress": true, "gas": true, +} + +// kstemPlural returns the de-pluralized candidate stems for a base-key tag, most +// specific first, per the plural() rules of KStem (the UMass CIIR stemmer shipped +// as Lucene's KStemmer.java, plural()). It is a STEM GENERATOR, not a decision: +// the caller folds onto a candidate only if the bundle attests it (see +// newTagFolder), so a nonsense stem is never reported. Guards: +// +// - letters-only words only (k8s, v1, oauth2 are left alone); +// - length > 2; +// - a plain final 's' is stripped only when length > 3 and the word does not +// end in 'ss' or 'ous' (keeps aws, ops, ios, dns, class, famous); +// - '-ies' -> try '-ie' then '-y'; '-es' -> try dropping 's' then 'es', +// skipping the '-e' candidate after a double 's' (crosses, processes); +// - Porter2 invariants (news, atlas, cosmos, bias, …) are never stripped. +// +// This is an adaptation of KStem's plural(): KStem decides between candidates +// with an English dictionary; here the bundle's own tag vocabulary is the +// dictionary, which is exactly what a near-duplicate finding needs (two attested +// spellings) and means no out-of-vocabulary stem can ever surface. +func kstemPlural(w string) []string { + if len(w) <= 2 || !lettersOnly(w) || pluralInvariants[w] { + return nil + } + if !strings.HasSuffix(w, "s") { + return nil + } + switch { + case strings.HasSuffix(w, "ies"): + // calories -> calorie (try) then category: policies -> policy. + stem := w[:len(w)-3] + return []string{stem + "ie", stem + "y"} + case strings.HasSuffix(w, "es"): + stem := w[:len(w)-2] // drop "es" + var cands []string + // Try dropping just the 's' (…e) UNLESS the char before "es" is itself + // 's' (crosses -> cross, not crosse; processes -> process, not processe). + doubleS := len(stem) > 0 && stem[len(stem)-1] == 's' + if !doubleS { + cands = append(cands, w[:len(w)-1]) // drop 's' -> …e + } + cands = append(cands, stem) // drop 'es' + return cands + default: + // plain trailing 's': strip only when len>3, not 'ss', not 'ous'. + if len(w) > 3 && !strings.HasSuffix(w, "ss") && !strings.HasSuffix(w, "ous") { + return []string{w[:len(w)-1]} + } + return nil + } +} + +func lettersOnly(s string) bool { + if s == "" { + return false + } + for _, r := range s { + if !isLetter(r) { + return false + } + } + return true +} + +// newTagFolder returns a bundle-scoped fold that maps a raw tag to its canonical +// form. The fold is two-pass and VOCABULARY-AWARE, which is why it takes the +// bundle: a plural is folded onto its stem only when that stem is itself an +// attested base key in this bundle. Pass 1 collects every attested base key +// (canonTag of every tag on every node). Pass 2 resolves a tag by walking its +// KStem plural candidates and folding onto the first one the bundle attests; +// absent any attested candidate the base key is canonical as-is. This keeps the +// anti-taxonomy posture (nonsense stems can never be reported) and makes lint and +// analyze agree on when two spellings are one tag. +func newTagFolder(b *Bundle) func(string) string { + attested := map[string]bool{} + for _, n := range b.Nodes { + for _, raw := range n.Tags() { + if base := canonTag(raw); base != "" { + attested[base] = true + } + } + } + return func(raw string) string { + base := canonTag(raw) + if base == "" { + return "" + } + for _, cand := range kstemPlural(base) { + if attested[cand] { + return cand + } + } + return base + } +} diff --git a/internal/okf/lint_taghygiene_test.go b/internal/okf/lint_taghygiene_test.go new file mode 100644 index 0000000..b93e38e --- /dev/null +++ b/internal/okf/lint_taghygiene_test.go @@ -0,0 +1,395 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package okf + +import ( + "fmt" + "strings" + "testing" +) + +// lintDocTags builds a node with a type, title, and a YAML tags list, so tests +// can exercise the tag-hygiene fold the way the real corpus declares tags. +func lintDocTags(typ, title string, tags []string, body string) string { + tl := "[" + strings.Join(tags, ", ") + "]" + return "---\ntype: " + typ + "\ntitle: " + title + "\ntags: " + tl + "\n---\n\n# " + title + "\n\n" + body + "\n" +} + +// The base fold is the contract. canonFold is the shared base fold used by +// type-hygiene (case, trim, single trailing 's'); it is UNCHANGED by the tag +// rework, which lives entirely on the tag path. +func TestCanonFold_SharedBaseFold(t *testing.T) { + cases := []struct{ in, want string }{ + {"Concept", "concept"}, + {"concept", "concept"}, + {"Concepts", "concept"}, // single trailing 's' dropped + {" Wine ", "wine"}, // trim + {"WINE", "wine"}, // case + {"runbooks", "runbook"}, // trailing 's' + {"run-book", "run-book"}, // base fold does NOT strip separators + {"s", "s"}, // len==1 guard: don't strip the only char + {"gas", "ga"}, // trailing 's' dropped even mid-word (existing type behavior) + {"", ""}, + } + for _, c := range cases { + if got := canonFold(c.in); got != c.want { + t.Errorf("canonFold(%q) = %q, want %q", c.in, got, c.want) + } + } +} + +// canonTag is the tag BASE KEY: lower, trim, separator-insensitive. It does NOT +// strip plurals — that is a vocabulary-aware, bundle-scoped decision (see +// TestTagFolder_PluralIsVocabularyAware). Separators are '-', '_', space, and a +// letter-flanked '/' or '.' (UAX #29). A leading '.' is NOT a separator. +func TestCanonTag_BaseKey(t *testing.T) { + cases := []struct{ in, want string }{ + {"Wine", "wine"}, + {"wine", "wine"}, + {"runbook", "runbook"}, + {"run-book", "runbook"}, // hyphen stripped + {"run_book", "runbook"}, // underscore stripped + {"Run Book", "runbook"}, // space stripped + case + {"Play Book", "playbook"}, // space stripped + case + {"home-lab", "homelab"}, // separator-only collision + {"homelab", "homelab"}, + {"node.js", "nodejs"}, // letter-flanked '.' is a separator + {"nodejs", "nodejs"}, + {"ci/cd", "cicd"}, // letter-flanked '/' is a separator + {"cicd", "cicd"}, + {".net", ".net"}, // leading '.' is NOT letter-flanked: stays apart from net + {"net", "net"}, + // Base key leaves plurals alone (no bundle context here). + {"runbooks", "runbooks"}, + {"design-patterns", "designpatterns"}, + {"hormesis", "hormesis"}, // the -is/-us/-ss class is untouched + {"css", "css"}, + {"gpt-oss", "gptoss"}, + } + for _, c := range cases { + if got := canonTag(c.in); got != c.want { + t.Errorf("canonTag(%q) = %q, want %q", c.in, got, c.want) + } + } +} + +// The plural merge is an adaptation of KStem's plural() (Lucene KStemmer.java): +// a plural folds onto its stem ONLY when that stem is itself attested in the +// bundle. The attested set is supplied, newTagFolder is built from it, and the +// fold is exercised through it. +func TestTagFolder_PluralIsVocabularyAware(t *testing.T) { + attested := []string{ + "patch", "patches", // -es, drop "es" -> patch + "policy", "policies", // -ies -> policy + "process", "processes", // -es after double-s -> process (not processe) + "anti-pattern", "anti-patterns", // plain 's' + separator + "runbook", "runbooks", // plain 's' + "category", "categories", // -ies -> category + // Negatives that must NOT fold even though they look plural-ish: + "new", "news", // Porter2 invariant: news never strips -> stays apart from new + "op", "ops", // len <= 3: ops never strips -> stays apart from op + } + files := map[string]string{} + idx := "---\ntype: Index\ntitle: Index\n---\n\n# Index\n\n" + for i, tag := range attested { + name := fmt.Sprintf("n%02d.md", i) + files[name] = lintDocTags("Concept", name, []string{tag}, "Body.") + idx += fmt.Sprintf("- [%s](%s)\n", name, name) + } + files["index.md"] = idx + b := mkLintBundle(t, files) + fold := newTagFolder(b) + + same := func(a, c string) { + t.Helper() + if fold(a) != fold(c) { + t.Errorf("expected %q and %q to fold together, got %q vs %q", a, c, fold(a), fold(c)) + } + } + diff := func(a, c string) { + t.Helper() + if fold(a) == fold(c) { + t.Errorf("expected %q and %q to stay distinct, both folded to %q", a, c, fold(a)) + } + } + + // Plurals fold onto an attested stem. + same("patch", "patches") + same("policy", "policies") + same("process", "processes") + same("runbook", "runbooks") + same("anti-pattern", "anti-patterns") + same("category", "categories") + + // Distinct tags do NOT merge. + diff("new", "news") // Porter2 invariant + diff("op", "ops") // length guard +} + +// kstemPlural is the pure stem generator (vocabulary decision is separate). +func TestKstemPlural_Guards(t *testing.T) { + cases := []struct { + in string + want []string + }{ + {"runbooks", []string{"runbook"}}, // plain 's' + {"patches", []string{"patche", "patch"}}, // -es, not double-s: try "patche" then "patch" + {"policies", []string{"policie", "policy"}}, + {"processes", []string{"process"}}, // double-s -> skip the "processe" candidate + {"aws", nil}, // len 3: not > 3 + {"ops", nil}, // len 3 + {"ios", nil}, // len 3 + {"dns", nil}, // len 3 + {"class", nil}, // double-s (penult 's') + {"famous", nil}, // -ous + {"news", nil}, // Porter2 invariant + {"k8s", nil}, // not letters-only + {"runbook", nil}, // no trailing 's' + {"", nil}, + } + for _, c := range cases { + got := kstemPlural(c.in) + if len(got) != len(c.want) { + t.Errorf("kstemPlural(%q) = %v, want %v", c.in, got, c.want) + continue + } + // "patches" generates ["patche","patch"]; only assert "patch" is present + // for the simple cases to keep the table readable. + for _, w := range c.want { + found := false + for _, g := range got { + if g == w { + found = true + } + } + if !found { + t.Errorf("kstemPlural(%q) = %v, want it to contain %q", c.in, got, w) + } + } + } +} + +// canonTag must NOT change the type path. Types still fold with canonType +// (== canonFold), which is separator- AND plural-insensitive only in the base +// sense. A hyphenated and an unhyphenated type do NOT collapse under the type +// fold, and the tag's plural vocabulary rule never touches types. +func TestCanonType_UnchangedByTagRules(t *testing.T) { + if canonType("run-book") == canonType("runbook") { + t.Fatalf("type fold must stay separator-sensitive: run-book and runbook must NOT collapse") + } + if canonType("Concepts") != canonType("concept") { + t.Fatalf("type fold must still be case+plural: Concepts and concept must collapse") + } +} + +// POSITIVE controls: each fires exactly one tag-hygiene finding. Covers case, +// plain plural, -es/-ies plurals, hyphen/space separators, and letter-flanked +// '/' and '.' separators. +func TestLint_TagHygiene_Positive(t *testing.T) { + cases := []struct { + name string + tagA string + tagB string + }{ + {"case", "Wine", "wine"}, + {"plural", "runbook", "runbooks"}, + {"plural-es", "patch", "patches"}, + {"plural-ies", "policy", "policies"}, + {"plural-double-s", "process", "processes"}, + {"separator-hyphen", "run-book", "runbook"}, + {"separator-space", "Play Book", "playbook"}, + {"separator-slash", "ci/cd", "cicd"}, + {"separator-dot", "node.js", "nodejs"}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + b := mkLintBundle(t, map[string]string{ + "index.md": "---\ntype: Index\ntitle: Index\n---\n\n# Index\n\n- [A](a.md)\n- [B](b.md)\n", + "a.md": lintDocTags("Concept", "A", []string{c.tagA}, "Body."), + "b.md": lintDocTags("Concept", "B", []string{c.tagB}, "Body."), + }) + th := findingsFor(Lint(b, LintOptions{}), "tag-hygiene") + if len(th) != 1 { + t.Fatalf("%s: expected one tag-hygiene finding for %q/%q, got %+v", c.name, c.tagA, c.tagB, th) + } + if th[0].Path != "" { + t.Fatalf("%s: tag-hygiene must be bundle-level (Path==\"\"), got %q", c.name, th[0].Path) + } + m := lc(th[0].Message) + if !strings.Contains(m, lc(c.tagA)) || !strings.Contains(m, lc(c.tagB)) { + t.Fatalf("%s: message must name both variants: %q", c.name, th[0].Message) + } + }) + } +} + +// The message pins the FULL string: variants ordered most-common-first (the +// dominant spelling to normalize toward leads), ties broken alphabetically, with +// per-node counts. runbook x2 (a, c) dominates runbooks x1 (b) and run-book x1 +// (d); the two one-node variants tie and sort alphabetically (run-book < runbook). +func TestLint_TagHygiene_MessageFullStringMostCommonFirst(t *testing.T) { + b := mkLintBundle(t, map[string]string{ + "index.md": "---\ntype: Index\ntitle: Index\n---\n\n# Index\n\n- [A](a.md)\n- [B](b.md)\n- [C](c.md)\n- [D](d.md)\n", + "a.md": lintDocTags("Concept", "A", []string{"runbook"}, "Body."), + "b.md": lintDocTags("Concept", "B", []string{"runbooks"}, "Body."), + "c.md": lintDocTags("Concept", "C", []string{"runbook"}, "Body."), + "d.md": lintDocTags("Concept", "D", []string{"run-book"}, "Body."), + }) + th := findingsFor(Lint(b, LintOptions{}), "tag-hygiene") + if len(th) != 1 { + t.Fatalf("expected one folded tag-hygiene finding, got %+v", th) + } + const want = "tag-hygiene: near-duplicate tag values likely refer to one tag: runbook (2 nodes), run-book (1 node), runbooks (1 node)" + if th[0].Message != want { + t.Fatalf("message mismatch:\n got: %q\nwant: %q", th[0].Message, want) + } +} + +// NEGATIVE controls: legitimately similar-looking tags stay silent. The +// new/news and op/ops pairs are LOAD-BEARING — they collide under a bare +// trailing-'s' rule and only stay silent because of the Porter2 invariant and +// the length guard. See TestLint_TagHygiene_NegativeControlsAreLoadBearing. +func TestLint_TagHygiene_Negative(t *testing.T) { + cases := []struct { + name string + tagA string + tagB string + }{ + {"version-digit", "v1", "v2"}, + {"oauth-digit", "oauth1", "oauth2"}, + {"distinct-words", "oncall", "incident"}, + {"porter2-invariant", "new", "news"}, // news never strips -> distinct from new + {"length-guard", "op", "ops"}, // ops len 3 -> never strips + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + b := mkLintBundle(t, map[string]string{ + "index.md": "---\ntype: Index\ntitle: Index\n---\n\n# Index\n\n- [A](a.md)\n- [B](b.md)\n", + "a.md": lintDocTags("Concept", "A", []string{c.tagA}, "Body."), + "b.md": lintDocTags("Concept", "B", []string{c.tagB}, "Body."), + }) + if n := len(findingsFor(Lint(b, LintOptions{}), "tag-hygiene")); n != 0 { + t.Fatalf("%s: %q/%q are distinct; expected 0 tag-hygiene findings, got %d", c.name, c.tagA, c.tagB, n) + } + }) + } +} + +// Proof the new/news and op/ops negative controls are LOAD-BEARING: under the +// OLD bare trailing-'s' fold (lower+trim+drop one 's'), both pairs collide, so a +// test that passes them only means something because the new fold deliberately +// keeps them apart. This reconstructs the old fold inline and asserts it WOULD +// have merged them — so a regression back to the bare rule turns the negative +// control red. +func TestLint_TagHygiene_NegativeControlsAreLoadBearing(t *testing.T) { + oldBareFold := func(s string) string { + c := strings.ToLower(strings.TrimSpace(s)) + if len(c) > 1 && strings.HasSuffix(c, "s") { + c = c[:len(c)-1] + } + return c + } + for _, p := range [][2]string{{"new", "news"}, {"op", "ops"}} { + if oldBareFold(p[0]) != oldBareFold(p[1]) { + t.Fatalf("negative control %q/%q is NOT load-bearing: the old bare fold did not merge them (%q vs %q)", + p[0], p[1], oldBareFold(p[0]), oldBareFold(p[1])) + } + } +} + +// A single tag spread over many nodes is not drift — it must stay silent. +func TestLint_TagHygiene_SingleTagManyNodesNoFinding(t *testing.T) { + b := mkLintBundle(t, map[string]string{ + "index.md": "---\ntype: Index\ntitle: Index\n---\n\n# Index\n\n- [A](a.md)\n- [B](b.md)\n- [C](c.md)\n", + "a.md": lintDocTags("Concept", "A", []string{"wine"}, "Body."), + "b.md": lintDocTags("Concept", "B", []string{"wine"}, "Body."), + "c.md": lintDocTags("Concept", "C", []string{"wine"}, "Body."), + }) + if n := len(findingsFor(Lint(b, LintOptions{}), "tag-hygiene")); n != 0 { + t.Fatalf("one tag on many nodes is not drift; expected 0 tag-hygiene findings, got %d", n) + } +} + +// The -is/-us/-ss digraph class (real-corpus measurement: 65 such tags, zero +// collisions) must each be silent. Two mechanisms keep them silent: 'ss' and +// 'ous' are never stripped at all (css, gpt-oss), and an '-is'/'-us' word only +// folds if its stripped stem is ATTESTED in the bundle — which it is not here, +// nor in the real corpus. Each is paired with an unrelated sibling so the only +// way a finding could fire is an over-eager fold; the vocabulary guard prevents +// it without a stop-list. +func TestLint_TagHygiene_DigraphClassSilent(t *testing.T) { + digraphs := []string{"hormesis", "hysteresis", "css", "freshness", "blast-radius", "gpt-oss"} + for _, d := range digraphs { + t.Run(d, func(t *testing.T) { + b := mkLintBundle(t, map[string]string{ + "index.md": "---\ntype: Index\ntitle: Index\n---\n\n# Index\n\n- [A](a.md)\n- [B](b.md)\n", + "a.md": lintDocTags("Concept", "A", []string{d}, "Body."), + "b.md": lintDocTags("Concept", "B", []string{"incident"}, "Body."), + }) + if n := len(findingsFor(Lint(b, LintOptions{}), "tag-hygiene")); n != 0 { + t.Fatalf("digraph tag %q must not collide with an unrelated sibling; got %d findings", d, n) + } + }) + } +} + +// The vocabulary guard is what keeps the '-is'/'-us' class quiet: hormesis WOULD +// strip to hormesi under the plain-'s' rule, but only folds if hormesi is an +// attested tag. This proves the stem generator does produce the candidate (so +// the guard is doing real work) AND that a bundle without the stem never folds. +func TestLint_TagHygiene_IsUsClassGuardedByVocabulary(t *testing.T) { + // Generator DOES offer the stripped candidate (plain 's', len>3, not ss/ous). + if got := kstemPlural("hormesis"); len(got) != 1 || got[0] != "hormesi" { + t.Fatalf("kstemPlural(hormesis) = %v, want [hormesi]", got) + } + // But with no attested "hormesi" in the bundle, the two stay distinct. + b := mkLintBundle(t, map[string]string{ + "index.md": "---\ntype: Index\ntitle: Index\n---\n\n# Index\n\n- [A](a.md)\n- [B](b.md)\n", + "a.md": lintDocTags("Concept", "A", []string{"hormesis"}, "Body."), + "b.md": lintDocTags("Concept", "B", []string{"incident"}, "Body."), + }) + if n := len(findingsFor(Lint(b, LintOptions{}), "tag-hygiene")); n != 0 { + t.Fatalf("hormesis must not fold without an attested hormesi stem; got %d findings", n) + } +} + +// The shared-fold extraction must leave type-hygiene byte-identical. This pins +// the exact message string the check emitted before the refactor, proving the +// type path did not move when canonType became a thin canonFold wrapper. +func TestLint_TypeHygiene_MessageByteIdenticalAfterFoldExtraction(t *testing.T) { + b := mkLintBundle(t, map[string]string{ + "index.md": "---\ntype: Index\ntitle: Index\n---\n\n# Index\n\n- [A](a.md)\n- [B](b.md)\n", + "a.md": lintDoc("Concept", "A", "Body."), + "b.md": lintDoc("Concepts", "B", "Body."), + }) + th := findingsFor(Lint(b, LintOptions{}), "type-hygiene") + if len(th) != 1 { + t.Fatalf("expected one type-hygiene finding, got %+v", th) + } + const want = "type-hygiene: near-duplicate type values likely refer to one type: Concept, Concepts" + if th[0].Message != want { + t.Fatalf("type-hygiene message changed by fold extraction:\n got: %q\nwant: %q", th[0].Message, want) + } + // And a separator-only type pair must NOT collapse (types stay + // separator-sensitive; the tag-only rules must not leak into the type path). + b2 := mkLintBundle(t, map[string]string{ + "index.md": "---\ntype: Index\ntitle: Index\n---\n\n# Index\n\n- [A](a.md)\n- [B](b.md)\n", + "a.md": lintDoc("run-book", "A", "Body."), + "b.md": lintDoc("runbook", "B", "Body."), + }) + if n := len(findingsFor(Lint(b2, LintOptions{}), "type-hygiene")); n != 0 { + t.Fatalf("type-hygiene must stay separator-sensitive: run-book/runbook types must NOT fold, got %d findings", n) + } +}