Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions cmd/lint.go
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,13 @@ func newLintCmd() *cobra.Command {
"node with the same basename exists elsewhere — a moved or mistyped path (a defect), " +
"distinct from a genuinely unwritten concept (a coverage gap, which analyze reports " +
"advisorily and lint stays quiet on).\n\n" +
"tag-hygiene reports two or more `tags` spellings that fold to one canonical form. " +
"The fold is a heuristic: case, trimmed, separator-insensitive (so run-book, run_book " +
"and runbook are one value), plus a vocabulary-aware plural merge (a plural folds onto " +
"its singular only when that singular is itself a tag in the bundle). Like type-hygiene, " +
"it never rejects a value: it only reports spellings that fold together, so a curator " +
"can pick one. Because the fold is heuristic, review a finding before normalizing — it " +
"groups spellings that look like one tag, not a proof that they are.\n\n" +
"--semantic adds similarity-driven checks (similar-but-unlinked pairs, nodes with " +
"no semantic neighbors) by reading the index built by 'okfctl-search index build'. " +
"Core only reads that index, so no embedding model is needed to lint.",
Expand Down
2 changes: 2 additions & 0 deletions docs/commands/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -408,6 +408,8 @@ lint surfaces judgment-worthy curation findings, not spec-floor violations (use

A broken-link finding reports an internal .md link that resolves to no node when a node with the same basename exists elsewhere — a moved or mistyped path (a defect), distinct from a genuinely unwritten concept (a coverage gap, which analyze reports advisorily and lint stays quiet on).

tag-hygiene reports two or more `tags` spellings that fold to one canonical form. The fold is a heuristic: case, trimmed, separator-insensitive (so run-book, run_book and runbook are one value), plus a vocabulary-aware plural merge (a plural folds onto its singular only when that singular is itself a tag in the bundle). Like type-hygiene, it never rejects a value: it only reports spellings that fold together, so a curator can pick one. Because the fold is heuristic, review a finding before normalizing — it groups spellings that look like one tag, not a proof that they are.

--semantic adds similarity-driven checks (similar-but-unlinked pairs, nodes with no semantic neighbors) by reading the index built by 'okfctl-search index build'. Core only reads that index, so no embedding model is needed to lint.

```
Expand Down
57 changes: 44 additions & 13 deletions internal/okf/analyze.go
Original file line number Diff line number Diff line change
Expand Up @@ -604,28 +604,59 @@ func analyzeConnectivity(b *Bundle) ConnectivityReport {
}

func analyzeClusters(b *Bundle, opts AnalyzeOptions) []ClusterFinding {
byTag := map[string][]string{}
// Adopt lint's shared, bundle-scoped tag fold (newTagFolder: case- and
// separator-insensitive plus a vocabulary-aware plural merge) so analyze and
// lint agree on when two spellings are one tag. Group by canonical key;
// within a key, keep the DISTINCT node set (a node tagged both `wine` and
// `Wine` counts once) and the lower-cased variants so the reported label is a
// real spelling and matches main's output for a case-only cluster (wine, not
// Wine — uppercase would otherwise sort first and change text + --json).
fold := newTagFolder(b)
type cluster struct {
nodes map[string]bool
variants map[string]bool
}
byKey := map[string]*cluster{}
for _, p := range sortedNodePaths(b) {
for _, tag := range nodeTags(b.Nodes[p]) {
key := strings.ToLower(strings.TrimSpace(tag))
key := fold(tag)
if key == "" {
continue
}
byTag[key] = append(byTag[key], p)
c := byKey[key]
if c == nil {
c = &cluster{nodes: map[string]bool{}, variants: map[string]bool{}}
byKey[key] = c
}
c.nodes[p] = true
c.variants[strings.ToLower(strings.TrimSpace(tag))] = true
}
}
var out []ClusterFinding
tags := make([]string, 0, len(byTag))
for t := range byTag {
tags = append(tags, t)
}
sort.Strings(tags)
for _, t := range tags {
members := byTag[t]
if len(members) >= opts.ClusterMin {
sort.Strings(members)
out = append(out, ClusterFinding{Tag: t, Nodes: members})
keys := make([]string, 0, len(byKey))
for k := range byKey {
keys = append(keys, k)
}
sort.Strings(keys)
for _, k := range keys {
c := byKey[k]
if len(c.nodes) < opts.ClusterMin {
continue
}
members := make([]string, 0, len(c.nodes))
for p := range c.nodes {
members = append(members, p)
}
sort.Strings(members)
// Report the lexicographically-first lower-cased variant as the cluster
// tag: a real spelling from the corpus, deterministic across runs, and
// matching main's lower-cased label for a case-only cluster.
variants := make([]string, 0, len(c.variants))
for v := range c.variants {
variants = append(variants, v)
}
sort.Strings(variants)
out = append(out, ClusterFinding{Tag: variants[0], Nodes: members})
}
return out
}
Expand Down
90 changes: 90 additions & 0 deletions internal/okf/analyze_clusters_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
// Copyright 2026 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package okf

import "testing"

// analyze adopts the shared tag fold, so three spellings of one tag over three
// nodes form a single cluster that reaches the default --cluster-min 3 — where
// today's case-only fold leaves them as three 1-node tags, all invisible.
func TestAnalyzeClusters_FoldedVariantsFormOneCluster(t *testing.T) {
b := mkLintBundle(t, map[string]string{
"index.md": "---\ntype: Index\ntitle: Index\n---\n\n# Index\n\n- [A](a.md)\n- [B](b.md)\n- [C](c.md)\n",
"a.md": lintDocTags("Concept", "A", []string{"runbook"}, "Body."),
"b.md": lintDocTags("Concept", "B", []string{"runbooks"}, "Body."),
"c.md": lintDocTags("Concept", "C", []string{"run-book"}, "Body."),
})
clusters := analyzeClusters(b, DefaultAnalyzeOptions())
found := false
var nodeCount int
for _, c := range clusters {
// The label is the lexicographically-first lower-cased variant;
// run-book sorts before runbook/runbooks.
if c.Tag == "run-book" {
found = true
nodeCount = len(c.Nodes)
break
}
}
if !found {
t.Fatalf("expected a folded runbook cluster labelled run-book at cluster-min 3, got clusters=%+v", clusters)
}
if nodeCount != 3 {
t.Fatalf("folded runbook cluster should span all 3 nodes, got %d", nodeCount)
}
}

// Case folding already merged Wine/wine before this change; that behavior must
// survive the fold swap, AND the label must stay the lower-cased `wine` that
// main emits — not an upper-cased variant that sorts first. Pins the exact Tag,
// which is the regression guard thread 2 asked for.
func TestAnalyzeClusters_CaseVariantsStillMerge(t *testing.T) {
b := mkLintBundle(t, map[string]string{
"index.md": "---\ntype: Index\ntitle: Index\n---\n\n# Index\n\n- [A](a.md)\n- [B](b.md)\n- [C](c.md)\n",
"a.md": lintDocTags("Concept", "A", []string{"Wine"}, "Body."),
"b.md": lintDocTags("Concept", "B", []string{"wine"}, "Body."),
"c.md": lintDocTags("Concept", "C", []string{"WINE"}, "Body."),
})
clusters := analyzeClusters(b, DefaultAnalyzeOptions())
found := false
for _, c := range clusters {
if c.Tag == "wine" && len(c.Nodes) == 3 {
found = true
}
// Guard against the upper-cased label regression: no case-variant label.
if c.Tag == "Wine" || c.Tag == "WINE" {
t.Fatalf("case-only cluster must be labelled lower-cased (wine), got %q", c.Tag)
}
}
if !found {
t.Fatalf("Wine/wine/WINE must still merge into one 3-node cluster labelled %q, got %+v", "wine", clusters)
}
}

// Distinct tags must not be merged by the fold (negative control for analyze).
func TestAnalyzeClusters_DistinctTagsNotMerged(t *testing.T) {
b := mkLintBundle(t, map[string]string{
"index.md": "---\ntype: Index\ntitle: Index\n---\n\n# Index\n\n- [A](a.md)\n- [B](b.md)\n- [C](c.md)\n",
"a.md": lintDocTags("Concept", "A", []string{"oauth1"}, "Body."),
"b.md": lintDocTags("Concept", "B", []string{"oauth2"}, "Body."),
"c.md": lintDocTags("Concept", "C", []string{"incident"}, "Body."),
})
clusters := analyzeClusters(b, DefaultAnalyzeOptions())
for _, c := range clusters {
if len(c.Nodes) >= 3 {
t.Fatalf("distinct tags must not be folded into a cluster, got %+v", c)
}
}
}
Loading
Loading