Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
301 changes: 301 additions & 0 deletions internal/okf/index_eol_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,301 @@
// Copyright 2026 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package okf

import (
"os"
"path/filepath"
"strings"
"testing"
)

// toCRLF rewrites every index.md under dir to CRLF, as a Windows checkout with
// core.autocrlf=true does (see issue #185). It is the test-side equivalent of
// the issue's `find kb -name index.md -exec perl -pi -e 's/\r?\n/\r\n/'`.
func toCRLF(t *testing.T, dir string) {
t.Helper()
err := filepath.Walk(dir, func(p string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if info.IsDir() || filepath.Base(p) != "index.md" {
return nil
}
b, rerr := os.ReadFile(p) //nolint:gosec // G304: test fixture under t.TempDir()
if rerr != nil {
return rerr
}
// Normalize to LF first, then to CRLF, so the result is pure CRLF
// regardless of the input's existing endings.
lf := strings.ReplaceAll(string(b), "\r\n", "\n")
crlf := strings.ReplaceAll(lf, "\n", "\r\n")
return os.WriteFile(p, []byte(crlf), 0o644) //nolint:gosec // G306: test fixture
})
if err != nil {
t.Fatal(err)
}
}

// readFile is a small test helper that returns a file's bytes as a string.
func readFileStr(t *testing.T, p string) string {
t.Helper()
b, err := os.ReadFile(p) //nolint:gosec // G304: test fixture under t.TempDir()
if err != nil {
t.Fatal(err)
}
return string(b)
}

// TestUsesCRLF_Table pins the dominant-ending detector: a file is CRLF only when
// EVERY line break is CRLF. A mixed-ending file, an all-LF file, and a file with
// no line break at all are NOT CRLF (new/mixed indexes stay LF on write).
func TestUsesCRLF_Table(t *testing.T) {
cases := []struct {
name string
in string
want bool
}{
{"pure LF", "a\nb\nc\n", false},
{"pure CRLF", "a\r\nb\r\nc\r\n", true},
{"mixed mostly CRLF one LF", "a\r\nb\nc\r\n", false},
{"mixed mostly LF one CRLF", "a\nb\r\nc\n", false},
{"no newline", "abc", false},
{"empty", "", false},
{"single CRLF line", "a\r\n", true},
{"single LF line", "a\n", false},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
if got := usesCRLF(c.in); got != c.want {
t.Errorf("usesCRLF(%q) = %v, want %v", c.in, got, c.want)
}
})
}
}

// TestNormalizeEOL folds CRLF to LF and leaves LF-only content untouched.
func TestNormalizeEOL(t *testing.T) {
if got := normalizeEOL("a\r\nb\r\n"); got != "a\nb\n" {
t.Errorf("normalizeEOL CRLF: got %q", got)
}
if got := normalizeEOL("a\nb\n"); got != "a\nb\n" {
t.Errorf("normalizeEOL LF: got %q", got)
}
// A lone CR not followed by LF is left alone (we only fold CRLF).
if got := normalizeEOL("a\rb"); got != "a\rb" {
t.Errorf("normalizeEOL lone CR: got %q", got)
}
}

// TestIndexInSync_CRLFIndexIsFresh is seam (b): a CRLF index whose normalized
// content matches the render reports IN SYNC. Issue #185: `index check` compares
// raw bytes to the LF render, so a line-ending-only difference counted as drift
// and Windows teams could not use it as a gate.
func TestIndexInSync_CRLFIndexIsFresh(t *testing.T) {
dir := t.TempDir()
_ = Scaffold(dir)
writeNode(t, dir, "wine/tannin.md", "Reference", "Tannin")

// Build the canonical LF tree, then convert to CRLF (the Windows checkout).
b, _ := Load(dir)
if err := WriteIndex(b); err != nil {
t.Fatal(err)
}
toCRLF(t, dir)

b2, _ := Load(dir)
if ok, diff := IndexInSync(b2); !ok {
t.Errorf("a CRLF index matching the render must be IN SYNC (content-only compare); diff:\n%s", diff)
}
}

// TestIndexInSync_CRLFGenuineDriftStillCaught is the NEGATIVE control for seam
// (b): normalizing line endings must not mask REAL content drift. A CRLF index
// that is genuinely stale (a node added after the build) must still report out
// of sync.
func TestIndexInSync_CRLFGenuineDriftStillCaught(t *testing.T) {
dir := t.TempDir()
_ = Scaffold(dir)
writeNode(t, dir, "wine/tannin.md", "Reference", "Tannin")

b, _ := Load(dir)
if err := WriteIndex(b); err != nil {
t.Fatal(err)
}
toCRLF(t, dir)

// Real content change AFTER the index was written: the CRLF index is now
// genuinely stale, not merely a different line ending.
writeNode(t, dir, "wine/acidity.md", "Reference", "Acidity")

b2, _ := Load(dir)
if ok, _ := IndexInSync(b2); ok {
t.Error("a CRLF index with genuinely stale content must still be OUT OF SYNC")
}
}

// TestWriteIndex_PreservesCRLF is seam (c), positive control: `index build` on
// an up-to-date CRLF checkout leaves every index.md BYTE-IDENTICAL (still CRLF),
// so git with autocrlf=false (or CRLF committed) sees no churn.
func TestWriteIndex_PreservesCRLF(t *testing.T) {
dir := t.TempDir()
_ = Scaffold(dir)
writeNode(t, dir, "wine/tannin.md", "Reference", "Tannin")

b, _ := Load(dir)
if err := WriteIndex(b); err != nil {
t.Fatal(err)
}
toCRLF(t, dir)

rootP := filepath.Join(dir, "index.md")
wineP := filepath.Join(dir, "wine", "index.md")
beforeRoot := readFileStr(t, rootP)
beforeWine := readFileStr(t, wineP)
if !strings.Contains(beforeRoot, "\r\n") {
t.Fatal("fixture precondition: root index should be CRLF after toCRLF")
}

// Rebuild from the CRLF checkout: content is unchanged, so the files must
// stay byte-identical and CRLF.
b2, _ := Load(dir)
if err := WriteIndex(b2); err != nil {
t.Fatal(err)
}
if got := readFileStr(t, rootP); got != beforeRoot {
t.Errorf("root index.md must be byte-identical after rebuild on a CRLF checkout\nbefore: %q\nafter: %q", beforeRoot, got)
}
if got := readFileStr(t, wineP); got != beforeWine {
t.Errorf("wine/index.md must be byte-identical after rebuild on a CRLF checkout\nbefore: %q\nafter: %q", beforeWine, got)
}
}

// TestWriteIndex_PreservesLF is seam (c), the LF arm: an up-to-date LF index
// stays byte-identical and LF after a rebuild (no CRLF leaks in).
func TestWriteIndex_PreservesLF(t *testing.T) {
dir := t.TempDir()
_ = Scaffold(dir)
writeNode(t, dir, "wine/tannin.md", "Reference", "Tannin")

b, _ := Load(dir)
if err := WriteIndex(b); err != nil {
t.Fatal(err)
}
rootP := filepath.Join(dir, "index.md")
before := readFileStr(t, rootP)
if strings.Contains(before, "\r\n") {
t.Fatal("fixture precondition: a freshly built index should be LF")
}

b2, _ := Load(dir)
if err := WriteIndex(b2); err != nil {
t.Fatal(err)
}
after := readFileStr(t, rootP)
if after != before {
t.Errorf("LF index.md must be byte-identical after rebuild\nbefore: %q\nafter: %q", before, after)
}
if strings.Contains(after, "\r\n") {
t.Errorf("an LF index must not gain CRLF endings on rebuild; got %q", after)
}
}

// TestWriteIndex_NewIndexIsLF: a directory that gains its first index.md on a
// build writes LF (new files stay LF; only an EXISTING CRLF file is preserved).
func TestWriteIndex_NewIndexIsLF(t *testing.T) {
dir := t.TempDir()
_ = Scaffold(dir)
writeNode(t, dir, "wine/tannin.md", "Reference", "Tannin")

b, _ := Load(dir)
if err := WriteIndex(b); err != nil {
t.Fatal(err)
}
wineP := filepath.Join(dir, "wine", "index.md")
got := readFileStr(t, wineP)
if got == "" {
t.Fatal("expected a nested wine/index.md to be written")
}
if strings.Contains(got, "\r\n") {
t.Errorf("a newly written index.md must be LF; got %q", got)
}
}

// TestWriteIndex_CuratorSuffixIdempotentOnCRLF is seam (a), the data-damage bug:
// a curator subdirectory description in a CRLF index must round-trip through
// `index build` with the tool-owned shape suffix appearing EXACTLY ONCE. Before
// the fix, the `$`-anchored suffix regex fails on a trailing \r, the stale
// suffix is captured into the preserved description, and build appends a fresh
// one: `... - Audit trail rules. (1 concept Β· reference) (1 concept Β· reference)`.
func TestWriteIndex_CuratorSuffixIdempotentOnCRLF(t *testing.T) {
dir := t.TempDir()
_ = Scaffold(dir)
writeNode(t, dir, "audit/rule.md", "reference", "Rule")

// Build the canonical index, then give the audit subdirectory entry a
// curator description (the Β§8 curator-owned text before the shape suffix).
b, _ := Load(dir)
if err := WriteIndex(b); err != nil {
t.Fatal(err)
}
rootP := filepath.Join(dir, "index.md")
rendered := readFileStr(t, rootP)

// Insert a curator description into the audit bullet: turn
// * [Audit](audit/) (N concept Β· ...)
// into
// * [Audit](audit/) - Audit trail rules. (N concept Β· ...)
const linkMarker = "](audit/)"
i := strings.Index(rendered, linkMarker)
if i < 0 {
t.Fatalf("expected an audit subdirectory bullet in the root index; got:\n%s", rendered)
}
insertAt := i + len(linkMarker)
curated := rendered[:insertAt] + " - Audit trail rules." + rendered[insertAt:]
if err := os.WriteFile(rootP, []byte(curated), 0o644); err != nil { //nolint:gosec // G306: test fixture
t.Fatal(err)
}

// A build on the LF curated index is idempotent and the suffix appears once.
bLF, _ := Load(dir)
if err := WriteIndex(bLF); err != nil {
t.Fatal(err)
}
lfResult := readFileStr(t, rootP)
if n := strings.Count(lfResult, "(1 concept"); n != 1 {
t.Fatalf("LF precondition: shape suffix should appear once, got %d:\n%s", n, lfResult)
}
if !strings.Contains(lfResult, "- Audit trail rules.") {
t.Fatalf("LF precondition: curator description should be preserved:\n%s", lfResult)
}

// Now convert to CRLF (the Windows checkout) and rebuild. The suffix must
// STILL appear exactly once and the curator description survive verbatim.
toCRLF(t, dir)
bCRLF, _ := Load(dir)
if err := WriteIndex(bCRLF); err != nil {
t.Fatal(err)
}
crlfResult := readFileStr(t, rootP)
if n := strings.Count(crlfResult, "(1 concept"); n != 1 {
t.Errorf("shape suffix must appear EXACTLY ONCE after a CRLF rebuild, got %d:\n%q", n, crlfResult)
}
if !strings.Contains(crlfResult, "- Audit trail rules.") {
t.Errorf("curator description must survive a CRLF rebuild:\n%q", crlfResult)
}
if !strings.Contains(crlfResult, "\r\n") {
t.Errorf("an up-to-date CRLF index must stay CRLF after rebuild:\n%q", crlfResult)
}
}
55 changes: 52 additions & 3 deletions internal/okf/reserved_lifecycle.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,32 @@ import (
"strings"
)

// normalizeEOL folds CRLF to LF so a CRLF-checked-out reserved file (Windows
// with core.autocrlf=true) parses and compares identically to its LF form. It
// is the "normalize on read" half of okfctl's index line-ending policy: an
// index.md is normalized before regex matching (preservedSubdirDescriptions) and
// before the staleness comparison (IndexInSync), so a line-ending-only
// difference is never mistaken for content drift (issue #185). A lone CR not
// part of a CRLF pair is left alone β€” only the \r\n pair is folded.
func normalizeEOL(s string) string {
return strings.ReplaceAll(s, "\r\n", "\n")
}

// usesCRLF reports whether a reserved file is dominantly CRLF: true only when
// EVERY line break in it is CRLF (every \n is preceded by \r). It is the
// "preserve on write" half of the policy β€” WriteIndex writes its render back in
// the ending the on-disk file already used, so an up-to-date CRLF index stays
// CRLF (zero churn, issue #185) while a new file or a mixed-ending file is
// written LF. The choice depends only on that one file, so it is deterministic
// on any machine. A file with no line break at all is not CRLF.
func usesCRLF(s string) bool {
nl := strings.Count(s, "\n")
if nl == 0 {
return false
}
return strings.Count(s, "\r\n") == nl
}

// neighborhood returns the top-level directory of a bundle-relative slash path,
// or "" for a root-level node (rendered under a "(root)" group).
func neighborhood(rel string) string {
Expand Down Expand Up @@ -320,7 +346,14 @@ func preservedSubdirDescriptions(b *Bundle, dir string, opts IndexShapeOptions)
if !ok || idx == nil {
return out
}
for _, line := range strings.Split(idx.Body, "\n") {
// Normalize line endings before splitting (OKF Β§8 parse path; issue #185):
// a CRLF-checked-out index leaves a trailing \r on every line, which stops
// the $-anchored subdirBulletRE / toolShapeSuffixRE from matching the
// tool-owned shape suffix β€” the stale suffix would then be captured into the
// curator description and doubled on the next build. Folding CRLF->LF here
// (normalize on read) fixes that at the same seam that keeps the description
// curator-owned/verbatim.
for _, line := range strings.Split(normalizeEOL(idx.Body), "\n") {
m := subdirBulletRE.FindStringSubmatch(line)
if m == nil {
continue
Expand Down Expand Up @@ -464,7 +497,18 @@ func WriteIndexWithOptions(b *Bundle, opts IndexShapeOptions) error {
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { //nolint:gosec // G301: shareable bundle content dir; 0o755 is intended
return err
}
if err := os.WriteFile(p, []byte(RenderDirIndexWithOptions(b, dir, opts)), 0o644); err != nil { //nolint:gosec // G306: a bundle index file is shareable content; 0o644 is intended
rendered := RenderDirIndexWithOptions(b, dir, opts)
// Preserve on write (issue #185): the render is produced LF, but if the
// index.md already on disk is dominantly CRLF (a Windows checkout), write
// the render back in CRLF so an up-to-date index stays byte-identical and
// git sees no whole-file line-ending churn. A new file, or one with mixed
// endings, is written LF. The choice depends only on this one file, so it
// is deterministic on any machine and leaves LF/Linux behavior unchanged.
out := rendered
if existing, rerr := os.ReadFile(p); rerr == nil && usesCRLF(string(existing)) { //nolint:gosec // G304: reading the tool's own generated index file
out = strings.ReplaceAll(rendered, "\n", "\r\n")
}
if err := os.WriteFile(p, []byte(out), 0o644); err != nil { //nolint:gosec // G306: a bundle index file is shareable content; 0o644 is intended
return err
}
}
Expand Down Expand Up @@ -507,7 +551,12 @@ func IndexInSyncWithOptions(b *Bundle, opts IndexShapeOptions) (bool, string) {
if err != nil {
return false, fmt.Sprintf("%s is missing or unreadable; run `okfctl index build`", filepath.ToSlash(rel(b.Root, p)))
}
if string(onDisk) != RenderDirIndexWithOptions(b, dir, opts) {
// Compare CONTENT, not bytes (issue #185): a CRLF-checked-out index
// (Windows, core.autocrlf=true) differs from the LF render only in line
// endings. Normalize on read so a line-ending-only difference is not
// reported as drift β€” any other difference is still drift, so genuine
// staleness is still caught.
if normalizeEOL(string(onDisk)) != RenderDirIndexWithOptions(b, dir, opts) {
return false, fmt.Sprintf("%s is out of date; run `okfctl index build` to regenerate", filepath.ToSlash(rel(b.Root, p)))
}
}
Expand Down
Loading