diff --git a/.claude/skills/collect_knowledge/SKILL.md b/.claude/skills/collect_knowledge/SKILL.md new file mode 100644 index 0000000..cfbd768 --- /dev/null +++ b/.claude/skills/collect_knowledge/SKILL.md @@ -0,0 +1,213 @@ +--- +name: collect_knowledge +description: Collect and import knowledge from local files or directories into the knowledge tree, translating any non-English content to English first and verifying the translation before anything is merged. Analyzes content to determine domain placement, evaluates whether the knowledge already exists in the tree, decides optimal organization (new file, merge, or skip), and generates frontmatter. Use this skill whenever importing local markdown, documentation, notes, or text files from an external directory into the tree — especially when the source may be in another language, when you need deduplication against existing content, or when a corpus needs organizing by domain. +allowed-tools: Read, Write, Glob, Grep +--- + +# Collect Knowledge + +Import local files and directories into the knowledge tree under `tree/`, in English, without breaking their structure. + +The hard part is not reading files — it is that a corpus in another language must come out the other side fully translated, with its headings, code, tables, links and frontmatter intact. That does not happen by intending it. Translation is expensive and every other step in this skill is cheap, so under load the expensive step is the one that silently gets skipped. This skill therefore treats translation as a **gated** step: staged, machine-checked, and blocked from merging until it passes. + +## When to use + +- You have a local directory of notes or documentation to integrate into the tree +- A corpus is in Chinese, Japanese, Korean, Russian, or any non-English language and needs to land in the tree in English +- You're consolidating multiple local sources and need to avoid duplicates +- You want to assess whether new knowledge is truly new or supplements existing tree content +- You need to audit an already-imported part of the tree for untranslated content + +## The tools + +Three scripts under `scripts/`, all path-agnostic — they take arguments, hold no embedded paths, and work on any markdown corpus: + +| Script | Role | +|---|---| +| `lang_scan.py` | Detect what still needs translating, and where. Plans chunks for large files. | +| `verify_translation.py` | The gate. Residual-language, structure-parity and anchor checks. | +| `mdlang.py` | Shared library — not run directly. | + +They resolve their own imports, so run them from any working directory. Python 3.9+, no third-party dependencies. + +## Workflow + +### 1. Triage the source + +```bash +python scripts/lang_scan.py --summary +``` + +This buckets every file by how much non-English text it holds, and names the dominant script. Use it to size the job before starting, then branch per file: + +| Bucket | Meaning | What to do | +|---|---|---| +| `CLEAN` | no non-Latin text found | skip to step 5 — nothing to translate | +| `TRACE` | under 5% of lines | translate whole-file; often just a stray line or a proper noun | +| `PARTIAL` | 5–25% of lines | translate whole-file, or chunk if long | +| `HEAVY` | over 25% of lines | chunk if over ~400 lines, then translate | + +Add `--show 20` on any single file to see exactly which lines and which category. + +### 2. Stage, don't edit in place + +Translate into a staging directory, never over the source. Keeping the original untouched is what makes the structure comparison in step 4 possible, and what makes a bad translation recoverable. + +Mirror the path each file has *relative to the corpus root* — step 4 pairs originals to translations by that relative path, and files that don't line up get no structure checking at all: + +```bash +SRC= # e.g. ~/corpus/domain-05 +STAGE=/tmp/collect-staging + +# For a file at $SRC/01-identity/03-tokens.md, stage it at +# $STAGE/01-identity/03-tokens.md +mkdir -p "$STAGE/01-identity" +``` + +### 3. Translate + +For each file that isn't `CLEAN`. + +**Read the translation rules below before starting**, and skim `references/translation-rules.md` for the cases the summary can't settle — mixed-language files, text that should legitimately stay in the source language, and ambiguous inline code. + +**Small files (under ~400 lines):** read, translate, write the whole file to staging. + +**Large files:** ask for a chunk plan first — + +```bash +python scripts/lang_scan.py --plan +``` + +It prints one line per chunk: + +``` +chunk 0 lines 1-56 ( 56) (frontmatter -- translate title/description/summary, keep slugs and dates) +chunk 1 lines 57-498 (442) +chunk 2 lines 499-999 (501) 2.6 Grype in continuous integration +chunk 3 lines 1000-1565 (566) 4.1 CVSS v3.1 base scoring +``` + +Ranges are contiguous and cover the file from line 1. Translate them **in order**, appending each result to the staged file (`>>`), so the reassembled document has every line accounted for. + +Chunk 0 is always the frontmatter when the file has any. It gets its own chunk precisely because it sits above the first heading — a plan built from heading boundaries alone would skip it, and untranslated `title`/`description`/`summary` is the costliest thing to miss. Body boundaries land on `##`/`#` headings and never inside a code fence, so each piece is self-contained. A chunk may run somewhat over the target size when a single section is long; that's expected. + +Chunking is not an optimization, it's a correctness measure. Asked to "translate this 2,700-line document," a model starts condensing somewhere in the middle; asked to translate a bounded section, it translates it. Carry forward the terminology you chose in earlier chunks so a term doesn't change spelling halfway through the file. + +Whatever you do, **translate — do not summarize, improve, or reorganize.** Every heading, row, bullet and code block in the source must appear in the output. Step 4 checks this, and a file that lost content will be sent back. + +### 4. Verify — the gate + +```bash +python scripts/verify_translation.py \ + --original-dir \ + --translated-dir /tmp/collect-staging +``` + +Exit code 0 means clean; 1 means at least one file failed. Failures are specific — line numbers for residual text, counts for dropped structure, named targets for broken anchors. **Fix and re-run until it exits 0.** Nothing moves into `tree/` before that. + +Warnings (`!`) don't block, but read them. The two that matter most: + +- **Line-count delta beyond ±15%** (the default `line_delta_tolerance`). A large *negative* delta usually means a section was summarized away rather than translated — check the chunk seams first. +- **Non-Latin text inside code.** Usually a comment that was missed. Occasionally a legitimate string literal, which is why it warns rather than blocks. + +### 5. Analyze, deduplicate, and place + +Only once the content is in English: + +- Determine the domain (`networking`, `infrastructure`, `security`, `ai`, etc.) and place under `tree//`, using subdirectories where a topic warrants it +- Query the tree for near-duplicates using [[query_knowledge]] — semantic similarity plus metadata overlap +- Decide per file: **new** → write it; **duplicate** → skip and note it; **supplement** → merge or cross-reference + +This ordering is deliberate. Similarity search against an English tree using a source-language query returns noise, so deduplication before translation produces a bad answer. + +### 6. Move files into tree + +Move staged files into the knowledge tree `tree/` folder. Use the folder or subfolders best suited and create the folders needed if they do not exist. + +--- + +## Translation rules + +The single question that governs everything: **will a reader or an embedding index see this text?** If yes it must be English. If it's a machine identifier — a path, a URL, a variable name — it must be left exactly as it is. + +### Always translate + +| What | Why it matters | +|---|---| +| Body prose, headings, list items, table cells | The obvious content | +| Frontmatter `title`, `description`, `summary`, `audience`, `intent_queries` | **These feed the embedding index.** Leaving them in the source language poisons search across the whole tree — the highest-cost mistake available here | +| Comments inside code blocks | They're explanation, not code | +| Link text and wikilink aliases — the part *after* the `\|` | Reader-visible | +| Blockquotes, callouts, admonitions | Reader-visible | + +### Never translate + +| What | What breaks if you do | +|---|---| +| URLs, and link targets inside `](...)` | Dead links | +| Wikilink targets — the part *before* the `\|` in `[[target\|alias]]` | The graph loses the edge | +| Code: identifiers, function names, string literals, CLI flags | The code stops working | +| Code fence info strings (` ```yaml `) | Syntax highlighting breaks; the verifier flags it | +| Frontmatter *keys*, and slug-valued fields (`tags`, `category`, `tier`, dates) | Schema drift | + +### Regenerate, don't translate + +**Heading anchors.** A table of contents entry like `[概述](#1-概述)` has two halves that behave differently: the link text is prose and gets translated, while the `#1-概述` target is *derived* from the heading. Translating it as text produces `#1-overview` only by luck; the reliable move is to rebuild every anchor from the final English heading using GitHub slug rules — lowercase, punctuation dropped, spaces to hyphens. `verify_translation.py` checks that every anchor resolves to a heading that actually exists. + +**Relative paths to other documents.** If a source file links to `../other-domain/doc.md` and that file lands somewhere else in `tree/`, the link needs remapping to the new location, not translating. Cross-corpus links that have no counterpart in the tree should be flagged in the report rather than silently left broken. + +For edge cases — mixed-language files, quoted source-language material that should stay, ambiguous inline code — see `references/translation-rules.md`. + +--- + +## Frontmatter convention + +Keep every key the source already has and keep its slug-valued fields untouched; translate the human-readable values; then add the two provenance fields below **if they aren't already present**: + +```yaml +--- +title: Human-readable title, in English +description: Brief summary, in English — this is indexed, so it must be English +tags: [domain, topic] # slugs, left as-is +created: '2026-08-29' +last_updated: 2026-08-29 +source_path: /original/path/to/file.md +original_language: Chinese # or English when no translation was needed +--- +``` + +`source_path` and `original_language` are what make a bad translation recoverable later — without them there's no way to find the original and redo it. The verifier warns when they're missing. + +--- + +## Auditing content that's already in the tree + +The scanner works on the tree itself, which is how you find content that was imported before this gate existed: + +```bash +python scripts/lang_scan.py tree/ --summary +python scripts/lang_scan.py tree/ --bucket HEAVY --show 5 # worst offenders +python scripts/lang_scan.py tree/ --gate # exit 1 if any remain +``` + +The `--gate` form is suitable for a pre-commit hook or CI step, so untranslated content can't re-enter. + +--- + +## Next steps + +After collecting: + +1. **Review the report** — placements, translations, duplicates, merge candidates +2. **Spot-check a translated file** against its original for terminology accuracy — the verifier checks structure and completeness, not whether "承载网" was rendered well +3. **Handle merge candidates** — integrate and update cross-references +4. **Verify organization** — confirm domain placement; add subdirectories where a topic has grown +5. **Index** — run `make ingest && make reindex` +6. **Query** — use [[query_knowledge]] to retrieve the new material + +## Related + +- Index collected knowledge: [[store_knowledge]] +- Search the knowledge tree: [[query_knowledge]] +- Run analysis on collected material: [[run_research_report]] +- Config (embedding model, chunk sizes): `kb/config.py` \ No newline at end of file diff --git a/.claude/skills/collect_knowledge/references/translation-rules.md b/.claude/skills/collect_knowledge/references/translation-rules.md new file mode 100644 index 0000000..5a85ad7 --- /dev/null +++ b/.claude/skills/collect_knowledge/references/translation-rules.md @@ -0,0 +1,182 @@ +# Translation rules — edge cases + +Read this when a file doesn't fit the straightforward cases in SKILL.md. The +governing question is always the same: **will a reader or the embedding index +see this text?** English if yes; untouched if it's a machine identifier. + +## Mixed-language files + +Common in technical corpora: English headings with source-language body, or +English body with source-language comments in code. Don't decide per file — +`lang_scan.py` reports per line and per category, so work from its output +rather than from an impression of the file. + +A file scanning as `TRACE` (under 5% of lines) is usually one of: + +- a stray untranslated bullet left over from an earlier partial pass +- a proper noun that has no English form — legitimate, see below +- a quoted error message or log line — see below + +All three need a decision, not a blanket skip. + +## Text that should stay in the source language + +These are real, and forcing them to English makes the document worse: + +- **Quoted terminal output, log lines, and error messages** produced by a + localized tool. The reader needs to match what their screen shows. Translate + the surrounding explanation; leave the quoted string, and add a bracketed + English gloss after it if the meaning matters. +- **Proper nouns with no established English form** — a company, a product, a + domestic standard. Use the official English name if one exists; otherwise + transliterate and put the original in parentheses on first use. +- **Deliberate examples of source-language input** — a document about CJK text + handling needs its CJK sample data intact. + +Put these inside inline code or a fenced block where the format allows. That +moves them into a `WARN` category rather than `BLOCK`, so the gate surfaces +them for review instead of failing the file. If a genuine case can't be marked +that way, note it in the collection report so a human can confirm. + +## Inline code containing source-language text + +Ambiguous, and worth slowing down for. Two cases: + +``` +`kubectl get pods` <- a command. Never translate. +`安全组` <- a term someone wrapped in backticks for emphasis. + Translate it, and consider whether it should be + bold rather than code. +``` + +Rule of thumb: if it would be typed into a terminal or appear in source code, +it's code. If it's a concept being named, it's prose that was formatted wrong. + +## Code blocks + +Translate comments. Leave everything else: + +```yaml +# 传统方式：长期 Token 存储在 Secret 中 <- translate this line +apiVersion: v1 +kind: Secret +metadata: + name: default-token-xxx <- never touch +``` + +Two traps: + +- **String literals that are user-facing.** A `message: "操作失败"` in an example + config is a judgement call — translating it makes the example clearer, + leaving it keeps the example faithful. Prefer translating, since these are + illustrative rather than executable, and note it if the example is copied + from a real system. +- **The info string.** ` ```yaml ` is a language identifier. `verify_translation.py` + compares the full sequence of info strings before and after and fails on any + change, because a translated info string breaks highlighting everywhere it + appears. + +## Heading anchors and tables of contents + +The most common silent breakage. A TOC entry has two halves that behave +differently: + +``` +[概述](#1-概述) + ^^^^ ^^^^^^^^ + prose derived identifier +``` + +Translate the first, **regenerate** the second from the final English heading +using GitHub slug rules: + +1. Strip markdown formatting (`**bold**`, `` `code` ``, links → their text) +2. Lowercase +3. Remove everything that isn't a word character, space, or hyphen +4. Spaces → hyphens +5. Duplicate slugs get `-1`, `-2`, … appended in document order + +`mdlang.slugify()` implements exactly this, and `verify_translation.py` fails +the file if any anchor doesn't resolve to a heading that exists. Never +hand-guess an anchor; derive it. + +## Wikilinks + +``` +[[domain-05-security/README.md|返回目录]] + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^ + target — a path alias — reader-visible +``` + +Translate the alias (`返回目录` → `Back to index`). The target is a path: + +- If the referenced document is also being collected, **remap** the target to + its new location under `tree/` +- If it isn't, the link will dangle. Flag it in the report rather than leaving + a broken edge or deleting the link silently — a dangling wikilink is a valid + marker of something worth collecting later, but only if someone knows it's there. + +A bare `[[Kubernetes|Kubernetes]]` with no path refers to a tree note by title; +leave those alone unless the tree uses a different title. + +## Frontmatter + +Any human-readable value must be English, because `title`, `description` and +`summary` are what get embedded — untranslated values there degrade retrieval +for every future query, not just for the file they're in. + +| Field kind | Examples | Action | +|---|---|---| +| Human-readable | `title`, `description`, `summary`, `abstract`, `audience`, `intent_queries` | Translate | +| Slug / enum | `tags`, `category`, `tier`, `difficulty`, `reading_level` | Leave | +| Date / version | `created`, `last_updated`, `k8s_versions` | Leave | +| Provenance | `source_path`, `original_language`, `authors` | Leave; add if missing | + +Unknown keys default to *must be translated*. A corpus you haven't seen before +is far more likely to hide untranslated prose behind an unfamiliar key than to +hold deliberate source-language text there. To exempt a key, add it to +`preserve_frontmatter` in a config file and pass `--config`. + +`trigger_keywords` and `keywords` are a mixed case: usually English technical +terms already, but any source-language entries in them should be translated, +since they exist to be matched against English queries. + +## Tuning the checks for a different corpus + +Everything is config-driven. Write a JSON file with only the keys you're +changing and pass `--config` to either script: + +```json +{ + "preserve_frontmatter": ["tags", "category", "author_native_name"], + "warn_categories": ["inline_code", "code", "code_comment"], + "max_chunk_lines": 250, + "line_delta_tolerance": 0.25 +} +``` + +Useful adjustments: + +- **`preserve_frontmatter`** — add keys that legitimately hold source-language + values, such as a native-script author name. +- **Move `code_comment` from blocking to warning** for a corpus where code + comments are quoted from external sources and shouldn't be rewritten. +- **Lower `max_chunk_lines`** if translations are drifting or losing content; + smaller chunks are more reliable and cost more passes. +- **Raise `line_delta_tolerance`** for language pairs with large length + differences. Chinese → English typically *grows* line count slightly when + prose reflows; a large negative delta is still a red flag in any pair. + +## When the gate keeps failing + +Re-run the failing file through `lang_scan.py --show 30` to see exactly which +lines remain. In order of likelihood: + +1. **Frontmatter was skipped.** The most common miss by a wide margin — the + body reads as English and the file looks done. +2. **The tail of a long file.** Attention drifted near the end. Re-translate + the last chunks specifically rather than the whole file. +3. **Code comments.** Easy to read past when scanning a fenced block. +4. **Anchors.** Headings became English and the TOC didn't follow. +5. **Structure counts off by a few.** Usually a table row or list item dropped + at a chunk boundary — check the seams between chunks first. diff --git a/.claude/skills/collect_knowledge/scripts/lang_scan.py b/.claude/skills/collect_knowledge/scripts/lang_scan.py new file mode 100644 index 0000000..9760866 --- /dev/null +++ b/.claude/skills/collect_knowledge/scripts/lang_scan.py @@ -0,0 +1,237 @@ +#!/usr/bin/env python3 +""" +lang_scan.py -- find content that still needs translating, and plan the work. + +This is the detector half of the translate-then-verify loop. It answers three +questions cheaply, with no model tokens involved: + + 1. Which files in this corpus are not in English, and how badly? + 2. Within a file, *which lines* -- and is that line prose that must be + translated, or a URL that must not be? + 3. How should a large file be cut into translation-sized pieces? + +Usage +----- + # Triage a corpus before starting + lang_scan.py ~/corpus --summary + + # Per-file detail with sample lines + lang_scan.py ~/corpus/doc.md --show 20 + + # Machine-readable, for driving a batch + lang_scan.py ~/corpus --json > scan.json + + # Plan the chunks for one large file + lang_scan.py ~/corpus/big.md --plan + + # Use as a CI / pre-merge gate (exit 1 if anything blocking remains) + lang_scan.py ./tree --gate + +Exit codes: 0 = no blocking findings (or --gate not passed), 1 = blocking +findings present, 2 = bad invocation. +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +# Resolve sibling imports so this script runs from any working directory. +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import mdlang +from mdlang import ( + SCRIPT_TO_LANGUAGE_HINT, + Finding, + density_bucket, + discover, + dominant_script, + load_config, + plan_chunks, + read_text, + scan_text, + severity, +) + + +def analyse(path: Path, config: dict) -> dict: + text = read_text(path) + findings, per_cat = scan_text(text, config) + total_lines = text.count("\n") + 1 + lines_with = len({f.line for f in findings}) + blocking = [f for f in findings if severity(f.category, config) == "BLOCK"] + warning = [f for f in findings if severity(f.category, config) == "WARN"] + script = dominant_script(text) + return { + "path": str(path), + "total_lines": total_lines, + "lines_with_non_latin": lines_with, + "bucket": density_bucket(lines_with, total_lines), + "dominant_script": script, + "language_hint": SCRIPT_TO_LANGUAGE_HINT.get(script or "", "unknown") if script else None, + "by_category": per_cat, + "blocking_chars": sum(f.count for f in blocking), + "warning_chars": sum(f.count for f in warning), + "n_blocking_lines": len({f.line for f in blocking}), + "findings": [ + { + "line": f.line, + "category": f.category, + "severity": severity(f.category, config), + "script": f.script, + "count": f.count, + "text": f.text, + } + for f in findings + ], + } + + +def print_file_report(rep: dict, show: int, config: dict) -> None: + rel = rep["path"] + hint = rep["language_hint"] or "-" + print(f"\n\033[1m{rel}\033[0m") + print( + f" {rep['bucket']} " + f"{rep['lines_with_non_latin']}/{rep['total_lines']} lines contain non-Latin text " + f"(likely {hint})" + ) + if rep["by_category"]: + parts = [] + for cat in mdlang.ALL_CATEGORIES: + n = rep["by_category"].get(cat) + if n: + sev = severity(cat, config) + mark = {"BLOCK": "!", "WARN": "~", "INFO": " "}[sev] + parts.append(f"{mark}{cat}={n}") + print(" " + " ".join(parts)) + if show: + blocking = [f for f in rep["findings"] if f["severity"] == "BLOCK"] + for f in blocking[:show]: + print(f" L{f['line']:<5} {f['category']:<14} {f['text']}") + if len(blocking) > show: + print(f" ... {len(blocking) - show} more blocking lines " + f"(raise --show to see them)") + + +def cmd_plan(paths: list[Path], config: dict, as_json: bool) -> int: + out = [] + for p in paths: + text = read_text(p) + chunks = plan_chunks(text, config["max_chunk_lines"]) + entry = { + "path": str(p), + "total_lines": text.count("\n") + 1, + "n_chunks": len(chunks), + "chunks": [ + { + "index": c.index, + "start_line": c.start_line, + "end_line": c.end_line, + "lines": c.end_line - c.start_line + 1, + "heading": c.heading, + } + for c in chunks + ], + } + out.append(entry) + if not as_json: + print(f"\n\033[1m{p}\033[0m ({entry['total_lines']} lines -> {len(chunks)} chunks)") + for c in chunks: + print(f" chunk {c.index:<3} lines {c.start_line}-{c.end_line} " + f"({c.end_line - c.start_line + 1:>4}) {c.heading}") + if as_json: + json.dump(out, sys.stdout, indent=2, ensure_ascii=False) + print() + return 0 + + +def main(argv: list[str] | None = None) -> int: + ap = argparse.ArgumentParser( + prog="lang_scan.py", + description="Detect untranslated content in a markdown corpus.", + ) + ap.add_argument("paths", nargs="+", help="files or directories to scan") + ap.add_argument("--json", action="store_true", help="machine-readable output") + ap.add_argument("--summary", action="store_true", + help="corpus-level counts only, no per-file detail") + ap.add_argument("--plan", action="store_true", + help="emit a chunk plan for each file instead of scanning") + ap.add_argument("--gate", action="store_true", + help="exit 1 if any blocking findings remain") + ap.add_argument("--show", type=int, default=8, + help="sample blocking lines to print per file (default 8)") + ap.add_argument("--clean-only", action="store_true", + help="list only files with no findings") + ap.add_argument("--bucket", choices=["HEAVY", "PARTIAL", "TRACE", "CLEAN"], + help="list only files in this density bucket") + ap.add_argument("--config", help="JSON config overriding the defaults") + ap.add_argument("--ext", action="append", + help="file extension to include (repeatable; default .md .markdown .txt .rst .mdx)") + args = ap.parse_args(argv) + + config = load_config(args.config) + + files: list[Path] = [] + for raw in args.paths: + p = Path(raw).expanduser() + if not p.exists(): + print(f"error: no such path: {p}", file=sys.stderr) + return 2 + files.extend(discover(p, args.ext)) + if not files: + print("error: no matching files found", file=sys.stderr) + return 2 + + if args.plan: + return cmd_plan(files, config, args.json) + + reports = [analyse(f, config) for f in files] + + if args.bucket: + reports = [r for r in reports if r["bucket"] == args.bucket] + if args.clean_only: + reports = [r for r in reports if r["blocking_chars"] == 0] + + if args.json: + json.dump( + { + "n_files": len(reports), + "n_blocking_files": sum(1 for r in reports if r["blocking_chars"]), + "files": reports, + }, + sys.stdout, indent=2, ensure_ascii=False, + ) + print() + else: + buckets: dict[str, int] = {} + scripts: dict[str, int] = {} + for r in reports: + buckets[r["bucket"]] = buckets.get(r["bucket"], 0) + 1 + if r["dominant_script"]: + scripts[r["dominant_script"]] = scripts.get(r["dominant_script"], 0) + 1 + + if not args.summary: + for r in reports: + if r["blocking_chars"] or r["warning_chars"] or args.clean_only: + print_file_report(r, args.show, config) + + print(f"\n\033[1m{len(reports)} file(s) scanned\033[0m") + for b in ("HEAVY", "PARTIAL", "TRACE", "CLEAN"): + if buckets.get(b): + print(f" {b:<8} {buckets[b]}") + if scripts: + print(" scripts: " + ", ".join(f"{k}={v}" for k, v in sorted(scripts.items()))) + n_block = sum(1 for r in reports if r["blocking_chars"]) + print(f" {n_block} file(s) have blocking findings " + f"({sum(r['blocking_chars'] for r in reports)} characters)") + + if args.gate and any(r["blocking_chars"] for r in reports): + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.claude/skills/collect_knowledge/scripts/mdlang.py b/.claude/skills/collect_knowledge/scripts/mdlang.py new file mode 100644 index 0000000..1245d61 --- /dev/null +++ b/.claude/skills/collect_knowledge/scripts/mdlang.py @@ -0,0 +1,781 @@ +#!/usr/bin/env python3 +""" +mdlang.py -- markdown + language primitives shared by the collect_knowledge skill. + +Why this exists +--------------- +A plain `grep -P '[\\x{4e00}-\\x{9fff}]'` cannot tell the difference between: + + 这是一段还没翻译的正文 <- a paragraph nobody translated + [文档](https://example.com/文档) <- a URL that must NEVER be translated + [[notes/index.md|返回目录]] <- target must stay, alias must translate + ](#1-供应链安全简介) <- an anchor that must be REGENERATED + +Treating those four the same produces either false alarms (blocking on URLs) or +false confidence (passing a file whose whole body is untranslated). So this +module paints a document into categories and reports non-Latin text per +category, which is what lets the scanner and the verifier make useful calls. + +Nothing here is specific to any repository or language pair. Callers supply +paths and, optionally, a config dict; the defaults are meant to be reasonable +for any markdown corpus. +""" + +from __future__ import annotations + +import json +import re +from dataclasses import dataclass, field +from pathlib import Path + +# -------------------------------------------------------------------------- +# Script detection +# -------------------------------------------------------------------------- +# Ranges chosen to cover the scripts that realistically show up in technical +# documentation corpora. Greek is deliberately absent: alpha/beta/sigma appear +# constantly in legitimately-English math and would generate endless noise. + +SCRIPT_RANGES: list[tuple[int, int, str]] = [ + (0x0400, 0x04FF, "Cyrillic"), + (0x0500, 0x052F, "Cyrillic"), + (0x0590, 0x05FF, "Hebrew"), + (0x0600, 0x06FF, "Arabic"), + (0x0750, 0x077F, "Arabic"), + (0x0900, 0x097F, "Devanagari"), + (0x0980, 0x09FF, "Bengali"), + (0x0E00, 0x0E7F, "Thai"), + (0x1100, 0x11FF, "Hangul"), + (0x3000, 0x303F, "CJK-Punctuation"), + (0x3040, 0x309F, "Hiragana"), + (0x30A0, 0x30FF, "Katakana"), + (0x3400, 0x4DBF, "CJK"), + (0x4E00, 0x9FFF, "CJK"), + (0xA960, 0xA97F, "Hangul"), + (0xAC00, 0xD7AF, "Hangul"), + (0xD7B0, 0xD7FF, "Hangul"), + (0xF900, 0xFAFF, "CJK"), + (0xFE30, 0xFE4F, "CJK-Punctuation"), + (0xFF00, 0xFFEF, "Fullwidth"), + (0x20000, 0x2A6DF, "CJK"), +] + +# Fullwidth digits/latin live in the Fullwidth block and signal CJK authoring, +# but the block also holds halfwidth katakana and a few neutral forms. Keep it. + +_SCRIPT_CACHE: dict[str, str | None] = {} + + +def script_of(ch: str) -> str | None: + """Return the script name for a non-Latin character, or None if Latin/neutral.""" + cached = _SCRIPT_CACHE.get(ch) + if cached is not None or ch in _SCRIPT_CACHE: + return cached + cp = ord(ch) + result = None + for lo, hi, name in SCRIPT_RANGES: + if lo <= cp <= hi: + result = name + break + _SCRIPT_CACHE[ch] = result + return result + + +def dominant_script(text: str) -> str | None: + """The most frequent non-Latin script in `text`, or None if there is none.""" + counts: dict[str, int] = {} + for ch in text: + s = script_of(ch) + if s and s not in ("CJK-Punctuation", "Fullwidth"): + counts[s] = counts.get(s, 0) + 1 + if not counts: + # Fall back to punctuation-only evidence (a doc of pure 。、「」 is odd + # but still not English). + for ch in text: + s = script_of(ch) + if s: + counts[s] = counts.get(s, 0) + 1 + if not counts: + return None + return max(counts.items(), key=lambda kv: kv[1])[0] + + +# Human-friendly names for report output. +SCRIPT_TO_LANGUAGE_HINT = { + "CJK": "Chinese (or Japanese kanji)", + "Hiragana": "Japanese", + "Katakana": "Japanese", + "Hangul": "Korean", + "Cyrillic": "Russian (or another Cyrillic language)", + "Arabic": "Arabic", + "Hebrew": "Hebrew", + "Devanagari": "Hindi (or another Devanagari language)", + "Bengali": "Bengali", + "Thai": "Thai", +} + + +# -------------------------------------------------------------------------- +# Categories +# -------------------------------------------------------------------------- + +CAT_PROSE = "prose" +CAT_FRONTMATTER = "frontmatter" +CAT_HEADING = "heading" +CAT_CODE = "code" +CAT_CODE_COMMENT = "code_comment" +CAT_INLINE_CODE = "inline_code" +CAT_ANCHOR = "anchor" +CAT_LINK_TARGET = "link_target" +CAT_WIKILINK_TARGET = "wikilink_target" + +ALL_CATEGORIES = [ + CAT_PROSE, + CAT_FRONTMATTER, + CAT_HEADING, + CAT_CODE_COMMENT, + CAT_ANCHOR, + CAT_INLINE_CODE, + CAT_CODE, + CAT_LINK_TARGET, + CAT_WIKILINK_TARGET, +] + +# What each category means for a translation gate: +# +# blocking -- untranslated content that will reach a reader or an embedding +# index. Never merge a file with these. +# warn -- probably should be translated, but a literal identifier or a +# deliberate quotation is plausible. Surface, do not block. +# info -- expected to stay in the source language (paths, URLs). +DEFAULT_BLOCKING = [CAT_PROSE, CAT_FRONTMATTER, CAT_HEADING, CAT_CODE_COMMENT, CAT_ANCHOR] +DEFAULT_WARN = [CAT_INLINE_CODE, CAT_CODE] +DEFAULT_INFO = [CAT_LINK_TARGET, CAT_WIKILINK_TARGET] + + +DEFAULT_CONFIG = { + # Frontmatter keys whose *values* carry human-readable text and therefore + # must end up in English. These feed titles, summaries and embeddings. + "translatable_frontmatter": [ + "title", "description", "summary", "abstract", "excerpt", + "audience", "intent_queries", "trigger_keywords", "keywords", + "aliases", "note", "notes", + ], + # Frontmatter keys exempt from the scan: slugs, dates, enums, provenance. + # A value here may legitimately hold non-Latin text (e.g. an author name). + "preserve_frontmatter": [ + "tags", "category", "tier", "created", "last_updated", "date", + "difficulty", "reading_level", "estimated_read_time", + "source_path", "original_language", "authors", "author", + "prerequisites", "k8s_versions", "versions", "slug", "id", "type", + ], + "blocking_categories": DEFAULT_BLOCKING, + "warn_categories": DEFAULT_WARN, + "max_chunk_lines": 400, + "line_delta_tolerance": 0.15, + # Line prefixes treated as comments inside a fenced code block. Comments + # carry explanation and should be translated; surrounding code should not. + # Override for corpora dominated by a language this list misses. + "comment_prefixes": ["#", "//", "--", "/*", "*", ";", "%", "