Run the pre-commit hooks on Windows

.claude/test/test_hil_remote.py

@@ -39,6 +39,9 @@
 needs_rig_python = unittest.skipUnless(shutil.which('python3', path=RIG_PATH), f'no python3 on the fake rig PATH ({RIG_PATH})')
 # the fake ssh, rsync and python3 are #! scripts Windows cannot exec: a run there would reach the real ones
 needs_fake_rig = unittest.skipIf(os.name == 'nt', 'the fake rig is #! scripts and a POSIX shell')
+# Windows refusals keep git, which the agentrc loader runs, from a dir with no real ssh or rsync
+GIT_DIR = os.path.dirname(shutil.which('git') or '')
+GIT_ALONE = bool(GIT_DIR) and not any(shutil.which(t, path=GIT_DIR) for t in ('ssh', 'rsync'))
 
 FAKE_SSH = f'''#!{sys.executable}
 import json, os, subprocess, sys
@@ -151,13 +154,15 @@ def build(self, *variants, root='cmake-build'):
     def env(self):
         e = {k: v for k, v in os.environ.items() if not k.startswith('HIL_') and k not in ('CI', 'GITHUB_ACTIONS')}
         e.update(ROOT_DIR=str(self.root), REMOTE='rig', REMOTE_DIR=str(self.remote), AGENTRC_DIR=AGENTRC['root'],
-                 PATH=f'{self.bin}:{os.environ["PATH"]}', FAKE_LOG=str(self.log),
+                 PATH=os.pathsep.join([str(self.bin), os.environ['PATH']]), FAKE_LOG=str(self.log),
                  FAKE_HOME=str(self.home), FAKE_RUN=str(self.run_file),
                  FAKE_DEFAULT=DEFAULT_REPORT)
         e.pop('CONFIG', None)
         if os.name == 'nt':
             # only the refusals run here: a REMOTE_DIR they accept, and no real ssh or rsync on PATH
-            e.update(REMOTE_DIR='/tmp/hil-remote-test', PATH=str(self.bin))
+            if not GIT_ALONE:
+                self.skipTest(f'git dir {GIT_DIR or "(none)"} also holds ssh or rsync')
+            e.update(REMOTE_DIR='/tmp/hil-remote-test', PATH=os.pathsep.join([str(self.bin), GIT_DIR]))
         return e
 
     def hil_remote(self, *argv, **env):

.codespellrc

@@ -1,7 +1,8 @@
 # See: https://github.com/codespell-project/codespell#using-a-config-file
 [codespell]
 # In the event of a false positive, add the problematic word, in all lowercase, to 'ignore-words.txt' (one word per line).
-# Or copy & paste the whole problematic line to 'exclude-file.txt'
+# Or copy & paste the whole problematic line to 'exclude-file.txt' (codespell 2.2.4 matches it with its line
+# ending, so an entry misses on a CRLF checkout, as Windows has: prefer ignore-words.txt)
 ignore-words = tools/codespell/ignore-words.txt
 exclude-file = tools/codespell/exclude-file.txt
 check-filenames =

.github/scripts/ci_set_matrix.py

@@ -118,7 +118,7 @@
 
 
 def pinned_boards(repo_root=REPO):
-    with open(os.path.join(repo_root, '.github', 'ci-pinned-boards.json')) as f:
+    with open(os.path.join(repo_root, '.github', 'ci-pinned-boards.json'), encoding='utf-8') as f:
         return {e['board'] for e in json.load(f)['boards']}
 
 
@@ -288,7 +288,7 @@ def main():
         if args.select:
             select = json.loads(args.select)
         elif args.select_file:
-            with open(args.select_file) as f:
+            with open(args.select_file, encoding='utf-8') as f:
                 select = json.load(f)
         elif args.base:
             r = subprocess.run([sys.executable, os.path.join(REPO, '.claude', 'skills', 'build', 'scripts',

.github/scripts/code_size_ci.py

@@ -19,6 +19,7 @@
 import argparse
 import json
 import os
+import shutil
 import subprocess
 import sys
 
@@ -33,11 +34,14 @@
 
 def gh(path, paginate=False):
     """Parsed `gh api` GET of `path`; with `paginate`, the list of every page."""
-    cmd = ['gh', 'api', '-H', 'Accept: application/vnd.github+json'] + (['--paginate', '--slurp'] if paginate else [])
+    # by PATH: Windows' CreateProcess searches System32 first and only for gh.exe
+    cmd = [shutil.which('gh') or 'gh', 'api', '-H', 'Accept: application/vnd.github+json'] + (['--paginate', '--slurp'] if paginate else [])
     try:
         ret = subprocess.run(cmd + [path], capture_output=True, text=True, check=False, timeout=GH_TIMEOUT)
     except subprocess.TimeoutExpired:
         raise RuntimeError(f'gh api {path}: no reply in {GH_TIMEOUT} s') from None
+    except OSError as e:  # no runnable gh is a failed lookup too
+        raise RuntimeError(f'gh api {path}: {e}') from None
     if ret.returncode != 0:
         raise RuntimeError(f'gh api {path}: {ret.stderr.strip()}')
     return json.loads(ret.stdout)
@@ -127,7 +131,7 @@ def baseline(args):
         info = lookup(args.repo, args.current)
     except (RuntimeError, KeyError, TypeError, ValueError) as e:  # an API failure or an unexpected reply
         info = {'sha': None, 'url': None, 'exact': False, 'run_id': None, 'note': f'the baseline lookup failed: {e}'}
-    with open(args.info, 'w') as f:
+    with open(args.info, 'w', encoding='utf-8') as f:
         json.dump(info, f, indent=1, sort_keys=True)
         f.write('\n')
     print(json.dumps(info))

.github/scripts/hil_ci_set_matrix.py

@@ -47,7 +47,7 @@ def main():
     hil = None
     try:
         if args.select_file:
-            with open(args.select_file) as f:
+            with open(args.select_file, encoding='utf-8') as f:
                 raw = f.read()
         if raw:
             # imported here, under the fall-open: a broken selector must still leave the
@@ -83,7 +83,7 @@ def append_build_arg(toolchain, build_arg):
             matrix[toolchain].append(build_arg)
 
     for config_file in args.config_files:
-        with open(_resolve_config_path(config_file)) as f:
+        with open(_resolve_config_path(config_file), encoding='utf-8') as f:
             config = json.load(f)
 
         for board in config['boards']:

.pre-commit-config.yaml

@@ -12,14 +12,19 @@ repos:
       (?x)^(
         hw/bsp/mcx/sdk/
       )
+  # AGENTS.md and .agents are symlinks: without core.symlinks (Windows) git checks them out as
+  # plain files holding the target, which would get a newline appended
   - id: end-of-file-fixer
     exclude: |
       (?x)^(
         .idea/|
-        hw/bsp/mcx/sdk/
+        hw/bsp/mcx/sdk/|
+        AGENTS\.md$|
+        \.agents$
       )
   - id: forbid-submodules
 
+# codespell itself is excluded from tools/codespell/, whose files hold the misspellings on purpose
 - repo: https://github.com/codespell-project/codespell
   rev: v2.2.4
   hooks:
@@ -28,32 +33,38 @@ repos:
     exclude: |
       (?x)^(
         lib/|
-        hw/bsp/mcx/sdk/
+        hw/bsp/mcx/sdk/|
+        tools/codespell/
       )
 
 - repo: local
   hooks:
+  # A multi-step or cd-ing entry goes through tools/hook_steps.py: Windows has no sh to run `sh -c`.
+  # A Python hook is language: python with a `python` entry, not language: system with
+  # `python3`: a python.org Windows install has no python3.exe, while pre-commit's venv
+  # supplies `python` on every OS. Non-Python commands (ceedling, cmake) still come from PATH.
   - id: unique-example-pids
     name: unique example USB PIDs
     files: usb_descriptors\.c$
-    entry: python3 tools/check_example_pids.py
+    entry: python tools/check_example_pids.py
     pass_filenames: false
-    language: system
+    language: python
 
   - id: gen-doc
     name: regenerate reference docs
     files: ^(hw/bsp/.*/(board\.h|family\.c)|tools/get_deps\.py|test/hil/(tinyusb|hfp)\.json|\.claude/skills/build-doc/scripts/gen_doc\.py)$
-    entry: python3 .claude/skills/build-doc/scripts/gen_doc.py
+    entry: python .claude/skills/build-doc/scripts/gen_doc.py
     pass_filenames: false
-    language: system
+    language: python
 
   - id: unit-test
     name: unit-test
     files: ^(src/|test/unit-test/)
-    entry: sh -c "cd test/unit-test && ceedling test:all"
+    entry: python tools/hook_steps.py
+    args: ['@test/unit-test ceedling test:all']
     pass_filenames: false
     types_or: [c, header]
-    language: system
+    language: python
 
   # Two hooks, split by what each suite RUNS, not by what it reads: discovery is
   # disjoint (test_hil*.py vs the two named suites) so nothing runs twice, but the
@@ -75,32 +86,43 @@ repos:
   # examples/device/mtp/src is in scope: test_hil_bounded parses README_TXT_CONTENT
   # and md5-checks the logo header from there as its MTP fixtures.
   # language: python so pre-commit's venv supplies pyusb and pyserial, which the rig
-  # scripts under test import.
+  # scripts under test import. language: python hooks run `python`, not `python3`:
+  # a Windows venv has no python3.exe, so python3 would be the system interpreter.
   - id: hil-test
     name: hil-test
     files: ^(test/hil/|examples/device/mtp/src/|\.claude/skills/hil-contract/scripts/hil_remote\.py$|\.github/workflows/pr_comment\.yml$)
-    entry: python3 -m unittest discover -s test/hil/test -p 'test_hil*.py'
+    entry: python -m unittest discover -s test/hil/test -p 'test_hil*.py'
     pass_filenames: false
     language: python
     additional_dependencies: [pyusb, pyserial]
   - id: family-json
     name: family-json
-    entry: sh -c "python3 tools/family_json.py fix && python3 test/hil/test/test_family_json.py"
+    entry: python tools/hook_steps.py
+    args: ['python tools/family_json.py fix', 'python test/hil/test/test_family_json.py']
     always_run: true
     pass_filenames: false
-    language: system
+    language: python
   - id: change-impact-test
     name: change-impact-test
     files: ^(hw/bsp/|hw/mcu/|src/|examples/|test/hil/|tools/(change_impact|build|build_utils|family_json|get_deps)\.py$|\.github/(scripts|workflows)/|\.circleci/)
-    entry: sh -c "python3 test/hil/test/test_change_impact.py && python3 test/hil/test/test_ci_metrics.py && cd test/hil/test && python3 -m unittest -q test_hil_util.BottomLayer"
+    entry: python tools/hook_steps.py
+    args: ['python test/hil/test/test_change_impact.py', 'python test/hil/test/test_ci_metrics.py',
+           '@test/hil/test python -m unittest -q test_hil_util.BottomLayer']
     pass_filenames: false
-    language: system
+    language: python
+  # the runner of the multi-step hooks above: stop at the first failure, @DIR, python -> venv
+  - id: hook-steps-test
+    name: hook-steps-test
+    files: ^(tools/hook_steps\.py|test/hil/test/test_hook_steps\.py)$
+    entry: python test/hil/test/test_hook_steps.py
+    pass_filenames: false
+    language: python
   - id: setup-toolchain-test
     name: setup-toolchain-test
     files: ^(\.github/actions/setup_toolchain/|test/hil/test/test_setup_toolchain\.py$)
-    entry: python3 test/hil/test/test_setup_toolchain.py
+    entry: python test/hil/test/test_setup_toolchain.py
     pass_filenames: false
-    language: system
+    language: python
   # a second copy lets GHA and CircleCI build on different IDF versions after a bump
   - id: esp-idf-image-pin
     name: esp-idf image pinned only in toolchain.json
@@ -115,15 +137,16 @@ repos:
   - id: claude-scripts
     name: claude-scripts
     files: ^(\.claude/(skills/.*/(scripts/|SKILL\.md|[^/]*\.py$)|test/)|tools/(build_utils|family_json|get_deps)\.py$|test/hil/(agentrc\.json|helper/agentrc\.py)$)
-    entry: python3 -m unittest discover -s .claude/test -p 'test_*.py'
+    entry: python -m unittest discover -s .claude/test -p 'test_*.py'
     pass_filenames: false
     language: python
     additional_dependencies: [pyyaml]
 
   - id: drivers-coverage
     name: drivers-coverage
     files: ^(\.github/ci-pinned-boards\.json$|\.github/scripts/ci_set_matrix\.py$|tools/drivers_coverage_check\.py$|tools/change_impact\.py$|tools/build_utils\.py$|hw/bsp/|src/portable/|examples/|test/hil/test/test_drivers_coverage\.py$|tools/build\.py$|test/hil/test/test_ci_boards\.py$|tools/code_size\.py$|tools/code_size_profile\.py$|test/hil/agentrc\.json$|test/hil/helper/agentrc\.py$|test/hil/test/test_code_size\.py$|\.github/scripts/code_size_ci\.py$|test/hil/test/test_code_size_ci\.py$|tools/membrowse_cli\.py$|test/hil/test/test_membrowse_cli\.py$|test/hil/tinyusb\.json$|test/hil/hfp\.json$)
-    entry: sh -c "cd test/hil/test && python3 -m unittest -q test_drivers_coverage test_ci_boards test_code_size test_code_size_ci test_membrowse_cli"
+    entry: python tools/hook_steps.py
+    args: ['@test/hil/test python -m unittest -q test_drivers_coverage test_ci_boards test_code_size test_code_size_ci test_membrowse_cli']
     pass_filenames: false
     language: python
     additional_dependencies: [pyyaml, pyelftools]
@@ -132,8 +155,8 @@ repos:
     name: membrowse-targets
     # build.py's toolchain-free target list vs a real configure: only the files that list reads
     files: ^(tools/(build|build_utils|membrowse_cli)\.py|hw/bsp/.*\.(cmake|mk)|examples/.*/(skip|only)\.txt|examples/.*CMakeLists\.txt|test/hil/test/test_membrowse_targets\.py)$
-    entry: python3 test/hil/test/test_membrowse_targets.py
-    language: system
+    entry: python test/hil/test/test_membrowse_targets.py
+    language: python
     pass_filenames: false
 
 #  - id: build-fuzzer

test/hil/helper/hil_report.py

@@ -835,7 +835,7 @@ def sequence(config_path, requested: list, report_dir: Path) -> dict:
     tests on the same rows. `current` is the last attempt's own run, its rows the requested boards'.
     summarize() stays the snapshot verdict."""
     c, s = core(), schema()
-    cfg = json.loads(Path(config_path).read_text())
+    cfg = json.loads(Path(config_path).read_text(encoding='utf-8'))
     roster = c.roster_identity(config_path)
     chain, locked_before, cells_before = [], None, {}
     for ctx in c.read_chain(report_dir, s):
@@ -931,7 +931,7 @@ def _report_args(argv: list, **parser) -> tuple | None:
                     help='boards to report on; default: every board in the config')
     ap.add_argument('--report-dir', default='.', help=f'where {REPORT_JSON} lives (default: cwd)')
     a = ap.parse_args(argv)
-    cfg = json.loads(Path(a.config_file).read_text())
+    cfg = json.loads(Path(a.config_file).read_text(encoding='utf-8'))
     jpath = Path(a.report_dir) / REPORT_JSON
     if not jpath.is_file():
         print(f'error: {jpath} not found -- did hil_test.py run in this directory?', file=sys.stderr)

test/hil/sysview_ci.py

@@ -22,7 +22,7 @@
 def workload_event():
     """The workload window's anchor: the span of tud_cdc_read activity, the throughput test's
     own footprint."""
-    with open(EVENT_MAP) as f:
+    with open(EVENT_MAP, encoding="utf-8") as f:
         return next(int(i) for i, n in json.load(f).items() if n == WORKLOAD_EVENT_NAME)
 
 GATE_MIN_N = 50
@@ -51,7 +51,7 @@ def load_set(d):
     # recursive: a base download puts each artifact in its own subdirectory
     for p in glob.glob(os.path.join(d, "**", "sysview-*.json"), recursive=True):
         try:
-            with open(p) as f:
+            with open(p, encoding="utf-8") as f:
                 j = json.load(f)
         except (OSError, ValueError):
             continue
@@ -431,9 +431,9 @@ def unrecorded_boards(reports_dir, pr_dir, roster_dir=None):
     for p in glob.glob(os.path.join(reports_dir, "hil-report-*", hil_report.REPORT_JSON)):
         roster = os.path.join(roster_dir, os.path.basename(os.path.dirname(p))[len("hil-report-"):])
         try:
-            with open(roster) as f:
+            with open(roster, encoding="utf-8") as f:
                 cfg = json.load(f)
-            with open(p) as f:
+            with open(p, encoding="utf-8") as f:
                 doc = json.load(f)
             rows = {r["board"]: r.get("cells") for r in doc.get("rows") or []
                     if isinstance(r, dict) and isinstance(r.get("board"), str) and not r.get("retired")}
@@ -549,7 +549,7 @@ def _err_excerpt(text, head=500, tail=500):
 def read_manifest(fw):
     """The image's <target>.sysview.json (family_support.cmake) and the image's sha256."""
     import hashlib
-    with open(os.path.splitext(fw)[0] + ".sysview.json") as f:
+    with open(os.path.splitext(fw)[0] + ".sysview.json", encoding="utf-8") as f:
         m = json.load(f)
     with open(fw, "rb") as f:
         return m, hashlib.sha256(f.read()).hexdigest()
@@ -647,7 +647,7 @@ def capture_result(board, commit, capture=None, error=None, park_error=None, ide
             "metrics": None, "error": error or (NOT_DECODED if capture else None), **(identity or {})}
 
 def write_result(out_dir, result):
-    with open(os.path.join(out_dir, f"sysview-{result['board']}.json"), "w") as f:
+    with open(os.path.join(out_dir, f"sysview-{result['board']}.json"), "w", encoding="utf-8") as f:
         json.dump(result, f)
 
 def capture_one(board, fw, manifest, out_dir):
@@ -856,7 +856,7 @@ def baseline(repo, pr, current):
     records = []
     for p in glob.glob(os.path.join(current, "**", "sysview-*.json"), recursive=True):
         try:
-            with open(p) as f:
+            with open(p, encoding="utf-8") as f:
                 records.append(json.load(f))
         except (OSError, ValueError):
             records.append(None)
@@ -922,17 +922,17 @@ def main():
             info = baseline(args.repo, args.pr, args.current)
         except Exception as e:   # an API failure or an unexpected reply is a result, the comment says so
             info = {**NO_BASELINE, "note": f"the baseline lookup failed: {type(e).__name__}: {e}"}
-        with open(args.info, "w") as f:
+        with open(args.info, "w", encoding="utf-8") as f:
             json.dump(info, f)
         print(json.dumps(info))
     elif args.cmd == "report":
         info = None
         if args.baseline_info:
-            with open(args.baseline_info) as f:
+            with open(args.baseline_info, encoding="utf-8") as f:
                 info = json.load(f)
         unrecorded = unrecorded_boards(args.hil_reports, args.pr_dir) if args.hil_reports else ()
         md = report(args.base_dir, args.pr_dir, info, unrecorded)
-        with open(args.out, "w") as f:
+        with open(args.out, "w", encoding="utf-8") as f:
             f.write(md)
         print(f"{'empty (no captures)' if not md else args.out}")
     elif args.cmd == "unrecorded":
@@ -945,13 +945,13 @@ def main():
         # recursive: a downloaded artifact sits in its own subdirectory, its raw files beside its records
         for path in sorted(glob.glob(os.path.join(args.dir, "**", "sysview-*.json"), recursive=True)):
             try:
-                with open(path) as f:
+                with open(path, encoding="utf-8") as f:
                     result = json.load(f)
                 capture = result.get("capture") or {}
                 # an undecoded capture only: a decoded one may have no raw left beside it to decode again
                 if capture.get("status") == "captured" and result.get("error") == NOT_DECODED:
                     result = decode_one(result, os.path.dirname(path), deadline)
-                    with open(path, "w") as f:
+                    with open(path, "w", encoding="utf-8") as f:
                         json.dump(result, f)
                 park_error = capture.get("park_error")
                 print(f"{result['board']}: {result['error'] or 'ok'}" + (f" ({park_error})" if park_error else ""))