226 explore using java for autos

.gitignore

@@ -1,6 +1,10 @@
 # This gitignore has been specially created by the WPILib team.
 # If you remove items from this file, intellisense might break.
 
+### Python ###
+__pycache__/
+*.pyc
+
 ### C++ ###
 # Prerequisites
 *.d

auto_generator/build_autos.py

@@ -2,51 +2,45 @@
 """
 build_autos.py
 
-Discovers and imports all auto-definition modules in src/, runs them to
-register autos with auto_lib, then writes a single Autos.json and/or
-publishes to the robot's tuning server.
+Orchestrates auto generation and publishing. The autos themselves are authored
+in Java (see src/main/java/frc/robot/autogen) and serialized to Autos.json by a fast Java
+generator that reuses the robot's own coppercore Gson configuration. This script
+invokes that generator, then writes the local Autos.json and/or publishes to the
+robot's tuning server (verifying a structural round-trip on PUT).
 
-Helper modules (auto_lib, shorthands, constants, field_locations, auto_action, units)
-are excluded from auto-discovery.
-
-Usage (from auto_generator/):
+Usage (from the repo root or auto_generator/):
     python build_autos.py              # write Autos.json to deploy dir
     python build_autos.py --sim        # also publish to sim (localhost:8088)
     python build_autos.py --robot      # also publish to robot (10.4.1.2:8088)
     python build_autos.py --url HOST   # also publish to a custom address
     python build_autos.py --no-file    # skip writing the local file
+    python build_autos.py --bootstrap  # recompile robot classes + classpath first
 """
 
 from __future__ import annotations
 
 import argparse
-import importlib
+import importlib.util
 import json
+import subprocess
 import sys
 import urllib.request
 import urllib.error
 from pathlib import Path
 from typing import Any
 
-# Ensure the auto_generator directory is on the path so `src` is importable
 _SCRIPT_DIR = Path(__file__).resolve().parent
-sys.path.insert(0, str(_SCRIPT_DIR))
-
-from src import auto_lib
-
-# Helper modules that should not be treated as auto files
-EXCLUDED_MODULES = {
-    "__init__",
-    "auto_lib",
-    "auto_action",
-    "units",
-    "shorthands",
-    "constants",
-    "field_locations",
-    "routines",
-}
-
-OUTPUT_DIR = _SCRIPT_DIR.parent / "src" / "main" / "deploy" / "constants"
+_REPO_ROOT = _SCRIPT_DIR.parent
+
+# The autos are authored in Java (src/main/java/frc/robot/autogen) and serialized to
+# Autos.json by the cross-platform generator driver, which reuses the robot's own
+# coppercore Gson configuration. Load it as a module (it sits next to this file).
+_GENERATOR_PATH = _SCRIPT_DIR / "generate.py"
+_spec = importlib.util.spec_from_file_location("autogen_generate", _GENERATOR_PATH)
+_autogen = importlib.util.module_from_spec(_spec)
+_spec.loader.exec_module(_autogen)
+
+OUTPUT_DIR = _REPO_ROOT / "src" / "main" / "deploy" / "constants"
 CONFIG_FILE = OUTPUT_DIR / "config.json"
 
 SIM_URL = "http://localhost:8088"
@@ -70,14 +64,16 @@ def detect_environment() -> str:
         return "comp"
 
 
-def discover_auto_modules() -> list[str]:
-    """Find all Python modules in src/ that are auto definitions (not helpers)."""
-    src_dir = _SCRIPT_DIR / "src"
-    modules = []
-    for f in sorted(src_dir.iterdir()):
-        if f.suffix == ".py" and f.stem not in EXCLUDED_MODULES:
-            modules.append(f"src.{f.stem}")
-    return modules
+def generate_autos_json(env: str, bootstrap: bool = False) -> str:
+    """Build the autos via the Java generator and return the Autos.json content for `env`.
+
+    Generator progress and native (HAL) diagnostics stream to stderr; the JSON is
+    returned in-memory.
+    """
+    try:
+        return _autogen.generate(env, bootstrap)
+    except subprocess.CalledProcessError as e:
+        raise SystemExit(f"Java auto generator failed with status {e.returncode}")
 
 
 def _is_json_number(value: Any) -> bool:
@@ -173,10 +169,10 @@ def publish(base_url: str, env: str, content: str) -> None:
     req = urllib.request.Request(url, method="POST")
     try:
         with urllib.request.urlopen(req, timeout=5) as resp:
-            body = resp.read().decode("utf-8", errors="replace")
+            # Drain the response but don't echo it; on success the body is just the
+            # full autos JSON, which is only noise. Failures are reported below.
+            resp.read()
             print(f"  POST {url} -> {resp.status}")
-            if body:
-                print(f"    Response: {body}")
     except urllib.error.HTTPError as e:
         body = e.read().decode("utf-8", errors="replace") if e.fp else ""
         message = f"POST {url} failed: HTTP {e.code} {e.reason}"
@@ -206,58 +202,31 @@ def parse_args() -> argparse.Namespace:
     parser.add_argument(
         "--env", type=str, default=None, help="Override environment (default: auto-detect from config.json)"
     )
+    parser.add_argument(
+        "--bootstrap",
+        action="store_true",
+        help="Force the Java generator to recompile robot classes and re-dump the classpath "
+        "(needed after changing robot/builder code or dependencies)",
+    )
     return parser.parse_args()
 
 
 def main() -> None:
     args = parse_args()
     env = args.env if args.env else detect_environment()
 
-    # Step 1: Import routines first so @routine decorators register before
-    #         any auto modules try to reference them via routines.<name>()
-    print("  Importing src.routines...")
-    importlib.import_module("src.routines")
-
-    # Step 2: Discover and import auto modules
-    auto_modules = discover_auto_modules()
-
-    if not auto_modules:
-        print("No auto modules found in src/")
-        return
-
-    print(f"Found {len(auto_modules)} auto module(s): {[m.split('.')[-1] for m in auto_modules]}")
-
-    for module_name in auto_modules:
-        print(f"  Importing {module_name}...")
-        importlib.import_module(module_name)
-
-    # Step 3: Serialize
-    autos = auto_lib.get_autos()
-    routine_map = auto_lib.get_routines()
-
-    if not autos and not routine_map:
-        print("Warning: No autos or routines were registered.")
-        return
-
-    autos_obj = {}
-    for name, action in autos.items():
-        autos_obj[name] = auto_lib._serialize_value(action)
-
-    routines_obj = {}
-    for name, action in routine_map.items():
-        routines_obj[name] = auto_lib._serialize_value(action)
-
-    output = {"autos": autos_obj, "routines": routines_obj}
-    content = json.dumps(output, indent=4)
+    # Generate Autos.json by running the Java auto generator (reuses the robot's
+    # own coppercore Gson config, so the output round-trips by construction).
+    content = generate_autos_json(env, bootstrap=args.bootstrap)
 
-    # Step 4: Write local file (unless --no-file)
+    # Write local file (unless --no-file)
     if not args.no_file:
         output_file = OUTPUT_DIR / env / "Autos.json"
         output_file.parent.mkdir(parents=True, exist_ok=True)
         output_file.write_text(content, encoding="utf-8")
-        print(f"Written {len(autos)} auto(s) and {len(routine_map)} routine(s) to: {output_file}")
+        print(f"Written autos to: {output_file}")
 
-    # Step 5: Publish to tuning server(s)
+    # Publish to tuning server(s)
     targets: list[str] = []
     if args.sim:
         targets.append(SIM_URL)

auto_generator/dump-classpath.gradle

@@ -0,0 +1,15 @@
+// Init script: dump the main runtime classpath so the Java auto generator can
+// compile and run without paying gradle's per-invocation configuration cost.
+// Used by generate.sh during the one-time (or after-dependency-change) bootstrap.
+gradle.projectsEvaluated {
+    rootProject.tasks.register("dumpAutoGenClasspath") {
+        doLast {
+            def cp = rootProject.sourceSets.main.runtimeClasspath.files
+                    .collect { it.absolutePath }
+                    .join(File.pathSeparator)
+            def out = new File(rootProject.projectDir, "build/autogen-classpath.txt")
+            out.text = cp
+            println "wrote ${out} (${cp.split(File.pathSeparator).length} entries)"
+        }
+    }
+}

auto_generator/generate.py

@@ -0,0 +1,182 @@
+#!/usr/bin/env python3
+"""
+Cross-platform driver for the Java auto generator.
+
+Compiles the auto-definition sources (src/main/java/frc/robot/autogen) against the
+already-compiled robot classes and runs them as a (headless) robot JVM with the WPILib
+native libraries on the path, so the auto code has full, unrestricted access to
+the rest of the robot code (HAL, Filesystem, NetworkTables), exactly as on the
+robot. Emits Autos.json to stdout.
+
+This deliberately bypasses gradle on the hot path. A warm gradle daemon still
+spends ~4s configuring an up-to-date compileJava task on this project; a bare
+javac + java cycle is well under that. Gradle is only invoked for the one-time
+bootstrap (compile robot classes, extract native libs, dump the classpath), or
+when --bootstrap is passed (e.g. after editing robot/builder code or changing
+dependencies).
+
+Usage:
+    python generate.py [ENVIRONMENT]              # print Autos.json (default env: comp)
+    python generate.py --bootstrap [ENVIRONMENT]  # force recompile + native extraction + dump
+
+All diagnostics (and native HAL stdout chatter) go to stderr so stdout is clean JSON.
+"""
+from __future__ import annotations
+
+import os
+import platform
+import subprocess
+import sys
+from pathlib import Path
+
+_THIS_DIR = Path(__file__).resolve().parent  # auto_generator/
+_ROOT = _THIS_DIR.parent
+
+CP_FILE = _ROOT / "build" / "autogen-classpath.txt"
+CLASSES = _ROOT / "build" / "classes" / "java" / "main"
+NATIVE_DIR = _ROOT / "build" / "jni" / "release"
+# Only the autogen package is fast-compiled on the hot path (the rest of the robot
+# is already compiled into CLASSES by the bootstrap's gradle compileJava).
+GEN_SRC = _ROOT / "src" / "main" / "java" / "frc" / "robot" / "autogen"
+GEN_OUT = _ROOT / "build" / "autogen-out"
+OUT_JSON = GEN_OUT / "Autos.json"
+MARKER = GEN_OUT / ".compiled"
+INIT_SCRIPT = _THIS_DIR / "dump-classpath.gradle"
+MAIN_CLASS = "frc.robot.autogen.GenerateAutos"
+
+_IS_WINDOWS = os.name == "nt"
+JVM_FAST = ["-XX:+UseSerialGC", "-XX:TieredStopAtLevel=1", "-XX:-UsePerfData"]
+
+
+def _log(message: str) -> None:
+    print(f"[generate] {message}", file=sys.stderr, flush=True)
+
+
+def _gradlew() -> str:
+    return str(_ROOT / ("gradlew.bat" if _IS_WINDOWS else "gradlew"))
+
+
+def _jdk_tool(name: str) -> str:
+    """Resolve a JDK tool (java/javac), preferring JAVA_HOME (set by the WPILib env)."""
+    java_home = os.environ.get("JAVA_HOME")
+    if java_home:
+        exe = name + (".exe" if _IS_WINDOWS else "")
+        candidate = Path(java_home) / "bin" / exe
+        if candidate.exists():
+            return str(candidate)
+    return name  # fall back to PATH
+
+
+def _java_sources() -> list[str]:
+    return [str(p) for p in sorted(GEN_SRC.rglob("*.java"))]
+
+
+def _sources_changed() -> bool:
+    if not MARKER.exists():
+        return True
+    marker_mtime = MARKER.stat().st_mtime
+    return any(Path(src).stat().st_mtime > marker_mtime for src in _java_sources())
+
+
+def _bootstrap() -> None:
+    _log("bootstrapping: gradle compileJava + native extraction + classpath dump...")
+    subprocess.run(
+        [
+            _gradlew(),
+            "--init-script",
+            str(INIT_SCRIPT),
+            "compileJava",
+            "extractReleaseNative",
+            "dumpAutoGenClasspath",
+            "-q",
+        ],
+        cwd=str(_ROOT),
+        check=True,
+        stdout=sys.stderr,  # keep gradle chatter out of stdout (which carries the JSON)
+    )
+
+
+def _native_loader_env() -> dict[str, str]:
+    """Environment with the native lib dir prepended to the OS's dynamic-loader path."""
+    env = dict(os.environ)
+    var = {
+        "Windows": "PATH",
+        "Darwin": "DYLD_LIBRARY_PATH",
+    }.get(platform.system(), "LD_LIBRARY_PATH")
+    existing = env.get(var, "")
+    env[var] = str(NATIVE_DIR) + (os.pathsep + existing if existing else "")
+    return env
+
+
+def generate(env: str = "comp", bootstrap: bool = False) -> str:
+    """Build the autos and return the Autos.json content for `env`."""
+    if bootstrap or not CP_FILE.exists() or not CLASSES.is_dir() or not NATIVE_DIR.is_dir():
+        _bootstrap()
+
+    classpath = CP_FILE.read_text(encoding="utf-8").strip() + os.pathsep + str(CLASSES)
+    GEN_OUT.mkdir(parents=True, exist_ok=True)
+
+    if _sources_changed():
+        _log("compiling generator sources...")
+        subprocess.run(
+            [
+                _jdk_tool("javac"),
+                "-J-XX:+UseSerialGC",
+                "-J-XX:TieredStopAtLevel=1",
+                "-J-XX:-UsePerfData",
+                "-cp",
+                classpath,
+                "-d",
+                str(GEN_OUT),
+                *_java_sources(),
+            ],
+            cwd=str(_ROOT),
+            check=True,
+        )
+        MARKER.touch()
+    else:
+        _log("generator sources unchanged; skipping compile")
+
+    # The generator selects the environment from the deploy constants/config.json (the same way
+    # the robot does); `env` here is only what the caller intends to write to, for logging.
+    _log(f"running generator (requested env '{env}'; environment comes from config.json)...")
+    # GEN_OUT goes first so the freshly fast-compiled autogen classes take precedence
+    # over the (possibly stale) copies the gradle bootstrap compiled into CLASSES.
+    run_classpath = str(GEN_OUT) + os.pathsep + classpath
+    # The generator writes JSON to OUT_JSON. Initializing HAL prints native chatter to
+    # the JVM's stdout, so we route that to our stderr to keep it out of the JSON.
+    subprocess.run(
+        [
+            _jdk_tool("java"),
+            *JVM_FAST,
+            f"-Djava.library.path={NATIVE_DIR}",
+            "-cp",
+            run_classpath,
+            MAIN_CLASS,
+            str(OUT_JSON),
+        ],
+        cwd=str(_ROOT),
+        check=True,
+        stdout=sys.stderr,
+        env=_native_loader_env(),
+    )
+    return OUT_JSON.read_text(encoding="utf-8")
+
+
+def main(argv: list[str]) -> int:
+    bootstrap = False
+    args = list(argv)
+    if args and args[0] == "--bootstrap":
+        bootstrap = True
+        args = args[1:]
+    env = args[0] if args else "comp"
+    sys.stdout.write(generate(env, bootstrap))
+    return 0
+
+
+if __name__ == "__main__":
+    try:
+        raise SystemExit(main(sys.argv[1:]))
+    except subprocess.CalledProcessError as exc:
+        print(f"[generate] command failed ({exc.returncode}): {exc.cmd}", file=sys.stderr)
+        raise SystemExit(exc.returncode)