From 9b403e1153485ec2c2dbe10531b36c6fd67955a0 Mon Sep 17 00:00:00 2001 From: simontreanor Date: Mon, 28 Sep 2026 02:42:34 +0100 Subject: [PATCH] cli: pyfun build --native compiles a program with mypyc pyfun build --native -o compiles the program natively (ladders and annotations), runs mypyc on it with --allow-redefinition-new and --local-partial-types, removes the Python source so an import finds the extension, and writes a __main__.py, so the result runs as python . It needs mypy and a C compiler in the interpreter's environment and says so when mypyc fails. Single file for now. bench/mypyc_run.py now builds its mypyc variant with this command, so the native-bench job exercises it. --- DESIGN.md | 6 +- ROADMAP.md | 4 +- bench/mypyc_run.py | 26 +++------ src/main.rs | 6 ++ src/native_build.rs | 135 ++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 154 insertions(+), 23 deletions(-) create mode 100644 src/native_build.rs diff --git a/DESIGN.md b/DESIGN.md index c8eb097..0bd03af 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -438,8 +438,10 @@ emitted after its classes); a type variable, a function type or an async result so the annotations never claim more than the checker proved. The output is written for mypy's newer redefinition rule (`--allow-redefinition-new --local-partial-types`): a Python local reused across match arms can hold differently narrowed types in different arms, which the default -first-assignment rule rejects. With both, `bench/mypyc_run.py` compiles every benchmark's native -output with mypyc unchanged: 1.3x to 13x faster than the same file on CPython, and on the +first-assignment rule rejects. `pyfun build --native -o ` does the whole trip: it +compiles natively, runs mypyc with those flags, leaves the extension and a `__main__.py` in `` +(so the program runs as `python `), and needs `mypy` plus a C compiler in the environment +(single file for now). With both, `bench/mypyc_run.py` builds every benchmark that way: 1.3x to 13x faster than the same file on CPython, and on the arithmetic-heavy one about 12x faster than hand-written Python. Native mode is opt-in and applies to every module of a project alike; the default emitter keeps `match`/`case` because it reads as the program was written. Every example and the whole end-to-end suite produce identical output in both modes. diff --git a/ROADMAP.md b/ROADMAP.md index 6ef2dfb..8b72f57 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -514,8 +514,8 @@ map_build 1.64× vs hand-written. union alias per sum type; mypyc run with `--allow-redefinition-new --local-partial-types`), and the same job measured mypyc at 1.56x / **13.2x** / 1.32x faster than the native file on CPython, landing at **1.40x / 0.08x / 1.34x of hand-written Python**: collatz, all int arithmetic, runs - about 12x faster than the hand-written baseline. Remaining, in order: a `--native` build step that runs mypyc - (`pyfun build`), typed emit for projects and for block-local functions, or-patterns that bind + about 12x faster than the hand-written baseline. `pyfun build --native -o ` (compile, mypyc, runnable + directory) followed too. Remaining, in order: typed emit and `build` for projects and for block-local functions, or-patterns that bind names, then the closure/`seq`/`_pyfun_rt` audit (project mode followed the same night) — the checker knows every binding's inferred type, so the emitter could produce fully annotated Python whose annotations cannot lie, then compile it with mypyc into a C extension — native speed diff --git a/bench/mypyc_run.py b/bench/mypyc_run.py index 9c65c31..d512287 100644 --- a/bench/mypyc_run.py +++ b/bench/mypyc_run.py @@ -4,7 +4,7 @@ hand-written baseline, checking all of them print the same output: native `pyfun compile --native`, run by CPython as plain Python - mypyc the same file compiled to a C extension by mypyc + mypyc `pyfun build --native`: the same program as a mypyc C extension baseline the hand-written Python in bench/_baseline.py The emitted program runs its work at import time, so each version is timed as @@ -71,32 +71,20 @@ def main(): [compiler, "compile", "--native", str(BENCH / f"{name}.pyfun"), "-o", str(native)], check=True, ) - compiled = work / f"{name}_mypyc.py" - shutil.copy(native, compiled) - shutil.copy(BENCH / f"{name}_baseline.py", work / f"{name}_base.py") - # Pyfun reuses a Python local across match arms, so a name can hold - # differently narrowed types in different arms; mypy's newer - # redefinition rule accepts that (`DESIGN.md` §5.6). + # The mypyc build is `pyfun build --native`, the command users run. + built = work / "built" build = subprocess.run( - [ - sys.executable, - "-m", - "mypyc", - "--allow-redefinition-new", - "--local-partial-types", - compiled.name, - ], - cwd=work, + [compiler, "build", "--native", str(BENCH / f"{name}.pyfun"), "-o", str(built)], capture_output=True, text=True, ) if build.returncode != 0: - print(f"{name:<11} mypyc failed:\n{build.stdout[-2000:]}{build.stderr[-2000:]}") + print(f"{name:<11} pyfun build failed:\n{build.stdout[-2000:]}{build.stderr[-2000:]}") continue - compiled.unlink() # so the import finds the extension, not the source + shutil.copy(BENCH / f"{name}_baseline.py", work / f"{name}_base.py") try: t_native, o_native = time_import(work, f"{name}_native", args.runs) - t_mypyc, o_mypyc = time_import(work, f"{name}_mypyc", args.runs) + t_mypyc, o_mypyc = time_import(built, name, args.runs) t_base, o_base = time_import(work, f"{name}_base", args.runs) except RuntimeError as err: print(f"{name:<11} {err}") diff --git a/src/main.rs b/src/main.rs index e4531fb..2676282 100644 --- a/src/main.rs +++ b/src/main.rs @@ -26,6 +26,7 @@ use pyfun::syntax::{Item, Module}; mod bundle; mod kernel; +mod native_build; mod packages; mod repl; @@ -70,6 +71,7 @@ fn main() -> ExitCode { Ok(parsed) => bundle::run(parsed), Err(msg) => fail(&msg), }, + Some("build") => native_build::run(&args[1..]), Some("add") => packages::add(&args[1..]), Some("install") => packages::install(), Some("remove") => packages::remove(&args[1..]), @@ -101,6 +103,10 @@ fn help() { ); eprintln!(" ladders, the form mypyc compiles"); eprintln!(" pyfun run [--] [args...] compile then execute with Python"); + eprintln!( + " pyfun build --native -o compile to a C extension with mypyc;" + ); + eprintln!(" run the result with `python `"); eprintln!( " (args after the path go to the program's sys.argv)" ); diff --git a/src/native_build.rs b/src/native_build.rs new file mode 100644 index 0000000..2fde307 --- /dev/null +++ b/src/native_build.rs @@ -0,0 +1,135 @@ +//! `pyfun build --native -o `: compile a program to a native +//! extension with mypyc (`DESIGN.md` §5.6). +//! +//! The program is compiled with `--native` (every match an `isinstance` ladder, +//! top-level functions annotated from their inferred types) into ``, mypyc +//! builds that module into a C extension beside it, the Python source is removed +//! so an import finds the extension, and a `__main__.py` imports it, so the +//! result runs as `python `. Needs `mypy` (which ships mypyc) and a C +//! compiler in the interpreter's environment; on Windows that means the MSVC +//! Build Tools. + +use std::path::Path; +use std::process::{Command, ExitCode}; + +use pyfun::python_emitter::PyTarget; + +/// Parse `build` arguments: `--native` (required for now, since native is the +/// only build there is), a path, and `-o `. +pub fn run(args: &[String]) -> ExitCode { + let mut native = false; + let mut path = None; + let mut out = None; + let mut i = 0; + while i < args.len() { + match args[i].as_str() { + "--native" => native = true, + "-o" | "--output" => { + i += 1; + out = args.get(i).cloned(); + } + p if path.is_none() => path = Some(p.to_string()), + other => return fail(&format!("unexpected argument `{other}`")), + } + i += 1; + } + if !native { + return fail( + "`build` compiles with mypyc and needs `--native` (pyfun build --native -o )", + ); + } + let Some(path) = path else { + return fail("`build` needs a file path"); + }; + let Some(out) = out else { + return fail("`build` needs an output directory (-o )"); + }; + build(Path::new(&path), Path::new(&out)) +} + +fn build(path: &Path, out: &Path) -> ExitCode { + let source = match std::fs::read_to_string(path) { + Ok(s) => s, + Err(e) => return fail(&format!("cannot read {}: {e}", path.display())), + }; + if let Ok(module) = pyfun::parse(&source) + && crate::has_imports(&module) + { + return fail("`build --native` supports a single file so far, not a project"); + } + let python = match pyfun::compile_with(&source, PyTarget::default(), true) { + Ok((py, notes)) => { + crate::report_notes(¬es); + py + } + Err(e) => { + eprintln!( + "{}", + pyfun::diagnostics::render( + &source, + pyfun::diagnostics::Level::Error, + &e.message(), + e.span() + ) + ); + return ExitCode::FAILURE; + } + }; + let Some(module) = pyfun::project::module_name_from_path(path).map(|m| m.to_lowercase()) else { + return fail("the file needs a name"); + }; + if let Err(e) = std::fs::create_dir_all(out) { + return fail(&format!("cannot create {}: {e}", out.display())); + } + let source_file = out.join(format!("{module}.py")); + if let Err(e) = std::fs::write(&source_file, &python) { + return fail(&format!("cannot write {}: {e}", source_file.display())); + } + let Some(interpreter) = crate::python_cmd() else { + return fail("no Python interpreter found (set PYFUN_PYTHON)"); + }; + // Pyfun reuses a Python local across match arms, so a name can hold + // differently narrowed types in different arms: mypy's newer redefinition + // rule accepts that where the default first-assignment rule does not. + let status = Command::new(&interpreter) + .args([ + "-m", + "mypyc", + "--allow-redefinition-new", + "--local-partial-types", + &format!("{module}.py"), + ]) + .current_dir(out) + .status(); + match status { + Ok(s) if s.success() => {} + Ok(_) => { + return fail( + "mypyc could not build the program (see its output above); \ + `pip install mypy` provides it, and it needs a C compiler", + ); + } + Err(e) => return fail(&format!("cannot run {interpreter}: {e}")), + } + // The extension now sits beside the source; remove the source so an import + // finds the extension, and give the directory an entry point. + let _ = std::fs::remove_file(&source_file); + let _ = std::fs::remove_dir_all(out.join("build")); + let main = format!( + "# Runs the mypyc-compiled Pyfun program `{module}`.\nimport {module} # noqa: F401\n" + ); + if let Err(e) = std::fs::write(out.join("__main__.py"), main) { + return fail(&format!("cannot write __main__.py: {e}")); + } + println!( + "built {} natively: run it with `python {}`", + path.display(), + out.display() + ); + ExitCode::SUCCESS +} + +fn fail(msg: &str) -> ExitCode { + eprintln!("error: {msg}"); + ExitCode::FAILURE +}