diff --git a/.github/workflows/native-bench.yml b/.github/workflows/native-bench.yml index e5522e0..e98f90f 100644 --- a/.github/workflows/native-bench.yml +++ b/.github/workflows/native-bench.yml @@ -30,3 +30,11 @@ jobs: - run: python bench/mypyc_run.py --runs 5 env: PYFUN_BIN: target/release/pyfun + # A multi-module project through the same build, checked against `pyfun run`. + - name: Build a project natively + run: | + target/release/pyfun build --native examples/modules/main.pyfun -o /tmp/modules-native + target/release/pyfun run examples/modules/main.pyfun > /tmp/plain.txt + python /tmp/modules-native > /tmp/native.txt + diff /tmp/plain.txt /tmp/native.txt + ls /tmp/modules-native diff --git a/DESIGN.md b/DESIGN.md index 0bd03af..a1e0156 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -440,8 +440,9 @@ redefinition rule (`--allow-redefinition-new --local-partial-types`): a Python l match arms can hold differently narrowed types in different arms, which the default first-assignment rule rejects. `pyfun build --native -o ` does the whole trip: it compiles natively, runs mypyc with those flags, leaves the extension and a `__main__.py` in `` -(so the program runs as `python `), and needs `mypy` plus a C compiler in the environment -(single file for now). With both, `bench/mypyc_run.py` builds every benchmark that way: 1.3x to 13x faster than the same file on CPython, and on the +(so the program runs as `python `), and needs `mypy` plus a C compiler in the environment. A +project builds the same way: every module is annotated from its own types and compiled by mypyc, +and the shared `_pyfun_rt.py` stays Python, imported by the compiled modules like any other. With both, `bench/mypyc_run.py` builds every benchmark that way: 1.3x to 13x faster than the same file on CPython, and on the arithmetic-heavy one about 12x faster than hand-written Python. Native mode is opt-in and applies to every module of a project alike; the default emitter keeps `match`/`case` because it reads as the program was written. Every example and the whole end-to-end suite produce identical output in both modes. diff --git a/README.md b/README.md index 6c42185..2babf86 100644 --- a/README.md +++ b/README.md @@ -468,7 +468,7 @@ On the repository's benchmarks (CPython 3.12, `bench/mypyc_run.py`), the mypyc b 13x faster than the same program as plain Python, and the arithmetic-heavy one runs about 12x faster than the hand-written Python it's measured against. `pyfun compile --native` alone, without mypyc, is already faster on match-heavy code, because CPython skips its pattern-matching -machinery. Native builds are single-file for now; [`DESIGN.md` §5.6](https://github.com/simontreanor/Pyfun/blob/main/DESIGN.md) +machinery. A multi-module project builds the same way. [`DESIGN.md` §5.6](https://github.com/simontreanor/Pyfun/blob/main/DESIGN.md) has the details. --- diff --git a/ROADMAP.md b/ROADMAP.md index 8b72f57..ccaa35a 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -515,7 +515,8 @@ map_build 1.64× vs hand-written. the same job measured mypyc at 1.56x / **13.2x** / 1.32x faster than the native file on CPython, landing at **1.40x / 0.08x / 1.34x of hand-written Python**: collatz, all int arithmetic, runs about 12x faster than the hand-written baseline. `pyfun build --native -o ` (compile, mypyc, runnable - directory) followed too. Remaining, in order: typed emit and `build` for projects and for block-local functions, or-patterns that bind + directory) followed too. Typed emit and `build` for projects followed too. Remaining, in + order: typed emit for block-local functions and for block-local functions, or-patterns that bind names, then the closure/`seq`/`_pyfun_rt` audit (project mode followed the same night) — the checker knows every binding's inferred type, so the emitter could produce fully annotated Python whose annotations cannot lie, then compile it with mypyc into a C extension — native speed diff --git a/src/lowering/mod.rs b/src/lowering/mod.rs index 383025a..6d62de3 100644 --- a/src/lowering/mod.rs +++ b/src/lowering/mod.rs @@ -204,9 +204,11 @@ pub fn lower_in_project( float_literals: &HashSet, codecs: &crate::types::Codecs, native: bool, + binding_types: HashMap, ) -> Result { let mut lowerer = Lowerer::new(module); lowerer.native = native; + lowerer.binding_types = binding_types; lowerer.float_literals = float_literals.clone(); lowerer.codecs = codecs.clone(); lowerer.imported_modules = ctx.modules.clone(); @@ -809,12 +811,15 @@ impl Lowerer { // The Python spelling of each user type: a record is its class, a sum // type an alias over its constructors' classes (skipped when a // constructor shares the type's name, which the alias would shadow). + // Classes this module defines, plus the ones a project imports from the + // shared runtime (`Some`, `None_`, `Ok`, `Error`). let class_names: HashSet = py .body .iter() - .filter_map(|s| match s { - PyStmt::ClassDef { name, .. } => Some(name.clone()), - _ => None, + .flat_map(|s| match s { + PyStmt::ClassDef { name, .. } => vec![name.clone()], + PyStmt::ImportFrom { module, names } if module == "_pyfun_rt" => names.clone(), + _ => vec![], }) .collect(); let mut spell: HashMap = HashMap::new(); diff --git a/src/native_build.rs b/src/native_build.rs index 2fde307..bfbeb2a 100644 --- a/src/native_build.rs +++ b/src/native_build.rs @@ -55,7 +55,7 @@ fn build(path: &Path, out: &Path) -> ExitCode { if let Ok(module) = pyfun::parse(&source) && crate::has_imports(&module) { - return fail("`build --native` supports a single file so far, not a project"); + return build_project(path, out); } let python = match pyfun::compile_with(&source, PyTarget::default(), true) { Ok((py, notes)) => { @@ -85,20 +85,64 @@ fn build(path: &Path, out: &Path) -> ExitCode { if let Err(e) = std::fs::write(&source_file, &python) { return fail(&format!("cannot write {}: {e}", source_file.display())); } + finish(path, out, &module, std::slice::from_ref(&module)) +} + +/// A project: every module compiled natively into `out`, and every one but the +/// shared `_pyfun_rt.py` built by mypyc (the runtime stays Python, which the +/// compiled modules import like any other module). +fn build_project(path: &Path, out: &Path) -> ExitCode { + let entry = path.to_string_lossy().to_string(); + let project = match crate::resolve_project(&entry) { + Ok(p) => p, + Err(code) => return code, + }; + if !crate::check_project_ok(&project) { + return ExitCode::FAILURE; + } + let files = match crate::lower_project(&project, PyTarget::default(), true) { + Ok(f) => f, + Err(code) => return code, + }; + if let Err(e) = std::fs::create_dir_all(out) { + return fail(&format!("cannot create {}: {e}", out.display())); + } + let mut modules = Vec::new(); + for (name, source) in &files { + if let Err(e) = std::fs::write(out.join(name), source) { + return fail(&format!("cannot write {name}: {e}")); + } + if let Some(stem) = name.strip_suffix(".py") + && stem != "_pyfun_rt" + { + modules.push(stem.to_string()); + } + } + let Some(entry_module) = pyfun::project::module_name_from_path(path).map(|m| m.to_lowercase()) + else { + return fail("the file needs a name"); + }; + finish(path, out, &entry_module, &modules) +} + +/// Run mypyc over `modules` in `out`, drop their sources so imports find the +/// extensions, and write the `__main__.py` that imports `entry`. +fn finish(path: &Path, out: &Path, entry: &str, modules: &[String]) -> ExitCode { let Some(interpreter) = crate::python_cmd() else { return fail("no Python interpreter found (set PYFUN_PYTHON)"); }; // Pyfun reuses a Python local across match arms, so a name can hold // differently narrowed types in different arms: mypy's newer redefinition // rule accepts that where the default first-assignment rule does not. + let mut args = vec![ + "-m".to_string(), + "mypyc".to_string(), + "--allow-redefinition-new".to_string(), + "--local-partial-types".to_string(), + ]; + args.extend(modules.iter().map(|m| format!("{m}.py"))); let status = Command::new(&interpreter) - .args([ - "-m", - "mypyc", - "--allow-redefinition-new", - "--local-partial-types", - &format!("{module}.py"), - ]) + .args(&args) .current_dir(out) .status(); match status { @@ -113,10 +157,12 @@ fn build(path: &Path, out: &Path) -> ExitCode { } // The extension now sits beside the source; remove the source so an import // finds the extension, and give the directory an entry point. - let _ = std::fs::remove_file(&source_file); + for m in modules { + let _ = std::fs::remove_file(out.join(format!("{m}.py"))); + } let _ = std::fs::remove_dir_all(out.join("build")); let main = format!( - "# Runs the mypyc-compiled Pyfun program `{module}`.\nimport {module} # noqa: F401\n" + "# Runs the mypyc-compiled Pyfun program `{entry}`.\nimport {entry} # noqa: F401\n" ); if let Err(e) = std::fs::write(out.join("__main__.py"), main) { return fail(&format!("cannot write __main__.py: {e}")); diff --git a/src/project/mod.rs b/src/project/mod.rs index c29c7f2..3282d0c 100644 --- a/src/project/mod.rs +++ b/src/project/mod.rs @@ -295,6 +295,9 @@ pub fn compile_with( // (`DESIGN.md` §6.1), which the lowering context below needs. let mut exports: HashMap = HashMap::new(); let mut codecs_by_module: HashMap = HashMap::new(); + // Native mode annotates functions from their types (`DESIGN.md` §5.6). + let mut types_by_module: HashMap> = + HashMap::new(); let float_spans: HashMap> = { let mut spans = HashMap::new(); for module in &project.modules { @@ -306,6 +309,12 @@ pub fn compile_with( let (_errors, types, module_exports, codecs) = crate::types::check_module_collecting(&module.ast, &imports); spans.insert(module.name.clone(), crate::float_literal_spans(&types)); + if native { + types_by_module.insert( + module.name.clone(), + types.iter().map(|t| (t.span, t.raw.clone())).collect(), + ); + } codecs_by_module.insert(module.name.clone(), codecs); exports.insert(module.name.clone(), module_exports); } @@ -374,7 +383,9 @@ pub fn compile_with( } let floats = float_spans.get(&module.name).unwrap_or(&no_floats); let codecs = codecs_by_module.get(&module.name).unwrap_or(&no_codecs); - let lowered = lowering::lower_in_project(&module.ast, &ctx, floats, codecs, native)?; + let binding_types = types_by_module.remove(&module.name).unwrap_or_default(); + let lowered = + lowering::lower_in_project(&module.ast, &ctx, floats, codecs, native, binding_types)?; needs_runtime |= lowered.uses_runtime; notes.extend( lowered