diff --git a/docs/source/basic-usage.md b/docs/source/basic-usage.md index 481205a1d..0012bb558 100644 --- a/docs/source/basic-usage.md +++ b/docs/source/basic-usage.md @@ -108,9 +108,9 @@ instead — `kernels` will then never load an incompatible kernel at all. ## Inspecting Loaded Kernels [`~kernels.get_loaded_kernels`] returns a snapshot of every kernel that has been loaded -into the current process. Each entry is a [`~kernels.LoadedKernel`] namedtuple with the -imported `module`, the `package_name`, and `repo_infos` (repo id, resolved -revision, and the backend argument that was passed). +into the current process. Each entry is a [`~kernels.LoadedKernel`] dataclass with +the kernel's `metadata`, the imported `module`, and optional `repo_info` +(repository ID and resolved revision). ```python from kernels import get_kernel, get_loaded_kernels @@ -118,12 +118,16 @@ from kernels import get_kernel, get_loaded_kernels get_kernel("kernels-community/activation", version=1) for loaded in get_loaded_kernels(): - print(loaded.package_name, loaded.repo_infos) + print(loaded.metadata.name, loaded.repo_info) + for url in loaded.metadata.upstream: + print(url) ``` -`repo_infos` is populated only for kernels loaded with [`~kernels.get_kernel`]. Kernels -loaded from a local path ([`~kernels.get_local_kernel`]) or via a lockfile -([`~kernels.get_locked_kernel`], [`~kernels.load_kernel`]) have `repo_infos=None`. +`repo_info` is populated when the kernel's Hub repository is known; otherwise it +is `None`, as for kernels loaded directly from a local path. +`metadata.upstream` is a list of original upstream repository URLs, empty when +none is specified. See the [migration guide](migration.md#018) for the change +from a single URL to a list in `kernels` 0.18. Browse through different kernels compatible with `kernels` from [here](https://huggingface.co/kernels). diff --git a/docs/source/builder/triton-autotune.md b/docs/source/builder/triton-autotune.md index 04f619989..db7d72a3c 100644 --- a/docs/source/builder/triton-autotune.md +++ b/docs/source/builder/triton-autotune.md @@ -37,7 +37,7 @@ files are picked up from the kenel's Python source directory, so add `json` to t [general] name = "gemm-triton-autotune" version = 1 -edition = 5 +edition = 6 license = "Apache-2.0" backends = ["cuda", "rocm", "xpu"] diff --git a/docs/source/builder/writing-kernels.md b/docs/source/builder/writing-kernels.md index a3e25b3b7..1f577e599 100644 --- a/docs/source/builder/writing-kernels.md +++ b/docs/source/builder/writing-kernels.md @@ -167,7 +167,7 @@ backends = [ ] name = "mykernel" version = 1 -edition = 5 +edition = 6 [general.hub] repo-id = "myorg/mykernel" @@ -199,12 +199,14 @@ The following sections enumerate all supported options for `build.toml`. by the `kernel-builder upload` command to upload the kernel to a version branch named `v`. - `edition` (required): the `build.toml` format edition. The current - edition is `5`. Older `build.toml` files can be migrated with + edition is `6`. Older `build.toml` files can be migrated with `kernel-builder update-build`. - `backends` (required): a list of supported backends. Must be one or more of `cpu`, `cuda`, `metal`, `rocm`, or `xpu`. -- `upstream`: Git-compatible URL (passable to `git clone`) of the original - upstream repository where the kernel source code comes from. +- `upstream`: list of Git-compatible URLs (passable to `git clone`) of the original + repositories where the kernel source code comes from. Editions before 6 use a + single URL string; `kernel-builder update-build` converts it to a list. For example: + `upstream = ["https://github.com/org/repo-a", "https://github.com/org/repo-b"]`. - `source`: Git-compatible URL (passable to `git clone`) of the kernel-builder formatted source repository. This repository must contain a `build.toml` and `flake.nix` so that it can be pulled and built with the kernel builder. diff --git a/docs/source/cli-info.md b/docs/source/cli-info.md index ad8c193b4..c9d59f33e 100644 --- a/docs/source/cli-info.md +++ b/docs/source/cli-info.md @@ -59,6 +59,11 @@ Python dependencies: - Backends: cuda, metal ``` +The `Upstream` line lists every upstream repository, separated by commas, or +`-` if none is specified. JSON output includes an `upstream` array (empty +when unspecified), including when reading legacy single-string metadata. +The field is omitted when metadata is unavailable. + ## See Also - [kernels versions](cli-versions.md) - List available versions of a kernel diff --git a/docs/source/kernel-requirements.md b/docs/source/kernel-requirements.md index ee3074c9c..581c196d9 100644 --- a/docs/source/kernel-requirements.md +++ b/docs/source/kernel-requirements.md @@ -87,8 +87,9 @@ metadata. Currently the following top-level keys are supported: installed. - `license` (`str`, required): the kernel license in. Refer to the list of [supported license identifiers](https://huggingface.co/docs/hub/repositories-licenses). -- `upstream` (`str`, optional): Git-compatible URL (passable to `git clone`) - of the original upstream repository where the kernel source code comes from. +- `upstream` (`str | list[str]`, optional): Git-compatible URLs (passable to + `git clone`) of the original repositories where the kernel source code comes + from. - `source` (`str`, optional): Git-compatible URL (passable to `git clone`) of the kernel-builder formatted source repository (must contain `build.toml` and `flake.nix`). diff --git a/docs/source/migration.md b/docs/source/migration.md index 3eebdd44d..d6a8486e6 100644 --- a/docs/source/migration.md +++ b/docs/source/migration.md @@ -186,3 +186,36 @@ class FastSiluAndMul(nn.Module): ``` For more information, see the [layer documentation](layers.md). + +## 0.18 + +### Upstream repositories are returned as a list + +The `upstream` property of kernel metadata now returns `list[str]` instead of +`str | None`. This is a breaking API change for code that reads +`loaded.metadata.upstream` from the [`~kernels.LoadedKernel`] entries returned by +[`~kernels.get_loaded_kernels`]. + +A kernel with one upstream repository returns `[url]`; one with no upstream +returns `[]`. Iterate over the list to handle kernels with multiple upstreams: + +```python +from kernels import get_loaded_kernels + +# Old: +for loaded in get_loaded_kernels(): + if loaded.metadata.upstream is not None: + print(loaded.metadata.upstream) + +# New: +for loaded in get_loaded_kernels(): + for url in loaded.metadata.upstream: + print(url) +``` + +Replace `upstream is None` checks with `not upstream` to detect an absent +upstream. Code that processes a URL string should now process each URL in the +list. + +Existing metadata files with a single URL string, `null`, or no `upstream` field +still load successfully. diff --git a/examples/kernels/cutlass-gemm-tvm-ffi/CARD.md b/examples/kernels/cutlass-gemm-tvm-ffi/CARD.md index 927ad93fd..17a507f37 100644 --- a/examples/kernels/cutlass-gemm-tvm-ffi/CARD.md +++ b/examples/kernels/cutlass-gemm-tvm-ffi/CARD.md @@ -55,7 +55,11 @@ No benchmark available yet. ## Upstream -The original source code for this kernel comes from {{ upstream }}. +The original source code for this kernel comes from: + +{% for url in upstream %} +- {{ url }} +{% endfor %} {% endif %} {% if source %} diff --git a/examples/kernels/cutlass-gemm/CARD.md b/examples/kernels/cutlass-gemm/CARD.md index 927ad93fd..17a507f37 100644 --- a/examples/kernels/cutlass-gemm/CARD.md +++ b/examples/kernels/cutlass-gemm/CARD.md @@ -55,7 +55,11 @@ No benchmark available yet. ## Upstream -The original source code for this kernel comes from {{ upstream }}. +The original source code for this kernel comes from: + +{% for url in upstream %} +- {{ url }} +{% endfor %} {% endif %} {% if source %} diff --git a/examples/kernels/gemm-triton-autotune/CARD.md b/examples/kernels/gemm-triton-autotune/CARD.md index b970ce415..a86252ce5 100644 --- a/examples/kernels/gemm-triton-autotune/CARD.md +++ b/examples/kernels/gemm-triton-autotune/CARD.md @@ -55,6 +55,10 @@ No benchmark available yet. ## Source code -Source code of this kernel originally comes from {{ upstream }} and it was repurposed for compatibility with `kernels`. +The original source code for this kernel comes from: + +{% for url in upstream %} +- {{ url }} +{% endfor %} {% endif %} diff --git a/examples/kernels/kernel-deps/CARD.md b/examples/kernels/kernel-deps/CARD.md index 927ad93fd..17a507f37 100644 --- a/examples/kernels/kernel-deps/CARD.md +++ b/examples/kernels/kernel-deps/CARD.md @@ -55,7 +55,11 @@ No benchmark available yet. ## Upstream -The original source code for this kernel comes from {{ upstream }}. +The original source code for this kernel comes from: + +{% for url in upstream %} +- {{ url }} +{% endfor %} {% endif %} {% if source %} diff --git a/examples/kernels/relu-archs-subset/CARD.md b/examples/kernels/relu-archs-subset/CARD.md index 2868c79f5..1cc3dd70c 100644 --- a/examples/kernels/relu-archs-subset/CARD.md +++ b/examples/kernels/relu-archs-subset/CARD.md @@ -55,7 +55,11 @@ No benchmark available yet. ## Upstream -The original source code for this kernel comes from {{ upstream }}. +The original source code for this kernel comes from: + +{% for url in upstream %} +- {{ url }} +{% endfor %} {% endif %} {% if source %} diff --git a/examples/kernels/relu-backprop-compile/CARD.md b/examples/kernels/relu-backprop-compile/CARD.md index 927ad93fd..17a507f37 100644 --- a/examples/kernels/relu-backprop-compile/CARD.md +++ b/examples/kernels/relu-backprop-compile/CARD.md @@ -55,7 +55,11 @@ No benchmark available yet. ## Upstream -The original source code for this kernel comes from {{ upstream }}. +The original source code for this kernel comes from: + +{% for url in upstream %} +- {{ url }} +{% endfor %} {% endif %} {% if source %} diff --git a/examples/kernels/relu-compiler-flags/CARD.md b/examples/kernels/relu-compiler-flags/CARD.md index 927ad93fd..17a507f37 100644 --- a/examples/kernels/relu-compiler-flags/CARD.md +++ b/examples/kernels/relu-compiler-flags/CARD.md @@ -55,7 +55,11 @@ No benchmark available yet. ## Upstream -The original source code for this kernel comes from {{ upstream }}. +The original source code for this kernel comes from: + +{% for url in upstream %} +- {{ url }} +{% endfor %} {% endif %} {% if source %} diff --git a/examples/kernels/relu-invalid-capability/CARD.md b/examples/kernels/relu-invalid-capability/CARD.md index b970ce415..a86252ce5 100644 --- a/examples/kernels/relu-invalid-capability/CARD.md +++ b/examples/kernels/relu-invalid-capability/CARD.md @@ -55,6 +55,10 @@ No benchmark available yet. ## Source code -Source code of this kernel originally comes from {{ upstream }} and it was repurposed for compatibility with `kernels`. +The original source code for this kernel comes from: + +{% for url in upstream %} +- {{ url }} +{% endfor %} {% endif %} diff --git a/examples/kernels/relu-metal-cpp/CARD.md b/examples/kernels/relu-metal-cpp/CARD.md index 927ad93fd..17a507f37 100644 --- a/examples/kernels/relu-metal-cpp/CARD.md +++ b/examples/kernels/relu-metal-cpp/CARD.md @@ -55,7 +55,11 @@ No benchmark available yet. ## Upstream -The original source code for this kernel comes from {{ upstream }}. +The original source code for this kernel comes from: + +{% for url in upstream %} +- {{ url }} +{% endfor %} {% endif %} {% if source %} diff --git a/examples/kernels/relu-nki/CARD.md b/examples/kernels/relu-nki/CARD.md index 927ad93fd..17a507f37 100644 --- a/examples/kernels/relu-nki/CARD.md +++ b/examples/kernels/relu-nki/CARD.md @@ -55,7 +55,11 @@ No benchmark available yet. ## Upstream -The original source code for this kernel comes from {{ upstream }}. +The original source code for this kernel comes from: + +{% for url in upstream %} +- {{ url }} +{% endfor %} {% endif %} {% if source %} diff --git a/examples/kernels/relu-specific-torch/CARD.md b/examples/kernels/relu-specific-torch/CARD.md index 927ad93fd..17a507f37 100644 --- a/examples/kernels/relu-specific-torch/CARD.md +++ b/examples/kernels/relu-specific-torch/CARD.md @@ -55,7 +55,11 @@ No benchmark available yet. ## Upstream -The original source code for this kernel comes from {{ upstream }}. +The original source code for this kernel comes from: + +{% for url in upstream %} +- {{ url }} +{% endfor %} {% endif %} {% if source %} diff --git a/examples/kernels/relu-torch-bounds/CARD.md b/examples/kernels/relu-torch-bounds/CARD.md index 927ad93fd..17a507f37 100644 --- a/examples/kernels/relu-torch-bounds/CARD.md +++ b/examples/kernels/relu-torch-bounds/CARD.md @@ -55,7 +55,11 @@ No benchmark available yet. ## Upstream -The original source code for this kernel comes from {{ upstream }}. +The original source code for this kernel comes from: + +{% for url in upstream %} +- {{ url }} +{% endfor %} {% endif %} {% if source %} diff --git a/examples/kernels/relu-torch-stable-abi/CARD.md b/examples/kernels/relu-torch-stable-abi/CARD.md index b970ce415..a86252ce5 100644 --- a/examples/kernels/relu-torch-stable-abi/CARD.md +++ b/examples/kernels/relu-torch-stable-abi/CARD.md @@ -55,6 +55,10 @@ No benchmark available yet. ## Source code -Source code of this kernel originally comes from {{ upstream }} and it was repurposed for compatibility with `kernels`. +The original source code for this kernel comes from: + +{% for url in upstream %} +- {{ url }} +{% endfor %} {% endif %} diff --git a/examples/kernels/relu-tpu/CARD.md b/examples/kernels/relu-tpu/CARD.md index 92ecc33da..3c835cb07 100644 --- a/examples/kernels/relu-tpu/CARD.md +++ b/examples/kernels/relu-tpu/CARD.md @@ -51,7 +51,11 @@ No benchmark available yet. ## Upstream -The original source code for this kernel comes from {{ upstream }}. +The original source code for this kernel comes from: + +{% for url in upstream %} +- {{ url }} +{% endfor %} {% endif %} {% if source %} diff --git a/examples/kernels/relu-triton/CARD.md b/examples/kernels/relu-triton/CARD.md index b970ce415..a86252ce5 100644 --- a/examples/kernels/relu-triton/CARD.md +++ b/examples/kernels/relu-triton/CARD.md @@ -55,6 +55,10 @@ No benchmark available yet. ## Source code -Source code of this kernel originally comes from {{ upstream }} and it was repurposed for compatibility with `kernels`. +The original source code for this kernel comes from: + +{% for url in upstream %} +- {{ url }} +{% endfor %} {% endif %} diff --git a/examples/kernels/relu-tvm-ffi-compiler-flags/CARD.md b/examples/kernels/relu-tvm-ffi-compiler-flags/CARD.md index 927ad93fd..17a507f37 100644 --- a/examples/kernels/relu-tvm-ffi-compiler-flags/CARD.md +++ b/examples/kernels/relu-tvm-ffi-compiler-flags/CARD.md @@ -55,7 +55,11 @@ No benchmark available yet. ## Upstream -The original source code for this kernel comes from {{ upstream }}. +The original source code for this kernel comes from: + +{% for url in upstream %} +- {{ url }} +{% endfor %} {% endif %} {% if source %} diff --git a/examples/kernels/relu-tvm-ffi/CARD.md b/examples/kernels/relu-tvm-ffi/CARD.md index 927ad93fd..17a507f37 100644 --- a/examples/kernels/relu-tvm-ffi/CARD.md +++ b/examples/kernels/relu-tvm-ffi/CARD.md @@ -55,7 +55,11 @@ No benchmark available yet. ## Upstream -The original source code for this kernel comes from {{ upstream }}. +The original source code for this kernel comes from: + +{% for url in upstream %} +- {{ url }} +{% endfor %} {% endif %} {% if source %} diff --git a/examples/kernels/relu/CARD.md b/examples/kernels/relu/CARD.md index 2868c79f5..1cc3dd70c 100644 --- a/examples/kernels/relu/CARD.md +++ b/examples/kernels/relu/CARD.md @@ -55,7 +55,11 @@ No benchmark available yet. ## Upstream -The original source code for this kernel comes from {{ upstream }}. +The original source code for this kernel comes from: + +{% for url in upstream %} +- {{ url }} +{% endfor %} {% endif %} {% if source %} diff --git a/examples/kernels/silu-and-mul-bad-registration/CARD.md b/examples/kernels/silu-and-mul-bad-registration/CARD.md index 927ad93fd..17a507f37 100644 --- a/examples/kernels/silu-and-mul-bad-registration/CARD.md +++ b/examples/kernels/silu-and-mul-bad-registration/CARD.md @@ -55,7 +55,11 @@ No benchmark available yet. ## Upstream -The original source code for this kernel comes from {{ upstream }}. +The original source code for this kernel comes from: + +{% for url in upstream %} +- {{ url }} +{% endfor %} {% endif %} {% if source %} diff --git a/examples/kernels/silu-and-mul/CARD.md b/examples/kernels/silu-and-mul/CARD.md index 927ad93fd..17a507f37 100644 --- a/examples/kernels/silu-and-mul/CARD.md +++ b/examples/kernels/silu-and-mul/CARD.md @@ -55,7 +55,11 @@ No benchmark available yet. ## Upstream -The original source code for this kernel comes from {{ upstream }}. +The original source code for this kernel comes from: + +{% for url in upstream %} +- {{ url }} +{% endfor %} {% endif %} {% if source %} diff --git a/kernel-builder/src/card.rs b/kernel-builder/src/card.rs index 284d81ce8..e8d9d7bac 100644 --- a/kernel-builder/src/card.rs +++ b/kernel-builder/src/card.rs @@ -142,7 +142,7 @@ fn render_card(build: &Build, kernel_dir: &Path) -> Result { functions => functions, layers => layers, has_benchmark => has_benchmark, - upstream => build.general.upstream.as_ref().map(|u| u.as_url().to_string()), + upstream => build.general.upstream.iter().map(|u| u.as_url().to_string()).collect::>(), source => build.general.source.as_ref().map(|u| u.as_url().to_string()), license => build.general.license.to_lowercase(), }) @@ -174,6 +174,55 @@ pub fn fill_card(kernel_dir: Option, output: Option) -> Result mod tests { use super::*; + #[test] + fn test_render_card_upstreams() { + let temp = tempfile::tempdir().unwrap(); + fs::write( + temp.path().join("CARD.md"), + Environment::new() + .render_str(include_str!("init/templates/CARD.md"), context! {}) + .unwrap(), + ) + .unwrap(); + let urls = [ + "https://github.com/ronghanghu/torch_generic_nms", + "https://github.com/ronghanghu/cc_torch", + ]; + for upstream in [ + "".to_owned(), + format!("upstream = [{:?}]", urls[0]), + format!("upstream = {urls:?}"), + ] { + fs::write( + temp.path().join("build.toml"), + format!( + r#" +[general] +name = "cv-utils" +version = 1 +edition = 6 +license = "MIT" +backends = ["cpu"] +{upstream} +[general.hub] +repo-id = "kernels-community/cv-utils" +[torch-noarch] +"# + ), + ) + .unwrap(); + let build = Build::open(temp.path()).unwrap(); + let card = render_card(&build, temp.path()).unwrap(); + assert_eq!( + card.contains("## Upstream"), + !build.general.upstream.is_empty() + ); + for url in &build.general.upstream { + assert!(card.contains(&format!("- {url}\n")), "{card}"); + } + } + } + #[test] fn test_extract_functions() { let temp_dir = tempfile::tempdir().unwrap(); diff --git a/kernel-builder/src/init/templates/CARD.md b/kernel-builder/src/init/templates/CARD.md index 0bb300c38..f019433af 100644 --- a/kernel-builder/src/init/templates/CARD.md +++ b/kernel-builder/src/init/templates/CARD.md @@ -55,7 +55,11 @@ No benchmark available yet. ## Upstream -The original source code for this kernel comes from {{ upstream }}. +The original source code for this kernel comes from: + +{% for url in upstream %} +- {{ url }} +{% endfor %} {% endif %} {% if source %} diff --git a/kernel-builder/src/init/templates/build.toml b/kernel-builder/src/init/templates/build.toml index ef2b63202..28a7ff661 100644 --- a/kernel-builder/src/init/templates/build.toml +++ b/kernel-builder/src/init/templates/build.toml @@ -7,7 +7,7 @@ backends = [ {% endfor %} ] version = 1 -edition = 5 +edition = 6 [general.hub] repo-id = "{{ repo_id }}" diff --git a/kernel-builder/src/main.rs b/kernel-builder/src/main.rs index 83a735d62..e8021edce 100644 --- a/kernel-builder/src/main.rs +++ b/kernel-builder/src/main.rs @@ -5,7 +5,7 @@ use std::path::PathBuf; use clap::{Args, CommandFactory, Parser, Subcommand}; use clap_complete::Shell; use eyre::{Context, Result}; -use kernels_common::config::{v5, Build, BuildCompat}; +use kernels_common::config::{Build, BuildCompat, CurrentConfig}; use kernels_common::git::Oid; mod card; @@ -475,15 +475,15 @@ fn update_build(kernel_dir: Option) -> Result<()> { let kernel_dir = check_or_infer_kernel_dir(kernel_dir)?; let build_compat = BuildCompat::open(&kernel_dir)?; - if matches!(build_compat, BuildCompat::V5(_)) { + if matches!(build_compat, BuildCompat::V6(_)) { return Ok(()); } let build: Build = build_compat .try_into() .context("Cannot update build configuration")?; - let v5_build: v5::Build = build.into(); - let pretty_toml = toml::to_string_pretty(&v5_build)?; + let current_build: CurrentConfig = build.into(); + let pretty_toml = toml::to_string_pretty(¤t_build)?; let build_toml = kernel_dir.join("build.toml"); let mut writer = diff --git a/kernel-builder/tests/init_e2e.rs b/kernel-builder/tests/init_e2e.rs index 61e11c23a..40177f6a2 100644 --- a/kernel-builder/tests/init_e2e.rs +++ b/kernel-builder/tests/init_e2e.rs @@ -36,6 +36,11 @@ fn test_init_creates_expected_files() { assert!(dir.join("my_kernel_metal").exists()); assert!(!dir.join("my_kernel_cpu").exists()); assert!(!dir.join("my_kernel_xpu").exists()); + + let config: toml::Value = + toml::from_str(&fs::read_to_string(dir.join("build.toml")).unwrap()).unwrap(); + assert_eq!(config["general"]["edition"].as_integer(), Some(6)); + kernels_common::config::Build::open(&dir).unwrap(); } #[test] diff --git a/kernel-builder/tests/update_build_e2e.rs b/kernel-builder/tests/update_build_e2e.rs new file mode 100644 index 000000000..7f1e2e6b4 --- /dev/null +++ b/kernel-builder/tests/update_build_e2e.rs @@ -0,0 +1,61 @@ +use std::fs; +use std::path::Path; +use std::process::Command; + +fn update_build(dir: &Path) { + let output = Command::new(env!("CARGO_BIN_EXE_kernel-builder")) + .arg("update-build") + .arg(dir) + .output() + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); +} + +#[test] +fn legacy_builds_migrate_to_edition_six() { + for (edition, framework) in [ + ("", ""), + ("", "[torch-noarch]"), + ("edition = 5", "[torch-noarch]"), + ] { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("build.toml"); + fs::write( + &path, + format!( + r#"[general] +name = "test" +version = 1 +license = "MIT" +backends = ["cpu"] +upstream = "https://github.com/example/kernel" +{edition} +{framework} +"# + ), + ) + .unwrap(); + + update_build(dir.path()); + let migrated = fs::read_to_string(&path).unwrap(); + let value: toml::Value = toml::from_str(&migrated).unwrap(); + assert_eq!(value["general"]["edition"].as_integer(), Some(6)); + assert_eq!( + value["general"]["upstream"].as_array().unwrap(), + &[toml::Value::from("https://github.com/example/kernel")] + ); + kernels_common::config::Build::open(dir.path()).unwrap(); + + // Already-current configurations must not be rewritten. + fs::write(&path, format!("# Keep this comment.\n{migrated}")).unwrap(); + update_build(dir.path()); + assert_eq!( + fs::read_to_string(path).unwrap(), + format!("# Keep this comment.\n{migrated}") + ); + } +} diff --git a/kernel-port/README.md b/kernel-port/README.md index 048ba9b44..8bee26758 100644 --- a/kernel-port/README.md +++ b/kernel-port/README.md @@ -521,7 +521,7 @@ Record one `[kernel.]` section for the manifest. It writes no files; the ` ```kdl manifest name="" backends="[,...]" torch_src="[,...]" - [version=N] [edition=N] [license="..."] [upstream="..."] + [version=N] [edition=N] [license="..."] [upstream="[,...]"] [repo_id="org/name"] [hub_branch="..."] [python_depends="..."] [cuda_minver="..."] [cuda_maxver="..."] [cuda_python_depends="..."] [torch_pyext="py,pyi,..."] [torch_include=",..."] @@ -532,6 +532,9 @@ manifest name="" backends="..." noarch=#true [noarch_pyext="..."] Generate `build.toml` from the `kernel` sections recorded before it, plus the `[general]` and `[torch]` settings given here. The manifest is always generated, never overlaid: if it needs a field this op cannot emit, extend the op. +`upstream` accepts one URL or a comma-separated list of upstream repository URLs. +A single URL is written as a string; multiple URLs are written as a TOML array. + `torch_src` selects the binding sources; `torch_include` adds include directories for them (which is how a binding can `#include` a header from the kernel directory instead of restating its declarations). `noarch=#true` switches to a `[torch-noarch]` manifest and rejects the torch-only arguments. ```sh diff --git a/kernel-port/src/ops/manifest.rs b/kernel-port/src/ops/manifest.rs index a7bb36678..feb8fc317 100644 --- a/kernel-port/src/ops/manifest.rs +++ b/kernel-port/src/ops/manifest.rs @@ -10,8 +10,8 @@ pub struct Manifest { name: String, version: Option, license: Option, - edition: Option, - upstream: Option, + edition: Option, + upstream: Vec, backends: Vec, repo_id: Option, hub_branch: Option, @@ -47,10 +47,16 @@ impl Manifest { v.parse::() .with_context(|| format!("version must be an integer, got {v:?}"))?; } - let edition = args.take_opt("edition"); - if let Some(e) = &edition { - e.parse::() - .with_context(|| format!("edition must be an integer, got {e:?}"))?; + let edition = args + .take_opt("edition") + .map(|e| { + e.parse::() + .with_context(|| format!("edition must be an integer, got {e:?}")) + }) + .transpose()?; + let upstream = comma_list(&args.take_opt("upstream").unwrap_or_default()); + if upstream.len() > 1 && edition.unwrap_or(0) < 6 { + bail!("multiple upstream repositories require edition 6 or later"); } let noarch = match args.take_opt("noarch").as_deref() { None => false, @@ -105,7 +111,7 @@ impl Manifest { version, license: args.take_opt("license"), edition, - upstream: args.take_opt("upstream"), + upstream, backends, repo_id: args.take_opt("repo_id"), hub_branch: args.take_opt("hub_branch"), @@ -153,8 +159,12 @@ impl Manifest { if let Some(edition) = &self.edition { toml.push_str(&format!("edition = {edition}\n")); } - if let Some(upstream) = &self.upstream { - toml.push_str(&format!("upstream = {upstream:?}\n")); + match self.upstream.as_slice() { + [] => {} + [upstream] if self.edition.unwrap_or(0) < 6 => { + toml.push_str(&format!("upstream = {upstream:?}\n")); + } + upstreams => toml.push_str(&Self::toml_list("upstream", upstreams)), } toml.push_str(&Self::toml_list("backends", &self.backends)); if !self.python_depends.is_empty() { diff --git a/kernel-port/src/tests.rs b/kernel-port/src/tests.rs index cc08de201..2a3189f1f 100644 --- a/kernel-port/src/tests.rs +++ b/kernel-port/src/tests.rs @@ -667,6 +667,51 @@ fn manifest_upstream_field() { )); } +#[test] +fn manifest_edition_six_single_upstream_is_a_list() { + let mut ws = Workspace::from_files(BTreeMap::new()); + run_recipe( + &mut ws, + "manifest name=\"einops\" version=1 license=\"MIT\" edition=6 upstream=\"https://github.com/arogozhnikov/einops.git\" backends=\"cpu\" noarch=#true\n", + ); + let toml = ws.get_text("build.toml").unwrap(); + assert!(toml.contains("upstream = [\"https://github.com/arogozhnikov/einops.git\"]")); +} + +#[test] +fn manifest_legacy_editions_reject_multiple_upstreams() { + for edition in ["", "edition=5"] { + let mut ws = Workspace::from_files(BTreeMap::new()); + let err = run_recipe_err( + &mut ws, + &format!( + "manifest name=\"k\" version=1 license=\"MIT\" {edition} upstream=\"https://github.com/org/a,https://github.com/org/b\" backends=\"cpu\" noarch=#true\n" + ), + ); + assert!( + err.contains("multiple upstream repositories require edition 6"), + "{err}" + ); + } +} + +#[test] +fn manifest_multiple_upstreams() { + let mut ws = Workspace::from_files(BTreeMap::new()); + run_recipe( + &mut ws, + r#"manifest name="cv-utils" version=1 license="MIT" edition=6 upstream="https://github.com/ronghanghu/torch_generic_nms,https://github.com/ronghanghu/cc_torch" backends="cpu" noarch=#true +"#, + ); + let toml = ws.get_text("build.toml").unwrap(); + assert!(toml.contains(concat!( + "upstream = [\n", + " \"https://github.com/ronghanghu/torch_generic_nms\",\n", + " \"https://github.com/ronghanghu/cc_torch\",\n", + "]\n", + ))); +} + #[test] fn manifest_general_cuda_table() { let mut ws = Workspace::from_files(BTreeMap::new()); diff --git a/kernels-common/src/config/compat.rs b/kernels-common/src/config/compat.rs index 69d070306..c159b92e3 100644 --- a/kernels-common/src/config/compat.rs +++ b/kernels-common/src/config/compat.rs @@ -6,7 +6,7 @@ use serde_value::Value; use crate::config::ConfigError; -use super::{Build, v3, v4, v5}; +use super::{Build, v3, v4, v5, v6}; #[derive(Debug)] #[allow(clippy::large_enum_variant)] @@ -14,6 +14,7 @@ pub enum BuildCompat { V3(v3::Build), V4(v4::Build), V5(v5::Build), + V6(v6::Build), } impl BuildCompat { @@ -48,6 +49,10 @@ impl<'de> Deserialize<'de> for BuildCompat { match edition { // Configs with editions + Some(6) => v6::Build::deserialize(value) + .map(BuildCompat::V6) + .map_err(de::Error::custom), + Some(5) => v5::Build::deserialize(value) .map(BuildCompat::V5) .map_err(de::Error::custom), @@ -81,6 +86,7 @@ impl TryFrom for Build { BuildCompat::V3(v3_build) => v3_build.try_into(), BuildCompat::V4(v4_build) => Ok(v4_build.into()), BuildCompat::V5(v5_build) => Ok(v5_build.into()), + BuildCompat::V6(v6_build) => Ok(v6_build.into()), } } } diff --git a/kernels-common/src/config/mod.rs b/kernels-common/src/config/mod.rs index d8d2888a3..edb973eaf 100644 --- a/kernels-common/src/config/mod.rs +++ b/kernels-common/src/config/mod.rs @@ -16,6 +16,7 @@ mod compat; pub use compat::BuildCompat; mod git_url; +pub(crate) mod upstream; pub use git_url::GitUrl; mod kernel_deps; @@ -29,13 +30,14 @@ mod parse; pub mod v3; pub mod v4; pub mod v5; +pub mod v6; use itertools::Itertools; use crate::version::Version; -pub type CurrentConfig = v5::Build; -pub const CURRENT_EDITION: usize = 5; +pub type CurrentConfig = v6::Build; +pub const CURRENT_EDITION: usize = 6; /// Baseline `kernels` version that can load kernels built with the current /// metadata format. @@ -44,6 +46,9 @@ const KERNELS_VERSION_BASELINE: Version<3> = Version::new([0, 14, 0]); /// First `kernels` version that can resolve kernel dependencies. const KERNELS_VERSION_KERNEL_DEPENDS: Version<3> = Version::new([0, 17, 0]); +/// First `kernels` version that can read multiple upstream repositories. +const KERNELS_VERSION_MULTIPLE_UPSTREAMS: Version<3> = Version::new([0, 18, 0]); + pub struct Build { pub general: General, pub kernels: HashMap, @@ -83,6 +88,10 @@ impl Build { required = required.max(KERNELS_VERSION_KERNEL_DEPENDS); } + if self.general.upstream.len() > 1 { + required = required.max(KERNELS_VERSION_MULTIPLE_UPSTREAMS); + } + required } } @@ -136,8 +145,8 @@ pub struct General { /// Hugging Face Hub license identifier. pub license: String, - /// Original upstream repository for the kernel code. - pub upstream: Option, + /// Original upstream repositories for the kernel code. + pub upstream: Vec, /// Kernel-builder formatted source repository (must contain build.toml and flake.nix). pub source: Option, @@ -529,7 +538,7 @@ mod tests { name: KernelName::new("test-kernel").unwrap(), version: 1, license: "apache-2.0".to_string(), - upstream: None, + upstream: vec![], source: None, backends: vec![Backend::Tpu], hub: None, diff --git a/kernels-common/src/config/parse.rs b/kernels-common/src/config/parse.rs index 44dbd65c5..494655895 100644 --- a/kernels-common/src/config/parse.rs +++ b/kernels-common/src/config/parse.rs @@ -4,16 +4,23 @@ use std::path::Path; use eyre::{Context, Result, bail}; -use super::{Build, BuildCompat, CurrentConfig}; +use super::{Build, BuildCompat, CURRENT_EDITION, CurrentConfig}; pub(crate) fn parse_and_validate(kernel_dir: impl AsRef) -> Result { - // Only v4 is auto-upgraded to v5 on load; older editions must be migrated + // v4 and v5 are auto-upgraded on load; older editions must be migrated // explicitly with `update-build`. match parse_and_validate_compat(kernel_dir)? { - BuildCompat::V5(build) => Ok(build), + BuildCompat::V6(build) => Ok(build), + BuildCompat::V5(build) => { + eprintln!( + "⚠️ build.toml uses edition 5; upgrading to edition {CURRENT_EDITION} in memory. \ + Run `kernel-builder update-build` to persist the upgrade." + ); + Ok(Build::from(build).into()) + } BuildCompat::V4(build) => { eprintln!( - "⚠️ build.toml uses the legacy v4 format; upgrading to edition 5 in memory. \ + "⚠️ build.toml uses the legacy v4 format; upgrading to edition {CURRENT_EDITION} in memory. \ Run `kernel-builder update-build` to persist the upgrade." ); Ok(Build::from(build).into()) diff --git a/kernels-common/src/config/upstream.rs b/kernels-common/src/config/upstream.rs new file mode 100644 index 000000000..742abc85c --- /dev/null +++ b/kernels-common/src/config/upstream.rs @@ -0,0 +1,35 @@ +//! Compatibility encoding for upstream repositories in metadata files. + +use serde::{Deserialize, Deserializer, Serialize, Serializer}; + +use super::GitUrl; + +pub fn deserialize<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + #[derive(Deserialize)] + #[serde(untagged)] + enum Upstream { + Single(GitUrl), + Multiple(Vec), + } + + // Older metadata may explicitly contain null for an absent upstream. + Ok(match Option::::deserialize(deserializer)? { + None => vec![], + Some(Upstream::Single(url)) => vec![url], + Some(Upstream::Multiple(urls)) => urls, + }) +} + +pub fn serialize(upstreams: &[GitUrl], serializer: S) -> Result +where + S: Serializer, +{ + // Keep single-source kernels readable by older versions of `kernels`. + match upstreams { + [url] => url.serialize(serializer), + urls => urls.serialize(serializer), + } +} diff --git a/kernels-common/src/config/v3.rs b/kernels-common/src/config/v3.rs index 0672b4b96..abee88d43 100644 --- a/kernels-common/src/config/v3.rs +++ b/kernels-common/src/config/v3.rs @@ -199,7 +199,7 @@ impl TryFrom for super::General { name: general.name, version: general.version.unwrap_or(1), license, - upstream: general.upstream, + upstream: general.upstream.into_iter().collect(), source: None, backends: general.backends.into_iter().map(Into::into).collect(), cuda: general.cuda.map(Into::into), diff --git a/kernels-common/src/config/v4.rs b/kernels-common/src/config/v4.rs index 08b8f7bf5..56c12a5a4 100644 --- a/kernels-common/src/config/v4.rs +++ b/kernels-common/src/config/v4.rs @@ -197,7 +197,7 @@ impl From for super::General { name: general.name, version: general.version, license: general.license, - upstream: general.upstream, + upstream: general.upstream.into_iter().collect(), source: general.source, backends: general.backends.into_iter().map(Into::into).collect(), cuda: general.cuda.map(Into::into), diff --git a/kernels-common/src/config/v5.rs b/kernels-common/src/config/v5.rs index dc1078ee2..cfd319533 100644 --- a/kernels-common/src/config/v5.rs +++ b/kernels-common/src/config/v5.rs @@ -224,7 +224,7 @@ impl From for super::General { name: general.name, version: general.version, license: general.license, - upstream: general.upstream, + upstream: general.upstream.into_iter().collect(), source: general.source, backends: general.backends.into_iter().map(Into::into).collect(), cuda: general.cuda.map(Into::into), @@ -419,220 +419,3 @@ impl From for super::Kernel { } } } - -impl From for Build { - fn from(build: super::Build) -> Self { - Self { - general: build.general.into(), - framework: build.framework.into(), - kernels: build - .kernels - .into_iter() - .map(|(k, v)| (k, v.into())) - .collect(), - } - } -} - -impl From for General { - fn from(general: super::General) -> Self { - Self { - name: general.name, - version: general.version, - edition: Default::default(), - license: general.license, - upstream: general.upstream, - source: general.source, - backends: general.backends.into_iter().map(Into::into).collect(), - cuda: general.cuda.map(Into::into), - hub: general.hub.map(Into::into), - kernel_depends: general.kernel_depends, - neuron: general.neuron.map(Into::into), - python_depends: general.python_depends, - tpu: general.tpu.map(Into::into), - xpu: general.xpu.map(Into::into), - } - } -} - -impl From for Framework { - fn from(framework: super::Framework) -> Self { - match framework { - super::Framework::Torch(torch) => Framework::Torch(torch.into()), - super::Framework::TorchNoarch(torch_noarch) => { - Framework::TorchNoarch(torch_noarch.into()) - } - super::Framework::TvmFfi(tvm_ffi) => Framework::TvmFfi(tvm_ffi.into()), - } - } -} - -impl From for CudaGeneral { - fn from(cuda: super::CudaGeneral) -> Self { - Self { - minver: cuda.minver, - maxver: cuda.maxver, - kernel_depends: cuda.kernel_depends, - python_depends: cuda.python_depends, - } - } -} - -impl From for NeuronGeneral { - fn from(neuron: super::NeuronGeneral) -> Self { - Self { - kernel_depends: neuron.kernel_depends, - python_depends: neuron.python_depends, - } - } -} - -impl From for TpuGeneral { - fn from(tpu: super::TpuGeneral) -> Self { - Self { - python_depends: tpu.python_depends, - } - } -} - -impl From for XpuGeneral { - fn from(xpu: super::XpuGeneral) -> Self { - Self { - kernel_depends: xpu.kernel_depends, - python_depends: xpu.python_depends, - } - } -} - -impl From for Hub { - fn from(hub: super::Hub) -> Self { - Self { - repo_id: hub.repo_id, - branch: hub.branch, - } - } -} - -impl From for Torch { - fn from(torch: super::Torch) -> Self { - Self { - include: torch.include, - minver: torch.minver, - maxver: torch.maxver, - pyext: torch.pyext, - src: torch.src, - stable_abi: torch.stable_abi, - cxx_flags: torch.cxx_flags, - } - } -} - -impl From for TorchNoarch { - fn from(torch_noarch: super::TorchNoarch) -> Self { - Self { - pyext: torch_noarch.pyext, - cuda_capabilities: torch_noarch.cuda_capabilities, - rocm_archs: torch_noarch.rocm_archs, - } - } -} - -impl From for TvmFfi { - fn from(tvm_ffi: super::TvmFfi) -> Self { - Self { - include: tvm_ffi.include, - pyext: tvm_ffi.pyext, - src: tvm_ffi.src, - cxx_flags: tvm_ffi.cxx_flags, - } - } -} - -impl From for Backend { - fn from(backend: super::Backend) -> Self { - match backend { - super::Backend::Cann => Backend::Cann, - super::Backend::Cpu => Backend::Cpu, - super::Backend::Cuda => Backend::Cuda, - super::Backend::Metal => Backend::Metal, - super::Backend::Neuron => Backend::Neuron, - super::Backend::Rocm => Backend::Rocm, - super::Backend::Tpu => Backend::Tpu, - super::Backend::Xpu => Backend::Xpu, - } - } -} - -impl From for Kernel { - fn from(kernel: super::Kernel) -> Self { - match kernel { - super::Kernel::Cpu { - cxx_flags, - depends, - include, - src, - } => Kernel::Cpu { - cxx_flags, - depends, - include, - src, - }, - super::Kernel::Cuda { - cuda_capabilities, - cuda_flags, - cuda_minver, - cxx_flags, - depends, - include, - src, - } => Kernel::Cuda { - cuda_capabilities, - cuda_flags, - cuda_minver, - cxx_flags, - depends, - include, - src, - }, - super::Kernel::Metal { - cxx_flags, - depends, - include, - src, - } => Kernel::Metal { - cxx_flags, - depends, - include, - src, - }, - super::Kernel::Rocm { - cxx_flags, - depends, - rocm_archs, - hip_flags, - include, - src, - } => Kernel::Rocm { - cxx_flags, - depends, - rocm_archs, - hip_flags, - include, - src, - }, - super::Kernel::Xpu { - cxx_flags, - depends, - sycl_flags, - include, - src, - } => Kernel::Xpu { - cxx_flags, - depends, - sycl_flags, - include, - src, - }, - } - } -} diff --git a/kernels-common/src/config/v6.rs b/kernels-common/src/config/v6.rs new file mode 100644 index 000000000..4e8f82478 --- /dev/null +++ b/kernels-common/src/config/v6.rs @@ -0,0 +1,639 @@ +use std::collections::HashMap; +use std::path::PathBuf; + +use monostate::MustBe; +use serde::{Deserialize, Serialize}; + +use super::{Dependency, GitUrl, KernelDependency, KernelName}; +use crate::version::Version; + +// `monostate` validates the edition on read but provides no `Serialize` impl for it. +fn serialize_edition(_edition: &MustBe!(6), serializer: S) -> Result +where + S: serde::Serializer, +{ + serializer.serialize_u64(6) +} + +#[derive(Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields, rename_all = "kebab-case")] +pub struct Build { + pub general: General, + + #[serde(flatten)] + pub framework: Framework, + + #[serde(rename = "kernel", default)] + pub kernels: HashMap, +} + +#[derive(Debug, Deserialize, Serialize)] +#[serde(rename_all = "kebab-case")] +pub enum Framework { + Torch(Torch), + TorchNoarch(TorchNoarch), + TvmFfi(TvmFfi), +} + +#[derive(Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields, rename_all = "kebab-case")] +pub struct General { + pub name: KernelName, + + pub version: usize, + + /// Build format edition. Must be `6` for this schema. + #[serde(serialize_with = "serialize_edition")] + pub edition: MustBe!(6), + + pub license: String, + + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub upstream: Vec, + + pub source: Option, + + pub backends: Vec, + + pub cuda: Option, + + pub hub: Option, + + pub kernel_depends: Option>, + + pub neuron: Option, + + pub python_depends: Option>, + + pub tpu: Option, + + pub xpu: Option, +} + +#[derive(Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields, rename_all = "kebab-case")] +pub struct CudaGeneral { + pub minver: Option>, + pub maxver: Option>, + pub kernel_depends: Option>, + pub python_depends: Option>, +} + +#[derive(Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields, rename_all = "kebab-case")] +pub struct NeuronGeneral { + pub kernel_depends: Option>, + pub python_depends: Option>, +} + +#[derive(Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields, rename_all = "kebab-case")] +pub struct TpuGeneral { + pub python_depends: Option>, +} + +#[derive(Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields, rename_all = "kebab-case")] +pub struct XpuGeneral { + pub kernel_depends: Option>, + pub python_depends: Option>, +} + +#[derive(Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields, rename_all = "kebab-case")] +pub struct Hub { + pub repo_id: Option, + pub branch: Option, +} + +#[derive(Debug, Deserialize, Clone, Serialize)] +#[serde(deny_unknown_fields, rename_all = "kebab-case")] +pub struct Torch { + pub include: Option>, + pub minver: Option>, + pub maxver: Option>, + pub pyext: Option>, + + #[serde(default)] + pub src: Vec, + + /// Single version for all backends, or a per-backend table. + pub stable_abi: Option, + + pub cxx_flags: Option>, +} + +#[derive(Debug, Deserialize, Clone, Serialize)] +#[serde(deny_unknown_fields)] +pub struct TorchNoarch { + pub pyext: Option>, + + #[serde(default)] + pub cuda_capabilities: Option>, + + #[serde(default)] + pub rocm_archs: Option>, +} + +#[derive(Debug, Deserialize, Clone, Serialize)] +#[serde(deny_unknown_fields, rename_all = "kebab-case")] +pub struct TvmFfi { + pub include: Option>, + pub pyext: Option>, + pub src: Vec, + pub cxx_flags: Option>, +} + +#[derive(Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields, rename_all = "kebab-case", tag = "backend")] +pub enum Kernel { + #[serde(rename_all = "kebab-case")] + Cpu { + cxx_flags: Option>, + depends: Vec, + include: Option>, + src: Vec, + }, + #[serde(rename_all = "kebab-case")] + Cuda { + cuda_capabilities: Option>, + cuda_flags: Option>, + cuda_minver: Option>, + cxx_flags: Option>, + depends: Vec, + include: Option>, + src: Vec, + }, + #[serde(rename_all = "kebab-case")] + Metal { + cxx_flags: Option>, + depends: Vec, + include: Option>, + src: Vec, + }, + #[serde(rename_all = "kebab-case")] + Rocm { + cxx_flags: Option>, + depends: Vec, + rocm_archs: Option>, + hip_flags: Option>, + include: Option>, + src: Vec, + }, + #[serde(rename_all = "kebab-case")] + Xpu { + cxx_flags: Option>, + depends: Vec, + sycl_flags: Option>, + include: Option>, + src: Vec, + }, +} + +#[derive(Clone, Copy, Debug, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] +#[serde(deny_unknown_fields, rename_all = "kebab-case")] +pub enum Backend { + Cann, + Cpu, + Cuda, + Metal, + Neuron, + Rocm, + Tpu, + Xpu, +} + +impl From for super::Build { + fn from(build: Build) -> Self { + let kernels: HashMap = build + .kernels + .into_iter() + .map(|(k, v)| (k, v.into())) + .collect(); + + Self { + general: build.general.into(), + framework: build.framework.into(), + kernels, + } + } +} + +impl From for super::General { + fn from(general: General) -> Self { + Self { + name: general.name, + version: general.version, + license: general.license, + upstream: general.upstream, + source: general.source, + backends: general.backends.into_iter().map(Into::into).collect(), + cuda: general.cuda.map(Into::into), + hub: general.hub.map(Into::into), + kernel_depends: general.kernel_depends, + neuron: general.neuron.map(Into::into), + python_depends: general.python_depends, + tpu: general.tpu.map(Into::into), + xpu: general.xpu.map(Into::into), + } + } +} + +impl From for super::Framework { + fn from(framework: Framework) -> Self { + match framework { + Framework::Torch(torch) => super::Framework::Torch(torch.into()), + Framework::TorchNoarch(torch_noarch) => { + super::Framework::TorchNoarch(torch_noarch.into()) + } + Framework::TvmFfi(tvm_ffi) => super::Framework::TvmFfi(tvm_ffi.into()), + } + } +} + +impl From for super::CudaGeneral { + fn from(cuda: CudaGeneral) -> Self { + Self { + minver: cuda.minver, + maxver: cuda.maxver, + kernel_depends: cuda.kernel_depends, + python_depends: cuda.python_depends, + } + } +} + +impl From for super::NeuronGeneral { + fn from(neuron: NeuronGeneral) -> Self { + Self { + kernel_depends: neuron.kernel_depends, + python_depends: neuron.python_depends, + } + } +} + +impl From for super::TpuGeneral { + fn from(tpu: TpuGeneral) -> Self { + Self { + python_depends: tpu.python_depends, + } + } +} + +impl From for super::XpuGeneral { + fn from(xpu: XpuGeneral) -> Self { + Self { + kernel_depends: xpu.kernel_depends, + python_depends: xpu.python_depends, + } + } +} + +impl From for super::Hub { + fn from(hub: Hub) -> Self { + Self { + repo_id: hub.repo_id, + branch: hub.branch, + } + } +} + +impl From for super::Torch { + fn from(torch: Torch) -> Self { + Self { + include: torch.include, + minver: torch.minver, + maxver: torch.maxver, + pyext: torch.pyext, + src: torch.src, + stable_abi: torch.stable_abi, + cxx_flags: torch.cxx_flags, + } + } +} + +impl From for super::TorchNoarch { + fn from(torch_noarch: TorchNoarch) -> Self { + Self { + pyext: torch_noarch.pyext, + cuda_capabilities: torch_noarch.cuda_capabilities, + rocm_archs: torch_noarch.rocm_archs, + } + } +} + +impl From for super::TvmFfi { + fn from(tvm_ffi: TvmFfi) -> Self { + Self { + include: tvm_ffi.include, + pyext: tvm_ffi.pyext, + src: tvm_ffi.src, + cxx_flags: tvm_ffi.cxx_flags, + } + } +} + +impl From for super::Backend { + fn from(backend: Backend) -> Self { + match backend { + Backend::Cann => super::Backend::Cann, + Backend::Cpu => super::Backend::Cpu, + Backend::Cuda => super::Backend::Cuda, + Backend::Metal => super::Backend::Metal, + Backend::Neuron => super::Backend::Neuron, + Backend::Rocm => super::Backend::Rocm, + Backend::Tpu => super::Backend::Tpu, + Backend::Xpu => super::Backend::Xpu, + } + } +} + +impl From for super::Kernel { + fn from(kernel: Kernel) -> Self { + match kernel { + Kernel::Cpu { + cxx_flags, + depends, + include, + src, + } => super::Kernel::Cpu { + cxx_flags, + depends, + include, + src, + }, + Kernel::Cuda { + cuda_capabilities, + cuda_flags, + cuda_minver, + cxx_flags, + depends, + include, + src, + } => super::Kernel::Cuda { + cuda_capabilities, + cuda_flags, + cuda_minver, + cxx_flags, + depends, + include, + src, + }, + Kernel::Metal { + cxx_flags, + depends, + include, + src, + } => super::Kernel::Metal { + cxx_flags, + depends, + include, + src, + }, + Kernel::Rocm { + cxx_flags, + depends, + rocm_archs, + hip_flags, + include, + src, + } => super::Kernel::Rocm { + cxx_flags, + depends, + rocm_archs, + hip_flags, + include, + src, + }, + Kernel::Xpu { + cxx_flags, + depends, + sycl_flags, + include, + src, + } => super::Kernel::Xpu { + cxx_flags, + depends, + sycl_flags, + include, + src, + }, + } + } +} + +impl From for Build { + fn from(build: super::Build) -> Self { + Self { + general: build.general.into(), + framework: build.framework.into(), + kernels: build + .kernels + .into_iter() + .map(|(k, v)| (k, v.into())) + .collect(), + } + } +} + +impl From for General { + fn from(general: super::General) -> Self { + Self { + name: general.name, + version: general.version, + edition: Default::default(), + license: general.license, + upstream: general.upstream, + source: general.source, + backends: general.backends.into_iter().map(Into::into).collect(), + cuda: general.cuda.map(Into::into), + hub: general.hub.map(Into::into), + kernel_depends: general.kernel_depends, + neuron: general.neuron.map(Into::into), + python_depends: general.python_depends, + tpu: general.tpu.map(Into::into), + xpu: general.xpu.map(Into::into), + } + } +} + +impl From for Framework { + fn from(framework: super::Framework) -> Self { + match framework { + super::Framework::Torch(torch) => Framework::Torch(torch.into()), + super::Framework::TorchNoarch(torch_noarch) => { + Framework::TorchNoarch(torch_noarch.into()) + } + super::Framework::TvmFfi(tvm_ffi) => Framework::TvmFfi(tvm_ffi.into()), + } + } +} + +impl From for CudaGeneral { + fn from(cuda: super::CudaGeneral) -> Self { + Self { + minver: cuda.minver, + maxver: cuda.maxver, + kernel_depends: cuda.kernel_depends, + python_depends: cuda.python_depends, + } + } +} + +impl From for NeuronGeneral { + fn from(neuron: super::NeuronGeneral) -> Self { + Self { + kernel_depends: neuron.kernel_depends, + python_depends: neuron.python_depends, + } + } +} + +impl From for TpuGeneral { + fn from(tpu: super::TpuGeneral) -> Self { + Self { + python_depends: tpu.python_depends, + } + } +} + +impl From for XpuGeneral { + fn from(xpu: super::XpuGeneral) -> Self { + Self { + kernel_depends: xpu.kernel_depends, + python_depends: xpu.python_depends, + } + } +} + +impl From for Hub { + fn from(hub: super::Hub) -> Self { + Self { + repo_id: hub.repo_id, + branch: hub.branch, + } + } +} + +impl From for Torch { + fn from(torch: super::Torch) -> Self { + Self { + include: torch.include, + minver: torch.minver, + maxver: torch.maxver, + pyext: torch.pyext, + src: torch.src, + stable_abi: torch.stable_abi, + cxx_flags: torch.cxx_flags, + } + } +} + +impl From for TorchNoarch { + fn from(torch_noarch: super::TorchNoarch) -> Self { + Self { + pyext: torch_noarch.pyext, + cuda_capabilities: torch_noarch.cuda_capabilities, + rocm_archs: torch_noarch.rocm_archs, + } + } +} + +impl From for TvmFfi { + fn from(tvm_ffi: super::TvmFfi) -> Self { + Self { + include: tvm_ffi.include, + pyext: tvm_ffi.pyext, + src: tvm_ffi.src, + cxx_flags: tvm_ffi.cxx_flags, + } + } +} + +impl From for Backend { + fn from(backend: super::Backend) -> Self { + match backend { + super::Backend::Cann => Backend::Cann, + super::Backend::Cpu => Backend::Cpu, + super::Backend::Cuda => Backend::Cuda, + super::Backend::Metal => Backend::Metal, + super::Backend::Neuron => Backend::Neuron, + super::Backend::Rocm => Backend::Rocm, + super::Backend::Tpu => Backend::Tpu, + super::Backend::Xpu => Backend::Xpu, + } + } +} + +impl From for Kernel { + fn from(kernel: super::Kernel) -> Self { + match kernel { + super::Kernel::Cpu { + cxx_flags, + depends, + include, + src, + } => Kernel::Cpu { + cxx_flags, + depends, + include, + src, + }, + super::Kernel::Cuda { + cuda_capabilities, + cuda_flags, + cuda_minver, + cxx_flags, + depends, + include, + src, + } => Kernel::Cuda { + cuda_capabilities, + cuda_flags, + cuda_minver, + cxx_flags, + depends, + include, + src, + }, + super::Kernel::Metal { + cxx_flags, + depends, + include, + src, + } => Kernel::Metal { + cxx_flags, + depends, + include, + src, + }, + super::Kernel::Rocm { + cxx_flags, + depends, + rocm_archs, + hip_flags, + include, + src, + } => Kernel::Rocm { + cxx_flags, + depends, + rocm_archs, + hip_flags, + include, + src, + }, + super::Kernel::Xpu { + cxx_flags, + depends, + sycl_flags, + include, + src, + } => Kernel::Xpu { + cxx_flags, + depends, + sycl_flags, + include, + src, + }, + } + } +} diff --git a/kernels-common/src/metadata.rs b/kernels-common/src/metadata.rs index cbf4564e8..ed5cedb71 100644 --- a/kernels-common/src/metadata.rs +++ b/kernels-common/src/metadata.rs @@ -99,8 +99,12 @@ pub struct Metadata { #[serde(skip_serializing_if = "Option::is_none")] pub kernels_minver: Option>, pub license: String, - #[serde(skip_serializing_if = "Option::is_none")] - pub upstream: Option, + #[serde( + default, + with = "crate::config::upstream", + skip_serializing_if = "Vec::is_empty" + )] + pub upstream: Vec, #[serde(skip_serializing_if = "Option::is_none")] pub source: Option, pub python_depends: Vec, @@ -211,7 +215,7 @@ mod tests { name: KernelName::new("test-kernel").unwrap(), version: 1, license: "apache-2.0".to_string(), - upstream: None, + upstream: vec![], source: None, backends: vec![Backend::Cuda, Backend::Rocm, Backend::Cpu], hub: None, @@ -268,7 +272,7 @@ mod tests { name: KernelName::new("test-kernel").unwrap(), version: 1, license: "apache-2.0".to_string(), - upstream: None, + upstream: vec![], source: None, backends: vec![Backend::Cuda], hub: None, @@ -458,7 +462,7 @@ mod tests { assert_eq!(metadata.id, "_einops_cpu_d45adda"); assert_eq!(metadata.version, 1); assert_eq!(metadata.license, "MIT"); - assert!(metadata.upstream.is_some()); + assert_eq!(metadata.upstream.len(), 1); assert!(metadata.source.is_none()); assert!(metadata.python_depends.is_empty()); assert!(metadata.kernel_depends.is_empty()); diff --git a/kernels-common/tests/upstream.rs b/kernels-common/tests/upstream.rs new file mode 100644 index 000000000..673b7fb86 --- /dev/null +++ b/kernels-common/tests/upstream.rs @@ -0,0 +1,228 @@ +use kernels_common::config::{Backend, Build, BuildCompat, CurrentConfig}; +use kernels_common::metadata::Metadata; +use kernels_common::version::Version; +use serde_json::json; + +const FIRST: &str = "https://github.com/ronghanghu/torch_generic_nms"; +const SECOND: &str = "git@github.com:ronghanghu/cc_torch.git"; + +fn config(edition: &str, upstream: &str) -> String { + format!( + r#"[general] +name = "cv-utils" +version = 1 +license = "MIT" +backends = ["cpu"] +{edition} +{upstream} +[torch-noarch] +"# + ) +} + +#[test] +fn build_upstreams_round_trip_through_metadata_and_current_config() { + for (field, expected) in [ + (String::new(), vec![]), + ("upstream = []".into(), vec![]), + (format!("upstream = [{FIRST:?}]"), vec![FIRST]), + ( + format!("upstream = [{FIRST:?}, {SECOND:?}]"), + vec![FIRST, SECOND], + ), + ] { + let compat: BuildCompat = toml::from_str(&config("edition = 6", &field)).unwrap(); + assert!(matches!(&compat, BuildCompat::V6(_))); + let build: Build = compat.try_into().unwrap(); + let metadata = Metadata::for_backend(&build, "cv-utils".into(), Backend::Cpu).unwrap(); + assert_eq!( + metadata + .upstream + .iter() + .map(ToString::to_string) + .collect::>(), + expected + ); + assert_eq!( + metadata.kernels_minver, + Some(if expected.len() > 1 { + Version::new([0, 18, 0]) + } else { + Version::new([0, 14, 0]) + }) + ); + + let serialized = serde_json::to_value(&metadata).unwrap(); + match expected.as_slice() { + [] => assert!(serialized.get("upstream").is_none()), + [url] => assert_eq!(serialized["upstream"], json!(url)), + urls => assert_eq!(serialized["upstream"], json!(urls)), + } + let parsed: Metadata = serde_json::from_value(serialized).unwrap(); + assert_eq!(parsed.upstream, metadata.upstream); + + let current: CurrentConfig = build.into(); + let serialized = toml::to_string(¤t).unwrap(); + let value: toml::Value = toml::from_str(&serialized).unwrap(); + assert_eq!(value["general"]["edition"].as_integer(), Some(6)); + if !expected.is_empty() { + assert_eq!( + value["general"]["upstream"].as_array().unwrap(), + &expected + .iter() + .map(|url| toml::Value::from(*url)) + .collect::>() + ); + } + let parsed: CurrentConfig = toml::from_str(&serialized).unwrap(); + assert_eq!(parsed.general.upstream, metadata.upstream); + } +} + +#[test] +fn legacy_build_upstream_survives_migration() { + for (edition, framework) in [ + ("", ""), + ("", "[torch-noarch]"), + ("edition = 5", "[torch-noarch]"), + ] { + for (field, expected) in [ + (String::new(), vec![]), + (format!("upstream = {FIRST:?}"), vec![FIRST]), + ] { + let input = config(edition, &field).replace("[torch-noarch]", framework); + let compat: BuildCompat = toml::from_str(&input).unwrap(); + if !edition.is_empty() { + assert!(matches!(&compat, BuildCompat::V5(_))); + } else if framework.is_empty() { + assert!(matches!(&compat, BuildCompat::V3(_))); + } else { + assert!(matches!(&compat, BuildCompat::V4(_))); + } + let build: Build = compat.try_into().unwrap(); + let current: CurrentConfig = build.into(); + let serialized = toml::to_string(¤t).unwrap(); + let parsed: BuildCompat = toml::from_str(&serialized).unwrap(); + assert!(matches!(parsed, BuildCompat::V6(_))); + assert_eq!( + current + .general + .upstream + .iter() + .map(ToString::to_string) + .collect::>(), + expected + ); + } + } +} + +#[test] +fn legacy_build_rejects_upstream_lists() { + for (edition, framework) in [ + ("", ""), + ("", "[torch-noarch]"), + ("edition = 5", "[torch-noarch]"), + ] { + for urls in [vec![], vec![FIRST], vec![FIRST, SECOND]] { + let input = config(edition, &format!("upstream = {urls:?}")) + .replace("[torch-noarch]", framework); + assert!(toml::from_str::(&input).is_err()); + } + } +} + +#[test] +fn build_open_upgrades_supported_editions_in_memory() { + for (edition, field) in [ + ("", format!("upstream = {FIRST:?}")), + ("edition = 5", format!("upstream = {FIRST:?}")), + ("edition = 6", format!("upstream = [{FIRST:?}]")), + ] { + let dir = tempfile::tempdir().unwrap(); + let input = config(edition, &field); + let path = dir.path().join("build.toml"); + std::fs::write(&path, &input).unwrap(); + let build = Build::open(dir.path()).unwrap(); + assert_eq!(build.general.upstream[0].to_string(), FIRST); + assert_eq!(std::fs::read_to_string(path).unwrap(), input); + } +} + +#[test] +fn unsupported_build_editions_are_rejected() { + let err = toml::from_str::(&config("edition = 7", "")).unwrap_err(); + assert!(err.to_string().contains("unsupported build edition 7")); + + let dir = tempfile::tempdir().unwrap(); + std::fs::write( + dir.path().join("build.toml"), + config("", "").replace("[torch-noarch]", ""), + ) + .unwrap(); + assert!( + Build::open(dir.path()) + .err() + .unwrap() + .to_string() + .contains("update-build") + ); +} + +#[test] +fn metadata_accepts_legacy_and_list_upstreams() { + for (upstream, expected) in [ + (json!(null), vec![]), + (json!([]), vec![]), + (json!(FIRST), vec![FIRST]), + (json!([FIRST]), vec![FIRST]), + (json!([FIRST, SECOND]), vec![FIRST, SECOND]), + ] { + let metadata: Metadata = serde_json::from_value(json!({ + "name": "cv-utils", "id": "cv-utils", "version": 1, "license": "MIT", + "python-depends": [], "backend": {"type": "cpu"}, "upstream": upstream, + })) + .unwrap(); + assert_eq!( + metadata + .upstream + .iter() + .map(ToString::to_string) + .collect::>(), + expected + ); + } +} + +#[test] +fn invalid_upstreams_are_rejected() { + for upstream in [ + json!(42), + json!({}), + json!("not a url"), + json!([FIRST, "ftp://example.com/repo"]), + json!([FIRST, null]), + ] { + let metadata = json!({ + "name": "cv-utils", "id": "cv-utils", "version": 1, "license": "MIT", + "python-depends": [], "backend": {"type": "cpu"}, "upstream": upstream, + }); + assert!(serde_json::from_value::(metadata).is_err()); + } + for upstream in [ + "42", + "\"https://example.com/repo\"", + "{}", + "\"not a url\"", + "[\"https://example.com/repo\", 42]", + "[\"ftp://example.com/repo\"]", + ] { + assert!( + toml::from_str::(&config( + "edition = 6", + &format!("upstream = {upstream}") + )) + .is_err() + ); + } +} diff --git a/kernels/rust/lib.rs b/kernels/rust/lib.rs index a502a9112..604311895 100644 --- a/kernels/rust/lib.rs +++ b/kernels/rust/lib.rs @@ -399,7 +399,7 @@ struct PyMetadata { version: usize, kernels_minver: Option, license: String, - upstream: Option, + upstream: Vec, source: Option, python_depends: Vec, kernel_depends: Vec, @@ -416,7 +416,7 @@ impl From for PyMetadata { version: m.version, kernels_minver: m.kernels_minver.map(Into::into), license: m.license, - upstream: m.upstream.map(|u| u.as_url().to_string()), + upstream: m.upstream.iter().map(|u| u.as_url().to_string()).collect(), source: m.source.map(|u| u.as_url().to_string()), python_depends: m.python_depends, kernel_depends: m.kernel_depends.into_iter().map(Into::into).collect(), @@ -481,9 +481,10 @@ impl PyMetadata { &self.license } + /// All upstream repositories, in declaration order. #[getter] - fn upstream(&self) -> Option<&str> { - self.upstream.as_deref() + fn upstream(&self) -> &[String] { + &self.upstream } #[getter] diff --git a/kernels/src/kernels/_rust.pyi b/kernels/src/kernels/_rust.pyi index 626d42062..4a1ee65db 100644 --- a/kernels/src/kernels/_rust.pyi +++ b/kernels/src/kernels/_rust.pyi @@ -754,7 +754,9 @@ class Metadata: @property def license(self) -> Optional[str]: ... @property - def upstream(self) -> Optional[str]: ... + def upstream(self) -> list[str]: + """All upstream repositories, in declaration order; empty if unspecified.""" + ... @property def source(self) -> Optional[str]: ... @property diff --git a/kernels/src/kernels/cli/info.py b/kernels/src/kernels/cli/info.py index 59cfd68f9..b4ee66719 100644 --- a/kernels/src/kernels/cli/info.py +++ b/kernels/src/kernels/cli/info.py @@ -147,7 +147,8 @@ def value(v) -> str: print(f"Version: {value(info.get('version'))}") print(f"Minimum kernels version: {value(info.get('kernels_minver'))}") print(f"License: {value(info.get('license'))}") - print(f"Upstream: {value(info.get('upstream'))}") + upstream = info.get("upstream") + print(f"Upstream: {', '.join(upstream) if upstream is not None and len(upstream) > 0 else '-'}") print(f"Source: {value(info.get('source'))}") python_depends = info.get("python_depends") print(f"Python dependencies: {', '.join(python_depends) if python_depends else '-'}") diff --git a/kernels/src/kernels/importer.py b/kernels/src/kernels/importer.py index c18d930a2..7f2f6aaa9 100644 --- a/kernels/src/kernels/importer.py +++ b/kernels/src/kernels/importer.py @@ -26,7 +26,7 @@ class LoadedKernel: - `kernels_minver` (`Version | None`): the minimum `kernels` library version required to load the kernel. - `license` (`str`): the license of the kernel. - - `upstream` (`str | None`): the original upstream repository of the kernel. + - `upstream` (`list[str]`): the original upstream repositories of the kernel. - `source` (`str | None`): the kernel-builder formatted source repository. - `python_depends` (`list[str]`): required Python dependencies. - `backend`: information about the kernel's backend. diff --git a/kernels/tests/test_cli_info.py b/kernels/tests/test_cli_info.py index 097987c24..b1c45b48c 100644 --- a/kernels/tests/test_cli_info.py +++ b/kernels/tests/test_cli_info.py @@ -32,7 +32,16 @@ def test_info_hub_rejects_revision_and_version(): print_kernel_info("kernels-community/activation", revision="main", version=1) -def test_info_local(tmp_path, capsys): +@pytest.mark.parametrize( + "upstream", + [ + None, + [], + "https://github.com/example/activation", + ["https://github.com/example/activation", "https://github.com/example/other"], + ], +) +def test_info_local(tmp_path, capsys, upstream): variant_dir = tmp_path / "build" / "torch28-cxx11-cu128-x86_64-linux" variant_dir.mkdir(parents=True) (variant_dir / "metadata.json").write_text( @@ -42,7 +51,7 @@ def test_info_local(tmp_path, capsys): "name": "activation", "version": 1, "license": "Apache-2.0", - "upstream": "https://github.com/example/activation", + "upstream": upstream, "python-depends": ["torch"], "backend": {"type": "cuda"}, } @@ -55,11 +64,18 @@ def test_info_local(tmp_path, capsys): assert info["name"] == "activation" assert info["version"] == 1 assert info["license"] == "Apache-2.0" - assert info["upstream"] == "https://github.com/example/activation" + upstreams = [upstream] if isinstance(upstream, str) else (upstream or []) + assert info["upstream"] == upstreams + assert "upstreams" not in info assert info["source"] is None assert info["python_depends"] == ["torch"] assert info["backends"] == ["cuda"] + print_kernel_info(str(tmp_path)) + out = capsys.readouterr().out + expected = ", ".join(upstreams) if upstreams else "-" + assert f"Upstream: {expected}\n" in out + def test_info_local_rejects_revision(tmp_path): with pytest.raises(SystemExit): diff --git a/kernels/tests/test_data_metadata.py b/kernels/tests/test_data_metadata.py index 3631399e2..16f66e90d 100644 --- a/kernels/tests/test_data_metadata.py +++ b/kernels/tests/test_data_metadata.py @@ -95,7 +95,7 @@ def test_metadata_load_full(tmp_path): assert m.version == 1 assert m.kernels_minver == Version.from_str("0.17.0") assert m.license == "Apache-2.0" - assert m.upstream == "https://github.com/example/kernel" + assert m.upstream == ["https://github.com/example/kernel"] assert m.source == "https://github.com/example/kernel-builder" assert m.python_depends == ["torch"] assert m.backend.backend_type == Backend.CUDA @@ -120,7 +120,7 @@ def test_metadata_load_minimal(tmp_path): assert m.version == 1 assert m.kernels_minver is None assert m.license == "Apache-2.0" - assert m.upstream is None + assert m.upstream == [] assert m.source is None assert m.python_depends == [] assert m.backend.backend_type == Backend.CPU @@ -223,3 +223,37 @@ def test_kernel_dependency_is_immutable(): dep.repo_id = "other/repo" with pytest.raises(AttributeError): dep.version = KernelVersion.Version(2) + + +@pytest.mark.parametrize( + ("upstream", "expected"), + [ + (None, []), + ([], []), + pytest.param( + "https://github.com/example/first", + ["https://github.com/example/first"], + id="legacy-string", + ), + (["https://github.com/example/first"], ["https://github.com/example/first"]), + ( + ["https://github.com/example/first", "https://github.com/example/second"], + ["https://github.com/example/first", "https://github.com/example/second"], + ), + ], +) +def test_metadata_upstreams(upstream, expected): + m = Metadata.from_bytes( + json.dumps( + { + "id": "test", + "name": "test", + "version": 1, + "license": "MIT", + "python-depends": [], + "backend": {"type": "cpu"}, + "upstream": upstream, + } + ).encode() + ) + assert m.upstream == expected