diff --git a/.gitignore b/.gitignore index 5496dac9..5401b0d9 100644 --- a/.gitignore +++ b/.gitignore @@ -151,4 +151,9 @@ package-lock.json.bak # MyPy .mypy_cache/ .dmypy.json -dmypy.json \ No newline at end of file +dmypy.json + +# Repo-root test-run artifacts (some ODCS CLI tests write the default +# output name to cwd as a side effect — they live in pytest's tmp_path +# but the executable path resolution may surface them at the repo root) +/mycontract-fluid.yaml \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 268f1b68..a68faa12 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,287 @@ All notable changes to FLUID Forge CLI will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Changed +- **Breaking: `fluid odps export` / `fluid opds export` default to Bitol + ODPS v1.0.0** instead of ODPI v4.1 (Linux Foundation). The output is a + Bitol product document plus one ODCS contract per output port: `--out-dir` + writes the canonical bundle (`.odps.` + `.odcs.`), + `--out FILE` writes the product to FILE (format inferred from its suffix + unless `-f/--format` is given) and the per-port contracts next to it, and + stdout receives the product. Get the 0.7.8 output with `--spec odpi-4.1` + (or the deprecated `--version 4.1`). Strict validation is on by default; + `--no-validate-strict` still validates but reports problems as warnings. + Without an explicit `--spec` the version follows `ODPS_API_VERSION`, then + an imported document's own `apiVersion`, then v1.0.0; the success message + reports the version actually written. `opds` is an alias of `odps`. + `--spec odpi-4.1` writes the 0.7.8 single JSON document and rejects + `--out-dir` / `-f yaml` with a clear error instead of ignoring them. +- **Breaking: `fluid dmm publish` and `fluid publish --catalog + datamesh-manager` send Bitol ODPS by default** (previously the legacy Data + Product Specification 0.0.1). Current Entropy Data / Entropy CE + organisations accept only `odps` and reject DPS with HTTP 400 + (`Specification type 'dps' is not supported`), so the old default was a + guaranteed failure there. Opt back into DPS with `--data-product-spec + 0.0.1`, `provider_hint="dps"`, the new `DMM_DATA_PRODUCT_SPEC` env var, or + (catalog path) catalog config `data_product_spec: "0.0.1"` / + `CatalogTarget.catalog_hints`. Resolution order: explicit value → provider + hint (`odps`/`opds` → ODPS, `dps` → 0.0.1) → `DMM_DATA_PRODUCT_SPEC` → + ODPS; the token `dps` means `0.0.1` everywhere a spec is accepted. The + ODPS `team.name` is the team id the publish auto-creates (the `--team-id` + value, or the owner team slugged, e.g. `customer-platform`). The catalog + path keeps its 0.7.8 defaults (a missing `metadata.status` publishes as + `active`; `metadata.domain` / `version` fall back to the top-level + fields) and now publishes one data contract per expose + (`{productId}.{exposeId}`) instead of one per product. Verified live + against Entropy CE 2.1.x on both publish paths at ODCS v3.1.0 and v3.2.0 + (`scripts/dmm_live_e2e.sh`). +- `BaseCatalogProvider.publish` / `update` take `(fluid, target=None)`; + `target` defaults to `CatalogTarget.from_fluid(fluid)`. The 0.7.8 + `publish(asset)` API keeps working for one release (see Deprecated): + `fluid publish` treats a provider as using the new API only when its + `publish` declares `(fluid, target)` (second parameter named `target` or + annotated `CatalogTarget`); every other provider, including duck-typed + plugins, takes the 0.7.8 `map_contract_to_asset` → `validate_asset` → + `publish(asset)` route. Missing `domain` / owner still fall back to + `general` / `unknown`; files that are not FLUID contracts (no + `name` or `metadata.name`, e.g. fragments, overlays, config files) are + rejected with a clear error. +- **Mapper round-trip architecture: verbatim stash + native overlay.** On + import every ODCS/ODPS section (document, schema object, property, port) + is stashed verbatim; on export the stash is the base layer and only the + FLUID-native fields that actually changed are overlaid (three-way merge, + `overlay_native`). This replaces the hand-maintained per-field + pass-through tuples and means fields added by future spec versions + round-trip with no code. FLUID edits to `owner`, `expects`, an expose + `binding` and `qos` (on any expose) of an imported contract reach the + stashed team / servers / `slaProperties`, for standalone ODCS imports and + for every per-port contract of an imported ODPS bundle; a server an + expose binding still selects is kept when its `expects` entry is removed. + Unchanged contracts still round-trip verbatim, and imported FLUID never + contains YAML anchors. + Validated lossless against verbatim upstream Bitol examples, including + the 192 KB v3.0.0 AdventureWorks contract. +- `OdcsProvider.validate_contract` / `BitolOdpsProvider.validate_product` + and `fluid odcs validate` select the JSON Schema from the document's + `apiVersion` rather than the provider's configured emit version. YAML + dates are validated as the ISO strings JSON would carry, so an unquoted + `2024-01-01` no longer fails a `type: string` check. +- `validate_via_vowl` skips with a warning (instead of failing the export) + when the installed `vowl` predates the document's `apiVersion` (`vowl` + 0.0.5 supports up to v3.1.0; 0.0.6 adds v3.2.0). +- The `odcs-strict` extra now also installs `open-data-contract-standard` + and `deepdiff`. `vowl` and `open-data-contract-standard` require Python + 3.10+, so on Python 3.9 the extra installs only `deepdiff` (the vowl pass + is skipped and `OdcsProvider.to_model()` reports the missing package). + `deepdiff` is also in the `dev` extra; it backs `roundtrip_check()`. +- `requests` is now a core dependency, so `fluid dmm publish` works on a + plain `pip install fluid-forge`. +- **ODCS v3.1.0 import is now lossless.** `team` in the v3.1.0 object form + and the v3.0.x array form, `required` (not the non-existent + `isNullable`), and the `slaProperties`, contract/object/property-level + `quality`, `relationships`, `support`, `price`, `customProperties`, + `authoritativeDefinitions` and `roles` sections are preserved end-to-end. + ODCS → FLUID → ODCS round-trips zero-diff on the full v3.1.0 fixture. + Import still rejects schema-invalid ODCS (`lenient=True` downgrades that + to a warning) and derives an expose `binding` from the server that + unambiguously backs each SchemaObject — never from `physicalType`. +- ODCS export of fresh FLUID: falls back to top-level `name`, + `description`, `tags`, `domain` and `lifecycle.state` when `metadata.*` + does not set them; field-level `quality` items are appended to the + auto-generated checks; `map` / `vector` and the v3.2.0-only server engines + are emitted only when the target is v3.2.0+ (below that they fall back to + `object` / `array` and `custom`, and an imported v3.2.0 contract exported + at an older version has its v3.2.0-only content downgraded, with a + warning, so it validates against the version it declares); each + `server.type` emits only the fields + its ODCS `ServerSource` allows, and a FLUID `binding.format` reaches the + server (local-platform contracts, examples 01–06 and every `fluid init` + template, export valid ODCS). +- **ODCS provider refactored into modular bidirectional mappers.** The + monolithic 1441-line `odcs.py` split into `provider.py`, `validation.py`, + `io.py`, and `mappers/{metadata,team,schema,quality,sla,servers,types}.py`. + Each section module exposes paired `to_fluid` / `to_odcs` functions. All + pass-through values live under a single `odcs_passthrough` namespace at + each level (metadata / expose / field). `fluid_build.providers.odcs.odcs` + remains as a compatibility shim. +- **Naming disambiguation.** The provider registry has three distinct + names: `odcs` (Open Data Contract Standard), `odps` (ODPI v4.1, Linux + Foundation; export-only), `odps_bitol` (Bitol Open Data Product Standard; + bidirectional). `fluid --help` labels `odps` as Bitol ODPS (default) / + ODPI v4.1 via `--spec`. +- Config: `~/.fluid/config.yaml` is read as an additional, lowest-priority + user layer; the first-found legacy file (`~/.fluidrc.yaml`, `~/.fluidrc`, + `~/.config/fluid/config.yaml`) is still read exactly as before and wins. + Config values may reference environment variables as `${VAR}`; they are + expanded when read and stored unexpanded, so `save_user_config()` writes + the placeholder, never the secret. + +### Added +- **ODCS v3.2.0 and Bitol ODPS v1.1.0** vendored alongside v3.1.0 / v1.0.0. + Validation is *version-selected* from the document's own `apiVersion` + (`resolve_schema_version`); older-than-vendored versions (e.g. v3.0.0) + validate against the newest vendored schema, a strict superset. Emit + defaults are unchanged (v3.1.0 / v1.0.0); opt into the newer version with + `OdcsProvider(api_version=…)` / `BitolOdpsProvider(api_version=…)`, the + `ODCS_API_VERSION` / `ODPS_API_VERSION` env vars, or + `fluid opds export --spec bitol-1.1.0`. Re-exports of imported documents + preserve the source's `apiVersion`. +- ODCS v3.2.0 content: `map` / `vector` logical types; `context`, + `synonyms`, `enum`, `semanticType`, `deprecated`, stable `id`s; ten new + `Server.type` engines (iceberg, hana, teradata, exasol, ingres, + vectorwise, versant, zen, btrieve, poet/fastobjects) with the + allowed-field tables the schema's `unevaluatedProperties: false` requires. +- ODPS v1.1.0 content: first-class product `type` (≥ v1.1.0 exports emit it + natively; v1.0.0 exports keep the legacy `customProperties` encoding), + `deprecated`, `context`, `synonyms`; relaxed `status` / port `version` / + `contractId` requirements honoured on import. A product rendered below + v1.1.0 is fitted to the vendored v1.0.0 schema, nested objects included + (management ports, SBOM entries, input contracts, support channels, team + members): a key v1.0.0 does not allow moves into that object's + `customProperties` when it has them, otherwise it is dropped, each with a + warning, so the document validates against the version it declares. +- **Bitol ODPS — bidirectional provider** (`BitolOdpsProvider`, registered + as `odps_bitol`). Export emits the canonical fragments layout: 1 ODPS doc + + N sibling `.odcs.yaml` files; each per-port ODCS `id` + follows the port's `contractId` (UUIDs included), minting + `{productId}.{port}` only when a port has none. Bundle import keeps each + port's complete ODCS state, so `fluid opds import` then `fluid opds + export` reproduces the product and every per-port ODCS document (servers, + team, SLA, top-level fields, per-document `apiVersion`) under strict + validation, schema-less per-port contracts included. Same-name ports that + differ only in version import as distinct exposes. Fresh output ports + carry the `customProperties` (server, environment, containsPii, + autoApprove, status) and the product the `type` / `domain` / + `fluidVersion` properties the 0.7.8 ODPS exporter emitted, and honour + `expose.contract_id` and `ODPS_DEFAULT_PORT_VERSION`. Output-port + `contractId`s are minted as `{productId}.{port}` (the id of the sibling + per-port ODCS), also by the legacy `OdpsStandardProvider` / `fluid + odps-bitol export`, which used `{exposeId}_contract` in 0.7.8; set + `contract_id` on an expose to keep a specific id. +- **Three import entry points** — single ODPS file, directory (ODPS + + sibling ODCS, or just ODCS files), or a lone ODCS file — via + `fluid opds import `. Inputs are schema-validated; the output is a + FLUID contract skeleton that may need edits (builds, bindings) before + `fluid validate` passes. An ODCS-only directory import keeps each + contract's id and document-level fields. `--lenient` downgrades + output-port resolution failures, malformed `contractId`s and + schema-invalid ODCS inputs to warnings (input ports are always lenient). + `-o FILE` picks the format from the suffix unless `-f` is given. FLUID + contracts passed as input are rejected with a clear error rather than + mistaken for ODPS. +- **`ContractResolver`** — resolves Bitol ODPS port `contractId` + references to ODCS documents through local-file probes and http(s) fetch, + with caching, JSON Schema validation, HTML-with-200 refusal, and + `--no-remote` hermetic mode. A local file is used only if its `id` + matches the requested contractId. +- **`fluid opds validate|info --spec bitol-1.0.0|bitol-1.1.0|odpi-4.1`.** + Without `--spec`, validate auto-detects Bitol ODPS documents (apiVersion + + `kind: DataProduct`) and validates them against the schema matching + their own apiVersion; any other document is validated as ODPI v4.1 as + before. YAML and JSON input are both accepted. +- **`fluid forge --seed-from `** — copilot accepts an ODCS contract, + a Bitol ODPS product file, or a directory bundle as a structural seed. + The seed's schema/qos are ground truth the LLM must not mutate; per-field + ODCS quality is not carried into FLUID, and the LLM fills in builds, + execution and the expose kind/binding/semantics. The ground truth is a + FLUID 0.7.2-valid view (round-trip bookkeeping and ODCS-only field keys + removed; each `exposeId` normalised to a FLUID identifier, e.g. + `Daily Orders` → `daily_orders`) used for both the prompt and the + `diff_against_seed()` guard, so a model that copies the seed verbatim + passes strict validation. FLUID contracts, unresolved output-port + contracts and seeds with no ground truth fail with a clear error; a run + that fails only on ground-truth violations reports + `failure_class: seed_mismatch`. +- **`OdcsProvider.roundtrip_check(odcs)`** — public diff primitive + returning `{equal, missing, extra, changed}`; `OdcsProvider.render(..., + contract_id=...)` and `OdcsProvider.import_contract(..., lenient=True)` + keyword options; `OdcsProvider.to_model()` — typed view via the + `open-data-contract-standard` pydantic model (Python 3.10+). +- `tests/fixtures/upstream/` — verbatim Bitol example documents (Apache-2.0) + used as gold-standard fixtures, and `tests/test_spec_version_matrix.py` + asserting lossless round-trip, version fidelity and correct schema + selection across every fixture. +- `scripts/dmm_live_e2e.sh` — opt-in, manual end-to-end check against a + real Entropy Data CE (Data Mesh Manager) instance: brings the CE compose + stack up, publishes the multi-port fixture through both + `fluid publish --catalog datamesh-manager` and `fluid dmm publish + --with-contract`, at ODCS v3.1.0 and v3.2.0, and asserts via the REST API + that the product and one contract per port landed with the expected + `apiVersion`. Runs `fluid` under an isolated `HOME` so the user's config + is untouched. Not run by CI (needs Docker and a DMM API key). +- New docs: `docs/BITOL_INTEROP.md` covering canonical-model architecture, + the `contractId` convention, resolver behaviour, and the forge + `--seed-from` flow. + +### Deprecated +- `fluid odps export --version 4.1` — use `--spec odpi-4.1` (warning logged). +- `fluid odps-bitol export|validate|info` — use `fluid opds … --spec + bitol-1.0.0`. A deprecation notice is printed on every invocation; kept + working for one release. +- `CatalogAsset`, `BaseCatalogProvider.map_contract_to_asset`, + `validate_asset`, `publish_asset` / `update_asset`, and calling + `publish(asset)` / `update(asset)` with a `CatalogAsset`. These keep + working for one release with a `DeprecationWarning` and publish the + asset's own fields (including edits made after `map_contract_to_asset`, + e.g. `sensitivity` → `is_public`), in positional and keyword form + (`publish(asset=...)`). Third-party catalog providers written against the + 0.7.8 API keep working through `fluid publish`, and `map_contract_to_asset` + / `validate_asset` overrides are honoured, including on subclasses of the + built-in providers, where they reach the published payload. Migrate to + `publish(fluid, CatalogTarget.from_fluid(fluid))`. +- `verify(asset_id=...)` / `delete(asset_id=...)` on catalog providers: the + parameter is now `contract_id`; `asset_id=` keeps working for one release + with a `DeprecationWarning`. +- `fluid_build.providers.odcs.mappers.types.physical_type_to_platform` is + no longer used by the importer and is kept only for API compatibility. + +### Fixed +- ODCS/ODPS round-trip lossiness surfaced by real upstream documents: + explicit `required: false` / `primaryKey: false` and empty `tags: []` were + dropped (800+ diffs on AdventureWorks); property-level `relationships` and + `transformSourceObjects` / `transformLogic` / `transformDescription` were + never passed through; `logicalType: map` was coerced to `string`; + re-export synthesised `schema[].logicalType`, a top-level `name`, and a + `servers[]` block for imported documents that had none; `apiVersion` was + not preserved on re-export; SchemaObjects without `properties` were + dropped. +- FLUID `number`, `float64`, `int64`, `smallint`, `timestamp_ntz` and + similar column types exported as ODCS `logicalType: string`. +- ODCS export dropped `name`, `description` and `domain` for schema-valid + FLUID contracts that set them at the top level. +- `OdpsStandardProvider.render` (the ODPS path behind `fluid dmm publish`) + raised `KeyError: 'id'` on exposes that use the FLUID 0.7.x `exposeId` + key, i.e. every shipped example contract. +- `fluid opds export` (ODPI stdout path) printed the payload through Rich, + wrapping lines at 80 columns and stripping `[bracket]` text; JSON/YAML + payloads now go to stdout raw. +- `FluidConfig` leaked file config into the module-level defaults across + instances. +- `tests/fixtures/odcs/contract-full.yaml` (relationship `from` must be + `table.column`) and `contract-minimal.yaml` (a `local` server requires + `format`) were invalid against the version they declare. + +### Security +- `FLUID_API_KEY` / `FLUID_CC_ENDPOINT` environment overrides are no longer + written into the stored config, so `save_user_config()` cannot persist + them to disk. +- ODPS `contractId`s that are absolute or contain path separators / `..` + are never used as local file names, symlinks that resolve outside a bundle + are ignored by `fluid opds import` and `fluid forge --seed-from`, and + bundle export refuses product/contract ids that would write outside + `--out-dir` (or next to `--out FILE`). +- Remote `contractId` fetches are size-capped (10 MiB), time-boxed (30 s + total, covering connect, slow response headers and chunked bodies) and + follow only http(s) redirects. +- YAML contracts (local ODCS/ODPS files and remote `contractId` fetches) are + refused when their aliases would expand beyond 1,000,000 nodes or are + recursive ("billion laughs"); JSON-Schema validation no longer duplicates + aliased subtrees in memory. +- `scripts/dmm_live_e2e.sh` uses a private `mktemp -d` work dir and passes + the API key through a 0600 header file instead of the curl command line. + ## [0.7.8] — 2026-04-03 ### Changed diff --git a/README.md b/README.md index da166f90..98fcc5b9 100644 --- a/README.md +++ b/README.md @@ -337,7 +337,38 @@ Providers are the bridge between your declarative contract and your target execu | 🌩️ **aws** | Amazon Web Services | S3, Glue, Athena, Redshift, MWAA, IAM. | | ❄️ **snowflake** | Snowflake | Databases, schemas, streams, tasks, RBAC, sharing. | -> Export-only providers for open data standards: **odps**, **odcs**, **datamesh-manager**. +> Open-standard providers: **odcs** (Open Data Contract Standard v3.1.0 / v3.2.0, bidirectional), **odps_bitol** (Bitol Open Data Product Standard v1.0.0 / v1.1.0, bidirectional), **odps** (ODPI v4.1, export-only), **datamesh-manager**. + +### Which ODPS? — disambiguation + +| Standard | Provider | CLI | What it is | +|---|---|---|---| +| **ODCS v3.1.0 / v3.2.0** | `odcs` | `fluid odcs ...` | Per-dataset contract (schema/quality/SLA). Bidirectional, lossless. Imports validate against the version the document declares; exports default to v3.1.0 (`ODCS_API_VERSION=v3.2.0` to opt in). Bitol.io. | +| **Bitol ODPS v1.0.0 / v1.1.0** | `odps_bitol` | `fluid opds --spec bitol-1.0.0` (default) / `bitol-1.1.0` | Data product wrapper referencing ODCS by `contractId`. Bidirectional, lossless. Imports accept either version. | +| **ODPI v4.1** | `odps` | `fluid opds --spec odpi-4.1` | Open Data Product Initiative v4.1 (Linux Foundation). Export-only single JSON. | + +```bash +# Export FLUID → Bitol bundle (1 ODPS doc + N sibling ODCS contracts): +# ./out/.odps.yaml + ./out/..odcs.yaml +fluid opds export contract.fluid.yaml --spec bitol-1.0.0 --out-dir ./out + +# Validate the product doc (Bitol vs ODPI is auto-detected; YAML or JSON) +fluid opds validate ./out/.odps.yaml + +# Import any of: ODPS file, directory bundle, or lone ODCS file. +# The result is a FLUID skeleton — complete fluidVersion/kind/id/name, +# expose kinds and bindings before `fluid validate` / `fluid plan`. +fluid opds import ./out/.odps.yaml -o product.fluid.yaml +fluid opds import ./out/ -o product.fluid.yaml +fluid opds import ./out/..odcs.yaml -o port.fluid.yaml + +# Seed the forge copilot with an existing standards doc: its schema and qos are ground +# truth (per-field ODCS quality is not carried over); the LLM adds builds/execution +# and each expose's kind/binding/semantics +fluid forge --seed-from existing-contract.odcs.yaml --context '{"goal":"add daily refresh"}' +``` + +See [`docs/BITOL_INTEROP.md`](docs/BITOL_INTEROP.md) for the full architecture, `contractId` convention, resolver behaviour, and round-trip guarantees. --- diff --git a/docs/BITOL_INTEROP.md b/docs/BITOL_INTEROP.md new file mode 100644 index 00000000..07fa54dc --- /dev/null +++ b/docs/BITOL_INTEROP.md @@ -0,0 +1,159 @@ +# Bitol Interoperability + +FLUID plugs into the wider Bitol data ecosystem through two open standards. + +## The three standards under one roof + +| Standard | What it describes | Authority | FLUID provider | CLI selector | +|---|---|---|---|---| +| **ODCS v3.1.0 / v3.2.0** | Per-dataset contract (schema, quality, SLA, governance) | [bitol-io/open-data-contract-standard](https://github.com/bitol-io/open-data-contract-standard) | `odcs` | `fluid odcs ...` | +| **Bitol ODPS v1.0.0 / v1.1.0** | Data product wrapper referencing ODCS contracts via `contractId` | [bitol-io/open-data-product-standard](https://github.com/bitol-io/open-data-product-standard) | `odps_bitol` | `fluid opds --spec bitol-1.0.0` (default) / `bitol-1.1.0` | +| **ODPI v4.1** | Open Data Product Initiative v4.1 (Linux Foundation) — single JSON document | [Open-Data-Product-Initiative/v4.1](https://github.com/Open-Data-Product-Initiative/v4.1) | `odps` | `fluid opds --spec odpi-4.1` | + +**Bitol ODPS** and **ODPI v4.1** are unrelated despite sharing the "ODPS" three-letter slug. Phase 5 of this work made the disambiguation explicit in every layer (provider names, CLI flag, docstrings, features.yaml). + +## Canonical-model architecture + +FLUID is the canonical hub. Every export goes FLUID → standard; every import goes standard → FLUID. The Bitol ODPS provider delegates per-port ODCS rendering to the ODCS provider — there's exactly one place that knows how to translate schema/quality/SLA across the boundary. + +``` + FLUID (canonical) + ↓ + ┌───────────────┼───────────────┐ + ↓ ↓ ↓ + OdcsProvider BitolOdpsProvider OdpsProvider + │ │ ↓ │ + │ │ └── per-port ODCS + │ │ (via OdcsProvider) + ↓ ↓ ↓ + *.odcs.yaml 1 ODPS doc single + + N *.odcs.yaml ODPI-v4.1 JSON +``` + +## Export pattern (Bitol ODPS) + +One FLUID contract with N output ports always emits: + +- 1 `.odps.yaml` — the Bitol ODPS product doc. +- N `.odcs.yaml` siblings — one per output port, referenced by `contractId`. + +`contractId` follows the convention `{productId}.{portName}` and matches the `id` of the corresponding ODCS contract exactly. This linking invariant is enforced inside the provider (`BitolOdpsProvider.render` raises `ProviderError` on mismatch) and asserted in tests. + +```bash +fluid opds export contract.fluid.yaml --spec bitol-1.0.0 --out-dir ./out +# ./out/.odps.yaml +# ./out/..odcs.yaml +# ./out/..odcs.yaml + +# --out FILE writes the product doc to FILE and the per-port ODCS contracts +# next to it; the format follows the suffix unless -f/--format is given. +# Use a different directory than --out-dir: a directory import accepts only +# one ODPS doc. +fluid opds export contract.fluid.yaml --out ./single/product.json +# ./single/product.json +# ./single/..odcs.json +# ./single/..odcs.json + +# No --out / --out-dir: the product doc is printed to stdout as JSON. +fluid opds export contract.fluid.yaml > product.json +``` + +With `--spec` omitted the emitted `apiVersion` follows `ODPS_API_VERSION`, else the imported document's own `apiVersion` (re-exports of imports), else `v1.0.0`; an explicit `--spec bitol-1.0.0` / `bitol-1.1.0` pins it. Exports are validated against the vendored schemas; `--no-validate-strict` still validates but reports problems as warnings on stderr instead of failing. + +`--out-dir` and `-f yaml` apply to the Bitol specs only; `--spec odpi-4.1` always writes a single JSON document to `--out` (or stdout) and rejects `--out-dir` and `-f yaml` with an error (`-f json` is accepted). + +No inline-embedding mode. The fragments layout is the only export pattern — it matches Bitol convention and `datacontract-cli` expectations. + +## Import — three entry points, one FLUID skeleton + +All three converge on the same FLUID skeleton. The input documents are schema-validated, but the output is a **skeleton, not yet a schema-valid FLUID contract**: add `fluidVersion` / `kind` / `id` / `name`, each expose's `kind`, and binding `format` / `location` (and map the platform to a supported provider) before `fluid validate` / `plan` / `apply` will accept it. The `*_passthrough` keys it carries are what make re-export lossless. + +```bash +# 1. Single ODPS file — resolver finds sibling ODCS by contractId. +fluid opds import out/.odps.yaml -o product.fluid.yaml + +# 2. Directory — auto-discovers the ODPS doc + all ODCS files inside. +# Also handles ODCS-only directories (no ODPS doc → multi-expose FLUID +# with no product wrapper, warning emitted). +fluid opds import out/ -o product.fluid.yaml + +# 3. Lone ODCS file — produces a single-expose FLUID. +fluid opds import out/..odcs.yaml -o port1.fluid.yaml +``` + +With `-o` the format follows the file suffix (`.json` → JSON, anything else → YAML) unless `-f/--format` is given. Without `-o` the FLUID document is written to stdout (YAML, or JSON with `-f json`). + +Flags: + +- `--no-remote` — refuse http(s) `contractId` resolution (hermetic CI). +- `--lenient` — downgrade output-port resolution failures to warnings. Input ports are always lenient because their contracts often live in other repos / registries. + +## Validate + +```bash +fluid opds validate out/.odps.yaml # auto-detected as Bitol +fluid opds validate out/.odps.yaml --spec bitol-1.1.0 # pin the schema version + +fluid opds export contract.fluid.yaml --spec odpi-4.1 --out odpi.json +fluid opds validate odpi.json --spec odpi-4.1 # ODPI v4.1 (also: --version 4.1) +``` + +YAML and JSON are both accepted. With `--spec` omitted, a document declaring `apiVersion` and `kind: DataProduct` is validated as Bitol ODPS against the vendored schema matching its own `apiVersion`; anything else is validated as ODPI v4.1, exactly as before. `fluid opds info --spec ` shows the details of one specification. + +## The `contractId` resolver + +`fluid_build.providers.odps_standard.resolver.ContractResolver` resolves `contractId` references with this priority: + +1. **Explicit hint** — when a caller passes a file path or URL alongside the id. +2. **Pre-indexed file** — files passed in via `additional_files=` or discovered by `index_directory()`. The directory import path uses this so local files always beat anything else. +3. **Local file probes** — across the cross-product of `{, contracts/}` × `{, }` × `{.odcs.yaml, .odcs.yml, .odcs.json, .yaml}` (16 candidates for a dotted id). Other layouts (an `odcs/` subdirectory, plain `.yml` / `.json` names) are found by directory import (`fluid opds import /`), which indexes every `.yaml` / `.yml` / `.json` file under the directory by its ODCS `id`. +4. **HTTP(S) URL** — when the `contractId` itself is a URL and `allow_remote=True`. Refuses HTML-with-200 responses and non-http(s) schemes. + +Every resolved document is validated against the vendored ODCS JSON Schema matching its own `apiVersion` (v3.2.0 and v3.1.0 are vendored; older versions are checked against the newest, which is a strict superset); `ContractValidationError` raises with the offending field name. + +## Round-trip guarantees + +- **ODCS → FLUID → ODCS** is structurally lossless. Pass-through namespace (`metadata.odcs_passthrough.*`, `expose.odcs_passthrough.*`, `field.odcs_passthrough.*`) preserves every section the FLUID model doesn't natively represent. +- **FLUID → Bitol ODPS → FLUID → Bitol ODPS** reproduces the product doc and every per-port ODCS contract zero-diff (tested in `tests/test_odps_bitol_import.py::TestBundleRoundTrip`). +- `OdcsProvider.roundtrip_check(odcs)` is a public primitive returning a structured `{equal, missing, extra, changed}` diff — used by tests and the forge ground-truth guard. + +## `fluid forge --seed-from` — standards as structural seed + +The copilot accepts the same three input shapes as `opds import` as a **structural seed**: + +```bash +# Lone ODCS — single-expose FLUID skeleton +fluid forge --seed-from existing-contract.odcs.yaml --context '{"goal":"add daily refresh"}' + +# Single Bitol ODPS file — multi-expose skeleton with sibling ODCS resolved +fluid forge --seed-from product.odps.yaml + +# Directory bundle — same as above; resolves sibling ODCS files by contractId +fluid forge --seed-from ./contracts/ +``` + +Each expose's schema and qos from the seed are **ground truth** the LLM must not modify, and every seed expose must be kept, in order, under its `exposeId`. They are presented to the LLM as valid FLUID 0.7.2: an ODCS schema object or ODPS port name that is not a FLUID identifier is normalised deterministically (lower-cased, other characters replaced by `_`, so `Daily Orders` becomes `daily_orders`), and ODCS-only keys such as per-field `quality` rules are **not carried** into the FLUID output; add them to the generated contract yourself if you need them. The pre-processor (`fluid_build.cli.forge_copilot_seed.load_seed`) emits a `SeedResult` carrying: + +- `fluid` — the imported skeleton the LLM augments. +- `provenance` — every contract source consumed (paths + URLs). +- `ground_truth_paths` — dotted paths inside `fluid` that must round-trip. +- `diff_against_seed(seed, candidate)` — primitive returning `{path, seed, candidate}` mismatches whenever the candidate mutated a ground-truth value (schema, qos or an `exposeId`). + +The LLM fills in builds (with their execution) and each expose's kind, binding and semantics — the things the standards don't model. A directory seed reads only files inside the directory: symlinks that resolve outside it are ignored. + +## File map + +| Concern | Lives in | +|---|---| +| ODCS provider orchestrator | `fluid_build/providers/odcs/provider.py` | +| ODCS section mappers (1 per section) | `fluid_build/providers/odcs/mappers/*.py` | +| ODCS validation + round-trip diff | `fluid_build/providers/odcs/validation.py` | +| Bitol ODPS provider orchestrator | `fluid_build/providers/odps_standard/provider.py` | +| Bitol ODPS section mappers | `fluid_build/providers/odps_standard/mappers/*.py` | +| Bitol ODPS schemas | `fluid_build/providers/odps_standard/schemas/odps-product-v1.1.0.json`, `…-v1.0.0.json` | +| ODCS schemas | `fluid_build/providers/odcs/odcs-schema-v3.2.0.json`, `…-v3.1.0.json` | +| Upstream example fixtures (gold standard) | `tests/fixtures/upstream/{odcs,odps}/` — verbatim Bitol examples, exercised by `tests/test_spec_version_matrix.py` | +| `contractId` resolver | `fluid_build/providers/odps_standard/resolver.py` | +| CLI dispatcher | `fluid_build/cli/opds.py` | +| Forge `--seed-from` pre-processor | `fluid_build/cli/forge_copilot_seed.py` | +| Tests | `tests/test_odcs_roundtrip.py`, `tests/test_odps_bitol_export.py`, `tests/test_odps_bitol_import.py`, `tests/test_odps_resolver.py`, `tests/test_opds_cli_spec.py`, `tests/test_forge_copilot_seed.py` | diff --git a/docs/DATA_MESH_MANAGER_PUBLISHING.md b/docs/DATA_MESH_MANAGER_PUBLISHING.md index f84a0d1c..6367cefa 100644 --- a/docs/DATA_MESH_MANAGER_PUBLISHING.md +++ b/docs/DATA_MESH_MANAGER_PUBLISHING.md @@ -62,8 +62,9 @@ fluid datamesh-manager publish contract.fluid.yaml Registers (or updates) the data product in Entropy Data. Output ports are included in the registration but **no data contracts are created yet** — the -ports show a `dataContractId` field so they're ready to be linked once contracts -are published. +ports carry a `contractId` field (`dataContractId` in the +[legacy DPS shape](#legacy-data-product-specification-001)) so they're ready to +be linked once contracts are published. --- @@ -76,9 +77,9 @@ fluid datamesh-manager publish contract.fluid.yaml --with-contract This is the full publish. It: 1. Registers the data product via `PUT /api/dataproducts/{id}` -2. For every expose port, generates an ODCS v3.1.0 contract and sends it via +2. For every expose port, generates an ODCS contract (v3.1.0 by default; set `ODCS_API_VERSION=v3.2.0` to emit v3.2.0) and sends it via `PUT /api/datacontracts/{productId}.{exposeId}` -3. Each output port's `dataContractId` field in the product record already +3. Each output port's `contractId` field in the product record already points to its contract — Entropy Data automatically links them. --- @@ -158,7 +159,7 @@ fluid dmm publish contract.fluid.yaml --dry-run ### ODCS contract (per expose port) -Each expose in your FLUID contract becomes one ODCS v3.1.0 document. The +Each expose in your FLUID contract becomes one ODCS document (v3.1.0 by default, v3.2.0 on request). The mapping is straightforward: | FLUID field | ODCS field | Notes | @@ -179,25 +180,63 @@ independently at its own URL: https://api.entropy-data.com/api/datacontracts/{productId}.{exposeId} ``` -### ODPS data product record +### ODPS data product record (default) -The top-level FLUID contract maps to Entropy Data's ODPS-aligned data product -shape: +By default the top-level FLUID contract is sent as a **Bitol Open Data Product +Standard (ODPS)** document — the only data product type current Entropy Data / +Entropy CE organisations accept: | FLUID field | ODPS / Entropy Data field | |---|---| -| `id` | `info.id` | -| `name` | `info.name` | -| `description` | `info.description` | -| `domain` | `info.domain` | -| `metadata.owner` | `teamId` | -| `exposes[]` | `outputPorts[]` (each with `dataContractId`) | +| — | `apiVersion: v1.0.0`, `kind: DataProduct` | +| `id` | `id` | +| `name` | `name` | +| `description` | `description.purpose` | +| `domain` | `domain` | +| `metadata.owner` | `team` (`name` + `members[]`); `--team-id` overrides `team.name` | +| `exposes[]` | `outputPorts[]` (each with `contractId`) | | `expects[]` | `inputPorts[]` | -The `dataContractId` on every output port follows the pattern +The `contractId` on every output port follows the pattern `{productId}.{exposeId}`, which is exactly the URL path used when the ODCS contracts are PUT — so the link is automatic. +### Legacy Data Product Specification (0.0.1) + +Older Entropy Data / Data Mesh Manager organisations that still expect the +legacy **Data Product Specification 0.0.1** shape can opt out of ODPS. The +legacy payload uses `dataProductSpecification: "0.0.1"`, a root `id`, +`info.title` / `info.owner`, `teamId`, and `outputPorts[].dataContractId` +(same `{productId}.{exposeId}` pattern). + +Opt out with any of: + +```bash +# per command +fluid dmm publish contract.fluid.yaml --data-product-spec 0.0.1 + +# for every publish in this shell / CI job (both fluid dmm publish and +# fluid publish --catalog datamesh-manager) +export DMM_DATA_PRODUCT_SPEC=0.0.1 +``` + +or, for `fluid publish --catalog datamesh-manager`, in the catalog config: + +```yaml +catalogs: + datamesh-manager: + endpoint: https://api.entropy-data.com + data_product_spec: "0.0.1" # or: provider_hint: dps +``` + +The specification is resolved in this order (first match wins): + +1. explicit `--data-product-spec` (or the catalog's `data_product_spec`) +2. provider hint: `odps` / `opds` → ODPS, `dps` → DPS 0.0.1 + (catalog `provider_hint`) +3. the `DMM_DATA_PRODUCT_SPEC` environment variable +4. default: ODPS + --- ## Why One Contract Per Expose Port? @@ -208,7 +247,7 @@ different schemas, different SLAs, and potentially different lifecycle states. Bundling them all into one contract would mean consumers can't selectively subscribe to or enforce contracts on individual ports. -ODCS v3.1.0 is designed as a per-dataset contract. Generating one document per +ODCS is designed as a per-dataset contract. Generating one document per expose port means: - Consumers can reference exactly the port they depend on @@ -263,5 +302,5 @@ fluid dmm publish contract.fluid.yaml --overlay overlays/$ENV.yaml \ After step 3, Entropy Data shows: - One data product entry with two output ports -- Two ODCS contracts, each linked to its port via `dataContractId` +- Two ODCS contracts, each linked to its port via `contractId` - The correct staging or production BigQuery locations in each `servers[]` block diff --git a/docs/index.html b/docs/index.html index 031a97b6..c6741e2c 100644 --- a/docs/index.html +++ b/docs/index.html @@ -708,7 +708,7 @@

Drift Detection

📦

Open Standards Export

-

Export to ODCS v3.1 (Bitol.io) and ODPS v4.1 (Linux Foundation) with bidirectional import. Publish to Data Mesh Manager catalogs.

+

Lossless, bidirectional ODCS v3.1/v3.2 and Bitol ODPS v1.0/v1.1 (Bitol.io), plus ODPI v4.1 export (Linux Foundation). Publish to Data Mesh Manager catalogs.

🏗️
diff --git a/fluid_build/build-manifest.yaml b/fluid_build/build-manifest.yaml index 6f67dda3..019a1572 100644 --- a/fluid_build/build-manifest.yaml +++ b/fluid_build/build-manifest.yaml @@ -15,7 +15,7 @@ metadata: # PyPI Publishing Strategy pypi: repository: "fluid-mono-pypi" - base_version: "0.7.7" + base_version: "0.7.8" # Version naming convention per build type # alpha: 0.7.1a{build_number} (e.g., 0.7.1a42) diff --git a/fluid_build/cli/datamesh_manager.py b/fluid_build/cli/datamesh_manager.py index 1af290a4..79429ab2 100644 --- a/fluid_build/cli/datamesh_manager.py +++ b/fluid_build/cli/datamesh_manager.py @@ -87,8 +87,10 @@ def add_parser(subparsers): pub.add_argument( "--data-product-spec", help=( - "Override dataProductSpecification value sent to Entropy Data " - "(e.g. 'odps' or '0.0.1')." + "Data product specification sent to Entropy Data. Default 'odps' " + "(Bitol Open Data Product Standard — the only type current Entropy " + "CE organisations accept); pass '0.0.1' (or 'dps') for the legacy Data " + "Product Specification. Also settable via DMM_DATA_PRODUCT_SPEC." ), ) pub.add_argument( diff --git a/fluid_build/cli/forge.py b/fluid_build/cli/forge.py index b4e8b32b..5c9ce09a 100644 --- a/fluid_build/cli/forge.py +++ b/fluid_build/cli/forge.py @@ -173,6 +173,7 @@ def _generate_copilot_artifacts_dependency( discovery_report: Any, project_memory: Any, capability_matrix: Any, + seed_options: Any = None, ) -> CopilotGenerationResult: return generate_copilot_artifacts( context, @@ -181,6 +182,7 @@ def _generate_copilot_artifacts_dependency( project_memory=project_memory, capability_matrix=capability_matrix, logger=LOG, + seed_options=seed_options, ) def _make_memory_store_dependency(self, project_root: Path) -> CopilotMemoryStore: @@ -282,6 +284,32 @@ def register(subparsers: argparse._SubParsersAction): "--discovery-path", help="Additional local file or directory path to scan for metadata-only discovery", ) + parser.add_argument( + "--seed-from", + dest="seed_from", + help=( + "Structural seed for the LLM. Accepts a Bitol ODPS file " + "(*.odps.yaml), a directory containing the ODPS doc + sibling " + "ODCS files (or only ODCS files), or a lone ODCS file " + "(*.odcs.yaml). Each expose's schema and qos from the seed are " + "treated as ground truth (exposeIds are kept, normalised to FLUID " + "identifiers); ODCS-only keys such as per-field quality rules are " + "not carried into the FLUID output. The LLM fills in builds (with " + "their execution) and each expose's kind, binding and semantics. " + "The copilot runtime enforces preservation via a ground-truth diff " + "after each attempt — any mutation triggers the existing 3-attempt " + "repair loop with a mismatch report." + ), + ) + parser.add_argument( + "--seed-no-remote", + dest="seed_no_remote", + action="store_true", + help=( + "When --seed-from has http(s) contractId references, refuse to " + "fetch them. Honoured end-to-end (pre-processor + runtime)." + ), + ) parser.add_argument( "--memory", dest="memory", diff --git a/fluid_build/cli/forge_copilot_agent.py b/fluid_build/cli/forge_copilot_agent.py index bdacb138..36a92cf1 100644 --- a/fluid_build/cli/forge_copilot_agent.py +++ b/fluid_build/cli/forge_copilot_agent.py @@ -200,6 +200,7 @@ def _generate_copilot_artifacts_dependency( discovery_report: Any, project_memory: Any, capability_matrix: Any, + seed_options: Any = None, ) -> CopilotGenerationResult: return generate_copilot_artifacts( context, @@ -208,6 +209,7 @@ def _generate_copilot_artifacts_dependency( project_memory=project_memory, capability_matrix=capability_matrix, logger=LOG, + seed_options=seed_options, ) def _make_memory_store_dependency(self, project_root: Path) -> CopilotMemoryStore: @@ -372,12 +374,14 @@ def generate_project_artifacts( """Generate and validate copilot artifacts via the LLM runtime.""" context = normalize_copilot_context(context) runtime_inputs = self.prepare_runtime_inputs(copilot_options) + seed_options = (copilot_options or {}).get("seed_options") return self._generate_copilot_artifacts_dependency( context, llm_config=runtime_inputs["llm_config"], discovery_report=runtime_inputs["discovery_report"], project_memory=runtime_inputs["project_memory"], capability_matrix=runtime_inputs["capability_matrix"], + seed_options=seed_options, ) def create_project( diff --git a/fluid_build/cli/forge_copilot_prompts.py b/fluid_build/cli/forge_copilot_prompts.py index f8cd3fce..d69ab9f4 100644 --- a/fluid_build/cli/forge_copilot_prompts.py +++ b/fluid_build/cli/forge_copilot_prompts.py @@ -33,11 +33,37 @@ def build_system_prompt( - capability_matrix: Mapping[str, Any], known_build_engines: Sequence[str] + capability_matrix: Mapping[str, Any], + known_build_engines: Sequence[str], + *, + has_seed_ground_truth: bool = False, ) -> str: - """System prompt for structured FLUID contract generation.""" + """System prompt for structured FLUID contract generation. + + When ``has_seed_ground_truth`` is True (``--seed-from`` in play), the + prompt gains a ground-truth-preservation clause that tells the LLM which + parts of its output must match the seed byte-for-byte. + """ providers = ", ".join(capability_matrix.get("providers") or []) engines = ", ".join(capability_matrix.get("build_engines") or list(known_build_engines)) + seed_clause = ( + "\nGROUND TRUTH FROM SEED (--seed-from is in play):\n" + "The user prompt carries a ``seed_ground_truth`` block with the exposes, " + "contract.schema (per property), and qos values pinned by an upstream Bitol " + "ODCS/ODPS contract, already expressed as valid FLUID 0.7.2. These fields are " + "AUTHORITATIVE — you MUST preserve them VERBATIM in your output. Do not add, " + "remove, rename, retype, or reorder any schema property. Do not change qos " + "values. Keep every seed expose, in the same order. Use the seed's top-level " + "``id``, ``name``, ``description``, ``domain``, ``tags`` and ``metadata.owner`` " + "when present. You may ONLY fill in `builds` (each with its `execution`), the " + "top-level fields the seed leaves out (such as `metadata.layer`), and each " + "expose's `kind`, `binding` and `semantics`. Each expose's ``exposeId``, " + "``contract.schema``, and ``qos`` must round-trip identically. If the user " + "prompt also includes ``ground_truth_paths``, those are the exact JSON paths " + "the runtime will diff against; a mismatch triggers a repair re-prompt.\n" + if has_seed_ground_truth + else "" + ) return ( "You are FLUID Forge Copilot. Generate a production-ready FLUID 0.7.2 contract and README " "that only use locally supported templates, providers, and build engines.\n" @@ -85,7 +111,7 @@ def build_system_prompt( "The semantics block enables AI agents and BI tools to generate correct queries without hallucination.\n\n" "Follow the seed_contract structure exactly as a reference for the correct schema shape.\n" f"Allowed providers: {providers}.\n" - "Only use build engines from the provided capability matrix." + "Only use build engines from the provided capability matrix." + seed_clause ) @@ -172,8 +198,15 @@ def build_user_prompt( previous_errors: Sequence[str], previous_payload: Optional[Mapping[str, Any]], project_memory: Optional[CopilotMemorySnapshot] = None, + seed_ground_truth: Optional[Mapping[str, Any]] = None, ) -> str: - """Build the attempt-specific user prompt.""" + """Build the attempt-specific user prompt. + + ``seed_ground_truth`` (when set) is the pre-formatted block returned by + :func:`fluid_build.cli.forge_copilot_seed.format_ground_truth_prompt_block`. + It carries the exposes/schema/qos the LLM must preserve verbatim, plus + the dotted paths the runtime will diff against. + """ interview_summary = _normalize_interview_summary(context) prompt: dict[str, Any] = { "attempt": attempt_index, @@ -190,6 +223,8 @@ def build_user_prompt( "prefer_manual_trigger_for_execute_compatibility": True, }, } + if seed_ground_truth: + prompt["seed_ground_truth"] = dict(seed_ground_truth) if project_memory: prompt["project_memory"] = project_memory.to_prompt_payload() if previous_errors: diff --git a/fluid_build/cli/forge_copilot_runtime.py b/fluid_build/cli/forge_copilot_runtime.py index 69493a52..065e2c8f 100644 --- a/fluid_build/cli/forge_copilot_runtime.py +++ b/fluid_build/cli/forge_copilot_runtime.py @@ -22,7 +22,7 @@ from __future__ import annotations import logging -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace from typing import Any, Dict, List, Mapping, Optional, Sequence from fluid_build.cli._common import redact_secrets, resolve_provider_from_contract @@ -104,6 +104,19 @@ from fluid_build.cli.forge_copilot_schema_inference import ( map_inferred_type_to_contract_type as _map_inferred_type_to_contract_type, ) + +# Optional Phase-7 seed integration — enables --seed-from's ground-truth +# preservation + repair loop. Import late-bound below so a missing seed +# module never breaks the runtime. +from fluid_build.cli.forge_copilot_seed import ( # noqa: F401 + SeedOptions, + SeedResult, + diff_against_seed, + format_ground_truth_prompt_block, + format_mismatch_report, + is_ground_truth_error, + load_seed, +) from fluid_build.schema_manager import FluidSchemaManager from fluid_build.util.contract import get_builds @@ -159,9 +172,20 @@ class CopilotGenerationResult: # --------------------------------------------------------------------------- -def build_system_prompt(capability_matrix: Mapping[str, Any]) -> str: - """Build the system prompt, injecting the known build engines list.""" - return _build_system_prompt_raw(capability_matrix, sorted(KNOWN_BUILD_ENGINES)) +def build_system_prompt( + capability_matrix: Mapping[str, Any], *, has_seed_ground_truth: bool = False +) -> str: + """Build the system prompt, injecting the known build engines list. + + ``has_seed_ground_truth`` forwards to the underlying prompt builder so + the Phase-7 preservation clause is included when the operator passed + ``--seed-from``. + """ + return _build_system_prompt_raw( + capability_matrix, + sorted(KNOWN_BUILD_ENGINES), + has_seed_ground_truth=has_seed_ground_truth, + ) def build_seed_contract( @@ -358,8 +382,22 @@ def generate_copilot_artifacts( capability_matrix: Optional[Mapping[str, Any]] = None, logger: Optional[logging.Logger] = None, max_attempts: int = 3, + seed_options: Optional[SeedOptions] = None, ) -> CopilotGenerationResult: - """Generate and validate copilot artifacts with a repair loop.""" + """Generate and validate copilot artifacts with a repair loop. + + When ``seed_options`` is set (from ``--seed-from``), the loop: + 1. Loads the seed once via :func:`load_seed`. + 2. Injects a ``seed_ground_truth`` block into every attempt's user + prompt + a preservation clause into the system prompt. + 3. After each attempt's normal validation passes, runs + :func:`diff_against_seed` on the generated contract. Any mutation + to a ground-truth path becomes an additional validation error, + which triggers the existing repair-loop re-prompt. + 4. The final attempt's ground-truth mismatches surface in the + :class:`CopilotGenerationError` context so operators see exactly + what the LLM refused to preserve. + """ capabilities = dict(capability_matrix or build_capability_matrix()) provider_adapter = get_llm_provider(llm_config.provider) scaffold_decision = _build_scaffold_decision( @@ -378,12 +416,33 @@ def generate_copilot_artifacts( project_memory=project_memory, ) + # Phase 7: load the ground-truth seed once, up-front. Failures here are + # fatal (unlike per-attempt LLM issues) because the operator explicitly + # asked for --seed-from and a broken seed can't be repaired by the LLM. + seed_result: Optional[SeedResult] = None + seed_ground_truth_prompt: Optional[Dict[str, Any]] = None + if seed_options is not None: + seed_result = load_seed(seed_options.seed_from, allow_remote=seed_options.allow_remote) + seed_ground_truth_prompt = format_ground_truth_prompt_block(seed_result) + if logger is not None: + logger.info( + "forge_copilot_seed_loaded", + extra={ + "seed_from": str(seed_options.seed_from), + "shape": seed_result.shape, + "exposes": len(seed_result.fluid.get("exposes") or []), + "ground_truth_paths": len(seed_result.ground_truth_paths), + }, + ) + attempts: List[GenerationAttemptReport] = [] previous_errors: List[str] = [] previous_payload: Optional[Dict[str, Any]] = None for attempt_index in range(1, max_attempts + 1): - system_prompt = build_system_prompt(capabilities) + system_prompt = build_system_prompt( + capabilities, has_seed_ground_truth=seed_result is not None + ) user_prompt = build_user_prompt( context=context, discovery_report=discovery_report, @@ -395,6 +454,7 @@ def generate_copilot_artifacts( previous_errors=previous_errors, previous_payload=previous_payload, project_memory=project_memory, + seed_ground_truth=seed_ground_truth_prompt, ) report = GenerationAttemptReport( @@ -429,6 +489,18 @@ def generate_copilot_artifacts( report.validation_errors = validation_errors report.validation_warnings = validation_warnings + # Phase 7 ground-truth guard: only run when structural validation + # passed AND a seed is in play. Mutations become validation errors so + # the existing repair-loop machinery re-prompts with the mismatch + # report — no separate control flow needed. + gt_mismatches: List[Dict[str, Any]] = [] + if seed_result is not None and not validation_errors: + gt_mismatches = diff_against_seed(seed_result, normalized["contract"]) + if gt_mismatches: + mismatch_errors = format_mismatch_report(gt_mismatches) + validation_errors = list(mismatch_errors) + report.validation_errors = validation_errors + if not validation_errors: return CopilotGenerationResult( suggestions=normalized["suggestions"], @@ -453,7 +525,7 @@ def generate_copilot_artifacts( elif report.validation_errors: joined = "; ".join(report.validation_errors[:4]) attempt_summaries.append(f"Attempt {report.attempt}: validation failed - {joined}") - failure_class = classify_generation_failure(attempts) + failure_class = _classify_attempt_failures(attempts) raise CopilotGenerationError( "copilot_generation_failed", "Forge copilot could not produce a valid contract after 3 attempts.", @@ -467,6 +539,31 @@ def generate_copilot_artifacts( ) +SEED_MISMATCH_FAILURE_CLASS = "seed_mismatch" + + +def _classify_attempt_failures(attempts: Sequence[GenerationAttemptReport]) -> str: + """Classify a failed run, keeping ``--seed-from`` guard failures apart. + + Ground-truth violation strings embed seed JSON (column descriptions, + quality ``dimension``/``metric`` keys, ...), which would otherwise trip the + keyword classifier into ``ambiguous_intent`` and start a semantic + clarification round that cannot fix a seed mismatch. They are left out of + the keyword scan; a run that failed only on the guard is + ``seed_mismatch``. + """ + filtered: List[GenerationAttemptReport] = [] + saw_seed_mismatch = False + for report in attempts: + errors = [e for e in report.validation_errors if not is_ground_truth_error(e)] + if len(errors) != len(report.validation_errors): + saw_seed_mismatch = True + filtered.append(replace(report, validation_errors=errors)) + if saw_seed_mismatch and not any(r.parse_error or r.validation_errors for r in filtered): + return SEED_MISMATCH_FAILURE_CLASS + return classify_generation_failure(filtered) + + def suggest_scaffold( context: Mapping[str, Any], discovery_report: DiscoveryReport, diff --git a/fluid_build/cli/forge_copilot_seed.py b/fluid_build/cli/forge_copilot_seed.py new file mode 100644 index 00000000..ce4daac9 --- /dev/null +++ b/fluid_build/cli/forge_copilot_seed.py @@ -0,0 +1,805 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +"""Pre-processor for ``fluid forge --seed-from ``. + +Turns a user-supplied ODCS contract, Bitol ODPS product, or directory +bundle into a structural FLUID skeleton the copilot can use as ground truth. +The skeleton's per-expose schema and qos are the truth — the LLM is expected +to fill in builds (with their execution) and the non-ground-truth parts of +each expose (kind, binding, semantics) only. + +Returns a :class:`SeedResult` with: + - ``fluid``: the imported FLUID dict (multi-expose where applicable). + - ``shape``: ``"odcs-file" | "odps-file" | "directory" | "odcs-only-directory"``. + - ``provenance``: an audit trail of every resolved contract source + (paths/URLs) so downstream consumers (system prompt, ground-truth + guard) can trace what came from where. + - ``ground_truth_paths``: dotted paths into the FLUID dict whose values + the LLM must not mutate (filled in by Phase 7's runtime guard). + +The prompt block and the guard both work on a FLUID-0.7.2-valid *view* of +the seed (see :func:`format_ground_truth_prompt_block`), so a model that +copies the ground truth verbatim passes strict schema validation and the +guard alike. +""" + +from __future__ import annotations + +import json +import logging +import re +from collections.abc import Mapping +from dataclasses import dataclass, field +from functools import lru_cache +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple, Union + +from fluid_build.providers.base import ProviderError +from fluid_build.providers.odcs import OdcsProvider +from fluid_build.providers.odcs.io import read_input +from fluid_build.providers.odps_standard import BitolOdpsProvider +from fluid_build.providers.odps_standard.provider import _is_within +from fluid_build.providers.odps_standard.resolver import ContractNotFound, ContractResolver + +LOG = logging.getLogger(__name__) + + +@dataclass +class SeedResult: + """The outcome of a ``--seed-from`` invocation.""" + + fluid: Dict[str, Any] + shape: str + provenance: List[Dict[str, Any]] = field(default_factory=list) + ground_truth_paths: List[str] = field(default_factory=list) + + +@dataclass +class SeedOptions: + """Runtime options that steer the copilot's seed hand-off. + + Constructed once at the CLI boundary from ``args.seed_from`` / + ``args.seed_no_remote`` and threaded through the copilot pipeline into + :func:`generate_copilot_artifacts`. ``None`` means "no seed" and the + runtime behaves exactly as it did before Phase 7 landed. + """ + + seed_from: Union[str, Path] + allow_remote: bool = True + + @classmethod + def from_args(cls, args: Any) -> "Optional[SeedOptions]": + """Build ``SeedOptions`` from an argparse ``Namespace``. + + Returns ``None`` when ``--seed-from`` wasn't passed, so callers can + thread the result through unconditionally. + """ + seed_from = getattr(args, "seed_from", None) + if not seed_from: + return None + return cls( + seed_from=seed_from, + allow_remote=not bool(getattr(args, "seed_no_remote", False)), + ) + + +_PASSTHROUGH_KEYS = frozenset({"odcs_passthrough", "odps_passthrough"}) + + +def _strip_passthrough(value: Any) -> Any: + """Drop round-trip stash buckets (``*_passthrough``) recursively. + + The stash is lossless-export machinery — opaque bytes the LLM should + neither see nor be asked to reproduce. Ground truth is the FLUID-native + projection only. + """ + if isinstance(value, Mapping): + return {k: _strip_passthrough(v) for k, v in value.items() if k not in _PASSTHROUGH_KEYS} + if isinstance(value, list): + return [_strip_passthrough(v) for v in value] + return value + + +# ---- FLUID 0.7.2 ground-truth view ---------------------------------------- +# +# The ODCS/ODPS importers produce a FLUID-shaped dict that still carries +# Bitol-only keys: per-field ``quality`` (``[]`` when the source had none) and +# ``classification``, an expose ``id`` next to ``exposeId``, qos values that +# are not ISO-8601 (ODCS ``latency: 4`` becomes ``freshnessSLO: '4'``), label +# values that are not strings, and so on. The copilot validates its output +# against the strict FLUID 0.7.2 schema, so presenting those keys as +# AUTHORITATIVE ground truth made strict validation and the ground-truth guard +# block each other. Instead the prompt block and :func:`diff_against_seed` +# share one FLUID-0.7.2-valid view of the seed: keys the schema forbids are +# dropped (or mapped: ``classification`` becomes ``sensitivity`` when the +# value is a FLUID sensitivity level) and values that fail their sub-schema +# are left out. The raw values stay in ``seed.fluid`` (and its +# ``*_passthrough`` buckets), so ODCS export stays lossless. + +GROUND_TRUTH_FLUID_VERSION = "0.7.2" + +_ROOT_DEF_PREFIX = "_fluid_root_" + + +@lru_cache(maxsize=1) +def _schema_document() -> Dict[str, Any]: + """The FLUID 0.7.2 schema, re-rooted so every fragment is a ``$defs`` entry. + + Root-level ``properties`` are aliased as ``$defs/_fluid_root_`` so a + single ``{"$defs": ..., "$ref": pointer}`` wrapper validates any fragment + without the root's ``required``/``additionalProperties``. + """ + schema_path = ( + Path(__file__).resolve().parent.parent + / "schemas" + / f"fluid-schema-{GROUND_TRUTH_FLUID_VERSION}.json" + ) + with open(schema_path, encoding="utf-8") as fh: + root = json.load(fh) + defs = dict(root.get("$defs") or {}) + for key, sub in (root.get("properties") or {}).items(): + defs[f"{_ROOT_DEF_PREFIX}{key}"] = sub + return {"$schema": root.get("$schema"), "$defs": defs} + + +def _resolve_pointer(pointer: str) -> Any: + node: Any = _schema_document() + for part in pointer.lstrip("#/").split("/"): + node = node[part] + return node + + +@lru_cache(maxsize=None) +def _validator(pointer: str) -> Any: + import jsonschema + + doc = _schema_document() + cls = jsonschema.validators.validator_for(doc) + return cls( + {"$schema": doc["$schema"], "$defs": doc["$defs"], "$ref": pointer}, + format_checker=cls.FORMAT_CHECKER, + ) + + +def _is_valid(pointer: str, value: Any) -> bool: + return bool(_validator(pointer).is_valid(value)) + + +def _json_safe(value: Any) -> Any: + """Normalise YAML-native values (dates, datetimes, ...) to JSON-safe ones. + + Applied to both sides of the diff and to the prompt block, so a seed with + an unquoted ``2024-01-01`` neither crashes ``json.dumps`` nor mismatches + the string an LLM returns for it. + """ + if isinstance(value, Mapping): + return {str(k): _json_safe(v) for k, v in value.items()} + if isinstance(value, (list, tuple)): + return [_json_safe(v) for v in value] + if value is None or isinstance(value, (bool, int, float, str)): + return value + return str(value) + + +_DROP = object() + + +def _coerce_for_key(key: str, value: Any) -> Any: + """Map common near-misses onto their FLUID shape before validation.""" + if key == "tags" and isinstance(value, list): + kept: List[str] = [] + for tag in value: + if isinstance(tag, str) and _is_valid("#/$defs/tags/items", tag) and tag not in kept: + kept.append(tag) + if value and not kept: + return _DROP + return kept + if key == "labels" and isinstance(value, Mapping): + return {str(k): v if isinstance(v, str) else json.dumps(v) for k, v in value.items()} + return value + + +def _object_view(value: Any, pointer: str) -> Optional[Dict[str, Any]]: + """Keep only the keys of ``value`` the object schema at ``pointer`` + declares, and only when their value validates against that key's schema.""" + if not isinstance(value, Mapping): + return None + properties = _resolve_pointer(pointer).get("properties") or {} + out: Dict[str, Any] = {} + for key, item in value.items(): + if key not in properties: + continue + item = _coerce_for_key(key, item) + if item is _DROP: + continue + if _is_valid(f"{pointer}/properties/{key}", item): + out[key] = item + return out + + +def _column_view(fld: Any) -> Any: + if not isinstance(fld, Mapping): + return fld + out = _object_view(fld, "#/$defs/column") or {} + if "sensitivity" not in out: + classification = fld.get("classification") + if isinstance(classification, str): + level = classification.strip().lower() + if _is_valid("#/$defs/sensitivityLevel", level): + out["sensitivity"] = level + return out + + +def _schema_view(schema: Any) -> Any: + if isinstance(schema, list): + return [_column_view(fld) for fld in schema] + return schema + + +def _qos_view(qos: Any) -> Optional[Dict[str, Any]]: + return _object_view(qos, "#/$defs/exposeQoS") or None + + +def _ground_truth_view_at(path: str, value: Any) -> Any: + """The FLUID-0.7.2 view of the value found at a ground-truth ``path``.""" + value = _json_safe(_strip_passthrough(value)) + if path.endswith(".contract.schema"): + return _schema_view(value) + if path.endswith(".qos"): + return _qos_view(value) + return value + + +def _root_field_view(key: str, value: Any) -> Any: + value = _coerce_for_key(key, _json_safe(value)) + if value is _DROP or value is None: + return _DROP + return value if _is_valid(f"#/$defs/{_ROOT_DEF_PREFIX}{key}", value) else _DROP + + +_IDENTIFIER_POINTER = "#/$defs/identifier" +_INVALID_IDENTIFIER_CHARS_RE = re.compile(r"[^a-z0-9_.-]") + + +def _sanitise_identifier(raw: str) -> str: + """Deterministically map a free-form ODCS/ODPS name onto the FLUID 0.7.2 + identifier pattern (``^[a-z0-9_][a-z0-9_.-]*[a-z0-9_]$``): lower-case it, + replace every other character with ``_`` and make sure it neither starts + nor ends with ``.``/``-``. ``"Daily Orders"`` becomes ``"daily_orders"``.""" + text = _INVALID_IDENTIFIER_CHARS_RE.sub("_", raw.strip().lower()) + if text[:1] in (".", "-"): + text = "_" + text[1:] + if text[-1:] in (".", "-"): + text = text[:-1] + "_" + return text + + +def _expose_id_views(exposes: Any) -> List[Optional[str]]: + """The FLUID-0.7.2 ``exposeId`` of each entry of ``exposes``. + + The id is the expose's ``exposeId`` (or legacy ``id``). One that is not a + valid FLUID identifier — ODCS schema object and ODPS port names are free + strings, e.g. ``CustomerOrders`` or ``Daily Orders`` — is sanitised with + :func:`_sanitise_identifier`, a missing or empty one becomes + ``expose_``, and a sanitised id that collides with an earlier one gets a + ``_2``/``_3``/... suffix. The mapping depends only on the list itself, so + the prompt block and :func:`diff_against_seed` (which applies it to both + the seed and the candidate) always agree. + """ + if not isinstance(exposes, list): + return [] + views: List[Optional[str]] = [] + seen: set = set() + for index, expose in enumerate(exposes): + if not isinstance(expose, Mapping): + views.append(None) + continue + raw = expose.get("exposeId") or expose.get("id") + if isinstance(raw, (int, float)) and not isinstance(raw, bool): + raw = str(raw) + candidate = raw if isinstance(raw, str) and _is_valid(_IDENTIFIER_POINTER, raw) else None + if candidate is None and isinstance(raw, str): + candidate = _sanitise_identifier(raw) + if not _is_valid(_IDENTIFIER_POINTER, candidate): + candidate = None + if candidate is None: + candidate = f"expose_{index + 1}" + unique = candidate + suffix = 2 + while unique in seen: + unique = f"{candidate}_{suffix}" + suffix += 1 + seen.add(unique) + views.append(unique) + return views + + +def _expose_id_paths(fluid: Mapping[str, Any]) -> List[str]: + return [ + f"exposes[{i}].exposeId" + for i, view in enumerate(_expose_id_views(fluid.get("exposes"))) + if view is not None + ] + + +def _effective_ground_truth_paths(seed: "SeedResult") -> List[str]: + """Paths :func:`diff_against_seed` enforces: every seed expose's + ``exposeId`` plus the ground-truth paths whose FLUID view has something to + enforce.""" + return _expose_id_paths(seed.fluid) + [ + path + for path in seed.ground_truth_paths + if _ground_truth_view_at(path, resolve_at_path(seed.fluid, path)) is not None + ] + + +def format_ground_truth_prompt_block(seed: "SeedResult") -> Dict[str, Any]: + """Return the JSON-serialisable block the copilot prompt injects when a + seed is present. + + ``ground_truth`` is a FLUID-0.7.2-valid fragment the LLM can copy + verbatim into its contract: root ``id``/``name``/``description``/ + ``domain``/``tags``, ``metadata.owner`` when the seed names one, and per + expose ``exposeId`` (sanitised to a FLUID identifier, see + :func:`_expose_id_views`), ``contract.schema`` and ``qos``. It is exactly + the view :func:`diff_against_seed` enforces. ``*_passthrough`` stash buckets, + ``odcs_*``/``odps_*`` keys and any other key the 0.7.2 schema does not + allow at that position are left out. + """ + fluid = seed.fluid + metadata = fluid.get("metadata") or {} + ground_truth: Dict[str, Any] = {} + + root_id = (fluid.get("contract") or {}).get("id") or fluid.get("id") + for key, value in ( + ("id", root_id), + ("name", metadata.get("name", fluid.get("name"))), + ("description", metadata.get("description", fluid.get("description"))), + ("domain", metadata.get("domain", fluid.get("domain"))), + ("tags", metadata.get("tags", fluid.get("tags"))), + ): + view = _root_field_view(key, value) + if view is not _DROP: + ground_truth[key] = view + + owner_source = fluid.get("owner") + if not isinstance(owner_source, Mapping): + owner_source = metadata.get("owner") + owner = _object_view( + _json_safe(owner_source), f"#/$defs/{_ROOT_DEF_PREFIX}metadata/properties/owner" + ) + if owner: + ground_truth["metadata"] = {"owner": owner} + + exposes: List[Dict[str, Any]] = [] + raw_exposes = fluid.get("exposes") or [] + expose_ids = _expose_id_views(raw_exposes) + for i, expose in enumerate(raw_exposes): + if not isinstance(expose, Mapping): + continue + prefix = f"exposes[{i}]" + view_expose: Dict[str, Any] = {"exposeId": expose_ids[i]} + contract = expose.get("contract") + raw_schema = contract.get("schema") if isinstance(contract, Mapping) else None + schema = _ground_truth_view_at(f"{prefix}.contract.schema", raw_schema) + view_expose["contract"] = {"schema": schema if isinstance(schema, list) else []} + qos = _ground_truth_view_at(f"{prefix}.qos", expose.get("qos")) + if qos: + view_expose["qos"] = qos + exposes.append(view_expose) + ground_truth["exposes"] = exposes + + return _json_safe( + { + "shape": seed.shape, + "fluidVersion": GROUND_TRUTH_FLUID_VERSION, + "ground_truth": ground_truth, + "ground_truth_paths": _effective_ground_truth_paths(seed), + "provenance": list(seed.provenance), + } + ) + + +GROUND_TRUTH_ERROR_PREFIX = "ground_truth violation at " +_GROUND_TRUTH_OVERFLOW_SUFFIX = "more mutated paths" + + +def is_ground_truth_error(message: Any) -> bool: + """True for the repair-feedback strings :func:`format_mismatch_report` emits.""" + if not isinstance(message, str): + return False + return message.startswith(GROUND_TRUTH_ERROR_PREFIX) or ( + message.startswith("... and ") and message.endswith(_GROUND_TRUTH_OVERFLOW_SUFFIX) + ) + + +def format_mismatch_report(mismatches: List[Dict[str, Any]]) -> List[str]: + """Render a ``diff_against_seed`` result as human-readable repair + feedback strings, one per mismatch. Fed back to the LLM via the existing + ``previous_errors`` slot in the repair-loop prompt. + """ + lines: List[str] = [] + for m in mismatches[:10]: # cap for prompt-size sanity + seed_val = json.dumps(_json_safe(m.get("seed")))[:120] + cand_val = json.dumps(_json_safe(m.get("candidate")))[:120] + lines.append( + f"{GROUND_TRUTH_ERROR_PREFIX}{m.get('path')}: " + f"seed had {seed_val} but you returned {cand_val}" + ) + if len(mismatches) > 10: + lines.append(f"... and {len(mismatches) - 10} {_GROUND_TRUTH_OVERFLOW_SUFFIX}") + return lines + + +SHAPE_ODCS_FILE = "odcs-file" +SHAPE_ODPS_FILE = "odps-file" +SHAPE_DIRECTORY = "directory" +SHAPE_ODCS_ONLY_DIRECTORY = "odcs-only-directory" + +_SEED_FILE_SUFFIXES = (".yaml", ".yml", ".json") +_BITOL_INPUT_HINT = ( + "--seed-from expects Bitol input: export a FLUID contract first " + "(`fluid odcs export` / `fluid opds export --spec bitol-1.0.0 --out-dir `) " + "and pass the resulting *.odcs.yaml, *.odps.yaml, or bundle directory." +) + + +def _is_fluid_document(data: Any) -> bool: + return isinstance(data, Mapping) and "fluidVersion" in data + + +def _reject_fluid_document(path: Path, data: Any) -> None: + if _is_fluid_document(data): + raise ProviderError( + f"--seed-from: {path} is a FLUID contract (fluidVersion " + f"{data.get('fluidVersion')!r}), not a Bitol ODCS/ODPS document. " + _BITOL_INPUT_HINT + ) + + +def load_seed(seed_from: Union[str, Path], *, allow_remote: bool = True) -> SeedResult: + """Detect the input shape and run the matching importer. + + Detection order: + 1. Directory → Bitol bundle (one ODPS doc plus its ODCS + files) or an ODCS-only directory. FLUID contracts inside the + directory are ignored with a warning. + 2. ``*.odcs.{yaml,yml,json}`` filename → OdcsProvider.import_contract. + 3. ``*.odps.{yaml,yml,json}`` filename → BitolOdpsProvider.import_contract. + 4. Sniff the file's ``kind`` field: ``DataContract`` → ODCS, + ``DataProduct`` → ODPS. + + A FLUID contract (any document carrying ``fluidVersion``) is rejected, + every ODPS output port must resolve to its ODCS contract, and a seed with + no ground truth at all is an error: a seed that silently enforces nothing + would defeat the point of ``--seed-from``. + """ + path = Path(seed_from) + if not path.exists(): + raise ProviderError(f"--seed-from path not found: {path}") + + if path.is_dir(): + return _load_directory(path, allow_remote=allow_remote) + + name_lower = path.name.lower() + named_odcs = name_lower.endswith((".odcs.yaml", ".odcs.yml", ".odcs.json")) + named_odps = name_lower.endswith((".odps.yaml", ".odps.yml", ".odps.json")) + + sniffed: Any + if named_odcs or named_odps: + try: + sniffed = read_input(path) + except Exception: # let the importer report parse errors as before + sniffed = None + else: + sniffed = read_input(path) + _reject_fluid_document(path, sniffed) + + if named_odcs: + return _load_odcs_file(path) + + if named_odps: + return _load_odps_file(path, allow_remote=allow_remote) + + # Last-resort: sniff `kind` + kind = sniffed.get("kind") if isinstance(sniffed, Mapping) else None + if kind == "DataContract": + return _load_odcs_file(path) + if kind == "DataProduct": + return _load_odps_file(path, allow_remote=allow_remote) + + raise ProviderError( + f"--seed-from: cannot determine input type of {path}. " + "Expected a directory, *.odcs.yaml, *.odps.yaml, or a Bitol document " + "with kind: DataContract (ODCS) or kind: DataProduct (ODPS)." + ) + + +# ---- shape-specific loaders ----------------------------------------------- + + +def _load_odcs_file(path: Path) -> SeedResult: + provider = OdcsProvider() + fluid = provider.import_contract(path) + provenance = [{"shape": SHAPE_ODCS_FILE, "source": "local", "origin": str(path)}] + ground_truth = _ground_truth_paths_for(fluid) + _check_ground_truth(path, fluid, ground_truth) + return SeedResult( + fluid=fluid, + shape=SHAPE_ODCS_FILE, + provenance=provenance, + ground_truth_paths=ground_truth, + ) + + +def _import_odps_requiring_outputs( + provider: BitolOdpsProvider, + odps_path: Path, + *, + base_path: Path, + allow_remote: bool, + resolver: ContractResolver, +) -> Dict[str, Any]: + """Import an ODPS doc, requiring every *output* port to resolve. + + Input ports stay lenient inside the provider (upstream contracts often + live elsewhere). An unresolved output port would leave that expose with + no schema ground truth, so it is a hard error here. + """ + try: + return provider.import_contract( + odps_path, + base_path=base_path, + allow_remote=allow_remote, + resolver=resolver, + lenient=False, + ) + except ContractNotFound as exc: + raise ProviderError( + f"--seed-from: output port contract {exc.contract_id!r} could not be " + f"resolved, so its schema cannot be used as ground truth. {exc}" + ) from exc + + +def _load_odps_file(path: Path, *, allow_remote: bool) -> SeedResult: + provider = BitolOdpsProvider() + provider.strict_validation = False # seed mode tolerates non-strict contracts + resolver = ContractResolver( + base_path=path.parent, + allow_remote=allow_remote, + odcs_provider=provider._odcs, + ) + fluid = _import_odps_requiring_outputs( + provider, path, base_path=path.parent, allow_remote=allow_remote, resolver=resolver + ) + provenance = [{"shape": SHAPE_ODPS_FILE, "source": "local", "origin": str(path)}] + provenance.extend(_resolver_provenance(resolver)) + ground_truth = _ground_truth_paths_for(fluid) + _check_ground_truth(path, fluid, ground_truth) + return SeedResult( + fluid=fluid, + shape=SHAPE_ODPS_FILE, + provenance=provenance, + ground_truth_paths=ground_truth, + ) + + +def _scan_directory(root: Path) -> Tuple[List[Path], List[Path], List[Path]]: + """Classify every YAML/JSON file under ``root`` (recursively). + + Mirrors :meth:`BitolOdpsProvider.import_directory`'s scan, including its + containment rule (a file whose resolved path, symlinks included, is + outside ``root`` is never read), except that FLUID contracts (which also + say ``kind: DataProduct``) are set aside instead of being taken for the + bundle's ODPS doc. + """ + odps_files: List[Path] = [] + odcs_files: List[Path] = [] + fluid_files: List[Path] = [] + for child in sorted(root.rglob("*")): + if not child.is_file() or not _is_within(child, root): + # Symlinks pointing outside the bundle are never read. + continue + if child.suffix.lower() not in _SEED_FILE_SUFFIXES: + continue + try: + data = read_input(child) + except Exception: + continue + if not isinstance(data, Mapping): + continue + if _is_fluid_document(data): + fluid_files.append(child) + continue + kind = data.get("kind") + if kind == "DataProduct": + odps_files.append(child) + elif kind == "DataContract": + odcs_files.append(child) + return odps_files, odcs_files, fluid_files + + +def _load_directory(path: Path, *, allow_remote: bool) -> SeedResult: + """Either a full bundle (ODPS + ODCS) or an ODCS-only directory.""" + odps_files, odcs_files, fluid_files = _scan_directory(path) + # Defence in depth: only in-bundle files reach the importer and the + # resolver's ``additional_files`` index. + odcs_files = [f for f in odcs_files if _is_within(f, path)] + if fluid_files: + LOG.warning( + "--seed-from: ignoring FLUID contract(s) in %s: %s", + path, + ", ".join(str(f) for f in fluid_files), + ) + if len(odps_files) > 1: + raise ProviderError( + f"--seed-from: found {len(odps_files)} ODPS docs in {path}; " + "expected exactly one (or zero for ODCS-only mode). " + f"Found: {[str(p) for p in odps_files]}" + ) + + provider = BitolOdpsProvider() + provider.strict_validation = False + provenance: List[Dict[str, Any]] + if not odps_files: + if not odcs_files: + skipped = f" ({len(fluid_files)} FLUID contract(s) ignored)" if fluid_files else "" + raise ProviderError( + f"--seed-from: {path} contains no Bitol ODPS or ODCS documents{skipped}. " + + _BITOL_INPUT_HINT + ) + shape = SHAPE_ODCS_ONLY_DIRECTORY + fluid = provider._import_odcs_only(path, odcs_files) + provenance = [{"shape": shape, "source": "local", "origin": str(path)}] + provenance.extend({"source": "local", "origin": str(f)} for f in odcs_files) + else: + shape = SHAPE_DIRECTORY + resolver = ContractResolver( + base_path=path, + allow_remote=allow_remote, + odcs_provider=provider._odcs, + additional_files=odcs_files, + ) + resolver.index_directory(path) + fluid = _import_odps_requiring_outputs( + provider, odps_files[0], base_path=path, allow_remote=allow_remote, resolver=resolver + ) + provenance = [ + {"shape": shape, "source": "local", "origin": str(path)}, + {"source": "local", "origin": str(odps_files[0])}, + ] + # Every contract the resolver actually consumed: nested directories, + # .yml/.json files and remote URLs included. + provenance.extend(_resolver_provenance(resolver)) + ground_truth = _ground_truth_paths_for(fluid) + _check_ground_truth(path, fluid, ground_truth) + return SeedResult( + fluid=fluid, + shape=shape, + provenance=provenance, + ground_truth_paths=ground_truth, + ) + + +def _resolver_provenance(resolver: ContractResolver) -> List[Dict[str, Any]]: + return [ + { + "contract_id": resolved.contract_id, + "source": resolved.source, + "origin": resolved.origin, + } + for resolved in resolver._cache.values() + ] + + +def _check_ground_truth(path: Path, fluid: Mapping[str, Any], ground_truth: List[str]) -> None: + """Refuse a seed with nothing to enforce; warn about unprotected exposes.""" + if not ground_truth: + raise ProviderError( + f"--seed-from: {path} produced no ground truth (no output port or " + "schema object with a contract), so there is nothing to preserve." + ) + for expose in fluid.get("exposes") or []: + if isinstance(expose, Mapping) and not isinstance(expose.get("contract"), Mapping): + LOG.warning( + "--seed-from: expose %r in %s has no contract; its schema is not " + "protected as ground truth", + expose.get("exposeId") or expose.get("id"), + path, + ) + + +# ---- ground-truth path enumeration ---------------------------------------- + + +def _ground_truth_paths_for(fluid: Mapping[str, Any]) -> List[str]: + """Enumerate dotted paths inside ``fluid`` the LLM must not mutate. + + Currently: every ``exposes[i].contract.schema`` and ``exposes[i].qos``, + plus ``exposes[i].contract.relationships`` and + ``exposes[i].contract.odcs_quality``. The runtime guard (Phase 7 + finishing touches) reads this list and diffs the LLM's output against + the seed values at each path. + """ + paths: List[str] = [] + for i, expose in enumerate(fluid.get("exposes") or []): + if not isinstance(expose, Mapping): + continue + prefix = f"exposes[{i}]" + if isinstance(expose.get("contract"), Mapping): + paths.append(f"{prefix}.contract.schema") + if "relationships" in expose["contract"]: + paths.append(f"{prefix}.contract.relationships") + if "odcs_quality" in expose["contract"]: + paths.append(f"{prefix}.contract.odcs_quality") + if "qos" in expose: + paths.append(f"{prefix}.qos") + return paths + + +_PATH_TOKEN_RE = re.compile(r"[^.\[\]]+|\[\d+\]") + + +def resolve_at_path(fluid: Mapping[str, Any], path: str) -> Any: + """Look up the value at a dotted/indexed path like ``exposes[0].contract.schema``. + + Returns ``None`` for anything missing, including an index past the end of + a list (the candidate has fewer exposes than the seed) or an index into a + non-list, so the guard reports a mismatch instead of raising. + """ + cursor: Any = fluid + for token in _PATH_TOKEN_RE.findall(path): + if token.startswith("[") and token.endswith("]"): + idx = int(token[1:-1]) + if not isinstance(cursor, (list, tuple)) or idx >= len(cursor): + return None + cursor = cursor[idx] + else: + if isinstance(cursor, Mapping): + cursor = cursor.get(token) + else: + cursor = getattr(cursor, token, None) + if cursor is None: + return None + return cursor + + +def diff_against_seed(seed: SeedResult, candidate: Mapping[str, Any]) -> List[Dict[str, Any]]: + """Compare ground-truth values in ``candidate`` against the seed. + + Returns a list of ``{"path", "seed", "candidate"}`` entries for paths + where the candidate's value differs from the seed. The runtime guard + raises (or triggers a repair loop) when this list is non-empty. + + Both sides go through the same FLUID-0.7.2 view the prompt shows (see + :func:`format_ground_truth_prompt_block`): ``*_passthrough`` stash buckets + and keys the 0.7.2 schema forbids (ODCS ``quality``, ``classification``, + ...) are ignored, values are JSON-normalised, and every expose's + ``exposeId`` is compared after the same identifier sanitisation. A candidate that copies + the prompt's ground truth verbatim, or the raw seed, is therefore clean, + and one that drops round-trip bookkeeping is not a violation. + """ + mismatches: List[Dict[str, Any]] = [] + seed_ids = _expose_id_views(seed.fluid.get("exposes")) + cand_exposes = candidate.get("exposes") if isinstance(candidate, Mapping) else None + cand_ids = _expose_id_views(cand_exposes) + for i, seed_id in enumerate(seed_ids): + if seed_id is None: + continue + cand_id = cand_ids[i] if i < len(cand_ids) else None + if seed_id != cand_id: + mismatches.append( + {"path": f"exposes[{i}].exposeId", "seed": seed_id, "candidate": cand_id} + ) + for path in seed.ground_truth_paths: + seed_value = _ground_truth_view_at(path, resolve_at_path(seed.fluid, path)) + if seed_value is None: + continue # nothing FLUID-expressible to enforce at this path + cand_value = _ground_truth_view_at(path, resolve_at_path(candidate, path)) + if seed_value != cand_value: + mismatches.append({"path": path, "seed": seed_value, "candidate": cand_value}) + return mismatches diff --git a/fluid_build/cli/forge_modes.py b/fluid_build/cli/forge_modes.py index ccc9e112..2fbc99f0 100644 --- a/fluid_build/cli/forge_modes.py +++ b/fluid_build/cli/forge_modes.py @@ -71,6 +71,14 @@ def run_ai_copilot_mode( "[dim]I'll help you create the perfect data product by understanding your needs...[/dim]\n" ) + # Phase 7 seed hand-off: build SeedOptions once at the CLI boundary + # and thread it through the copilot pipeline. ``None`` when the + # operator didn't pass ``--seed-from``, in which case the runtime + # behaves exactly as it did before Phase 7 landed. + from fluid_build.cli.forge_copilot_seed import SeedOptions as _SeedOptions + + seed_options = _SeedOptions.from_args(args) + context: Dict[str, Any] = {} copilot_options = { "llm_provider": get_cli_arg_fn(args, "llm_provider"), @@ -81,6 +89,7 @@ def run_ai_copilot_mode( "memory": get_cli_arg_fn(args, "memory", True), "save_memory": get_cli_arg_fn(args, "save_memory", False), "non_interactive": get_cli_arg_fn(args, "non_interactive", False), + "seed_options": seed_options, } context_arg = get_cli_arg_fn(args, "context") diff --git a/fluid_build/cli/help_formatter.py b/fluid_build/cli/help_formatter.py index d12f99e4..b5461f02 100644 --- a/fluid_build/cli/help_formatter.py +++ b/fluid_build/cli/help_formatter.py @@ -193,7 +193,7 @@ def _section(icon: str, title: str, color: str, rows: list[tuple[str, str]]) -> [ ("export-opds", "Export to ODPS [dim](Open Data Product Spec)[/dim]"), ("odcs", "ODCS v3.1 [dim](Open Data Contract Standard — Bitol.io)[/dim]"), - ("odps", "ODPS v4.1 [dim](Linux Foundation data product spec)[/dim]"), + ("odps", "Bitol ODPS v1.0 [dim](default)[/dim] · ODPI v4.1 [dim]--spec odpi-4.1[/dim]"), ("market", "Browse & install marketplace data products"), ("publish", "Publish a data product to the marketplace"), ], @@ -236,7 +236,7 @@ def _section(icon: str, title: str, color: str, rows: list[tuple[str, str]]) -> extras.add_row(" execute", "Run build jobs manually") extras.add_row(" wizard", "Step-by-step guided setup") extras.add_row(" marketplace", "Extended marketplace browser") - extras.add_row(" odps-bitol", "ODPS-Bitol v1.0 (Entropy Data)") + extras.add_row(" odps-bitol", "Deprecated — use odps export --spec bitol-1.0.0") extras.add_row(" preview", "Dry-run alias for apply") extras.add_row(" viz-plan", "Interactive plan visualization") console.print(" [dim]▸ More Commands[/dim]") @@ -530,6 +530,7 @@ def print_forge_help() -> None: ), ), "odps-bitol": ( + "Deprecated: use 'fluid opds export --spec bitol-1.0.0'. " "Work with ODPS-Bitol format for Entropy Data marketplace integration.", "", # keep existing epilog ), diff --git a/fluid_build/cli/odps_standard.py b/fluid_build/cli/odps_standard.py index 964afe61..26cd1414 100644 --- a/fluid_build/cli/odps_standard.py +++ b/fluid_build/cli/odps_standard.py @@ -26,6 +26,8 @@ import json import logging +import sys +import warnings from pathlib import Path from typing import Optional @@ -35,6 +37,23 @@ from fluid_build.loader import load_contract from fluid_build.providers.odps_standard import OdpsStandardProvider +DEPRECATION_MESSAGE = ( + "'fluid odps-bitol' is deprecated and will be removed in a future release; " + "use 'fluid opds export --spec bitol-1.0.0' " + "(or the equivalent 'fluid odps export --spec bitol-1.0.0') instead." +) + + +def _warn_deprecated() -> None: + """Emit the ``fluid odps-bitol`` deprecation as a warning *and* a stderr notice. + + ``DeprecationWarning`` is hidden by default outside ``__main__``, so the + stderr line is what users actually see; the warning lets test suites and + ``-W error`` runs catch it. + """ + warnings.warn(DEPRECATION_MESSAGE, DeprecationWarning, stacklevel=3) + print(f"⚠ {DEPRECATION_MESSAGE}", file=sys.stderr) + @click.group(name="odps-bitol") def odps_bitol_cli(): @@ -78,6 +97,7 @@ def export_command(contract: str, output: Optional[str], format: str, include_cu """ logger = logging.getLogger(__name__) + click.echo(f"⚠ {DEPRECATION_MESSAGE}", err=True) try: # Load FLUID contract click.echo(f"Loading FLUID contract: {contract}") @@ -229,8 +249,11 @@ def register(subparsers) -> None: odps_bitol = subparsers.add_parser( "odps-bitol", - help="ODPS-Bitol (Bitol.io Data Product Standard for Entropy Data)", + help="Deprecated: use 'fluid opds export --spec bitol-1.0.0' (ODPS-Bitol)", description=""" + DEPRECATED: use 'fluid opds export --spec bitol-1.0.0' (or + 'fluid odps export --spec bitol-1.0.0') instead. + Work with ODPS-Bitol (Bitol.io Data Product Standard) format. ODPS-Bitol is used for data marketplace integration with Entropy Data. @@ -270,6 +293,8 @@ def _run_odps_export(args): """Run ODPS export command.""" import logging + _warn_deprecated() + from fluid_build.cli.bootstrap import load_contract_with_overlay logger = logging.getLogger(__name__) @@ -300,6 +325,8 @@ def _run_odps_validate(args): import json from pathlib import Path + _warn_deprecated() + file_path = Path(args.odps_file) with open(file_path) as f: if file_path.suffix in (".yaml", ".yml"): @@ -327,6 +354,7 @@ def _run_odps_validate(args): def _run_odps_info(args): """Run ODPS info command.""" + _warn_deprecated() provider = OdpsStandardProvider() cprint("ODPS (Open Data Product Standard) Provider") cprint("=" * 50) diff --git a/fluid_build/cli/opds.py b/fluid_build/cli/opds.py index 669e73e5..dd120b55 100644 --- a/fluid_build/cli/opds.py +++ b/fluid_build/cli/opds.py @@ -14,28 +14,29 @@ # fluid_build/cli/opds.py """ -ODPS (Open Data Product Specification) CLI Commands +ODPS / OPDS CLI Commands — two specifications under one user-facing command. -Provides commands for exporting FLUID contracts to ODPS format with support -for multiple specification versions. +Specifications selected via ``--spec``: -Official Specification: -- v4.1: https://github.com/Open-Data-Product-Initiative/v4.1 -- Schema: https://github.com/Open-Data-Product-Initiative/v4.1/blob/main/source/schema/odps.yaml +- ``bitol-1.0.0`` (**default**) — Bitol Open Data Product Standard v1.0.0. + Emits 1 ODPS doc + N sibling ``.odcs.yaml`` files (canonical + Bitol fragments layout). Backed by :class:`BitolOdpsProvider`. +- ``odpi-4.1`` — Open Data Product Initiative v4.1 (Linux Foundation). + Emits a single JSON document. Backed by the legacy ``odps`` provider. -Standards Compliance: -- ODPS v4.1 (default) - Linux Foundation / Open Data Product Initiative -- Future version support through version parameter -- Full metadata preservation -- Validation against official schema +The legacy ``--version 4.1`` flag still works as a deprecated alias for +``--spec odpi-4.1``. -Note: The CLI command is 'opds' for historical reasons, but this implements -the official ODPS (Open Data Product Specification) standard. +Machine-readable payloads (JSON/YAML written to stdout) are emitted raw via +``sys.stdout.write`` so pipelines receive exactly the serialised document; +human-facing status lines keep going through the Rich console. Usage: - fluid opds export [--version 4.1] [--out file.json] - fluid opds validate [--version 4.1] - fluid opds info [--version 4.1] + fluid opds export [--spec bitol-1.0.0|bitol-1.1.0|odpi-4.1] + [--out FILE | --out-dir DIR] [-f yaml|json] + fluid opds import [--spec bitol-1.0.0] [--no-remote] [--lenient] [-o OUT] + fluid opds validate [--spec ...] (auto-detects Bitol vs ODPI when omitted) + fluid opds info [--spec ...] """ from __future__ import annotations @@ -43,8 +44,10 @@ import argparse import json import logging -from pathlib import Path -from typing import Any, Dict +import os +import sys +from pathlib import Path, PurePosixPath, PureWindowsPath +from typing import Any, Dict, List, Mapping, Optional from fluid_build.cli.console import cprint from fluid_build.cli.console import error as console_error @@ -74,6 +77,85 @@ DEFAULT_VERSION = next(v for v, info in ODPS_VERSIONS.items() if info.get("default", False)) +# Specification selector — the user-facing ``--spec`` flag dispatches between +# the Bitol Open Data Product Standard (v1.0.0 / v1.1.0) and the legacy Open +# Data Product Initiative v4.1. The default stays on bitol-1.0.0 so existing +# pipelines keep producing the document shape they validate against; opt into +# v1.1.0 explicitly. Imports accept either Bitol version (the validator picks +# the schema from the document's own apiVersion). +SPEC_BITOL_1_0_0 = "bitol-1.0.0" +SPEC_BITOL_1_1_0 = "bitol-1.1.0" +SPEC_ODPI_4_1 = "odpi-4.1" +DEFAULT_SPEC = SPEC_BITOL_1_0_0 +SUPPORTED_SPECS = (SPEC_BITOL_1_0_0, SPEC_BITOL_1_1_0, SPEC_ODPI_4_1) +BITOL_SPEC_API_VERSIONS = {SPEC_BITOL_1_0_0: "v1.0.0", SPEC_BITOL_1_1_0: "v1.1.0"} +BITOL_SPEC_URL = "https://github.com/bitol-io/open-data-product-standard" +BITOL_SPECS: Dict[str, Dict[str, Any]] = { + SPEC_BITOL_1_0_0: { + "apiVersion": "v1.0.0", + "spec_url": BITOL_SPEC_URL, + "description": ( + "Bitol Open Data Product Standard v1.0.0 - 1 ODPS doc + N sibling ODCS contracts" + ), + "status": "stable", + "default": True, + }, + SPEC_BITOL_1_1_0: { + "apiVersion": "v1.1.0", + "spec_url": BITOL_SPEC_URL, + "description": ( + "Bitol Open Data Product Standard v1.1.0 - additive over v1.0.0 " + "(type, deprecated, context, synonyms)" + ), + "status": "stable", + "default": False, + }, +} + +_FORMAT_BY_SUFFIX = {".json": "json", ".yaml": "yaml", ".yml": "yaml"} + + +def _write_stdout(payload: str) -> None: + """Write a machine-readable payload to stdout verbatim (never through Rich). + + Rich would hard-wrap long lines at the console width and treat + ``[a-z0-9_]``-style substrings as markup, corrupting YAML/JSON. + """ + sys.stdout.write(payload if payload.endswith("\n") else payload + "\n") + sys.stdout.flush() + + +def _warn(logger: logging.Logger, msg: str) -> None: + """Human-facing warning on stderr (keeps stdout clean for payloads).""" + logger.warning(msg) + print(f"⚠ {msg}", file=sys.stderr) + + +def _str_arg(args: argparse.Namespace, name: str) -> Optional[str]: + """Return a string CLI arg, or None when unset (tolerates mock namespaces).""" + value = getattr(args, name, None) + return value if isinstance(value, str) else None + + +def resolve_spec(args: argparse.Namespace) -> str: + """Resolve the active ``--spec`` from CLI args. + + Precedence: ``--spec`` > legacy ``--version 4.1`` > default. + Emits a deprecation warning when ``--version`` is the only signal. + """ + spec = getattr(args, "spec", None) + if isinstance(spec, str) and spec in SUPPORTED_SPECS: + return spec + legacy_version = getattr(args, "version", None) + if isinstance(legacy_version, str) and legacy_version in ODPS_VERSIONS: + if legacy_version == "4.1": + LOG.warning( + "--version 4.1 is deprecated; use --spec odpi-4.1 instead", + ) + return SPEC_ODPI_4_1 + return DEFAULT_SPEC + + def get_version_info(version: str) -> Dict[str, Any]: """Get information about a specific ODPS version.""" if version not in ODPS_VERSIONS: @@ -83,104 +165,485 @@ def get_version_info(version: str) -> Dict[str, Any]: def cmd_opds_export(args: argparse.Namespace, logger: logging.Logger) -> int: + """Export FLUID contract — dispatched by ``--spec``.""" + from fluid_build.cli.bootstrap import load_contract_with_overlay + + spec = resolve_spec(args) + try: + contract = load_contract_with_overlay(args.contract, getattr(args, "env", None), logger) + except Exception as e: + logger.error("contract_load_failed", extra={"error": str(e)}) + console_error(f"Error loading contract: {e}") + return 1 + + if spec in BITOL_SPEC_API_VERSIONS: + # Only an explicit Bitol ``--spec`` pins the emitted apiVersion. When + # it is omitted the provider's own chain applies: ODPS_API_VERSION env + # > the imported document's source apiVersion > default v1.0.0. + explicit = _str_arg(args, "spec") in BITOL_SPEC_API_VERSIONS + return _export_bitol( + args, + contract, + logger, + api_version=BITOL_SPEC_API_VERSIONS[spec] if explicit else None, + ) + if spec == SPEC_ODPI_4_1: + return _export_odpi_v41(args, contract, logger) + console_error(f"Unsupported --spec {spec!r}; supported: {SUPPORTED_SPECS}") + return 2 + + +def _unsafe_file_stem(name: Any) -> Optional[str]: + """Return why ``name`` cannot be used as a file stem, or None when it is safe.""" + if not isinstance(name, str) or not name.strip(): + return "it is empty" + if PurePosixPath(name).is_absolute() or PureWindowsPath(name).is_absolute(): + return "it is an absolute path" + if "/" in name or "\\" in name: + return "it contains a path separator" + if ".." in name: + return "it contains '..'" + return None + + +def _resolve_bitol_format(explicit_fmt: Optional[str], out_file: Optional[Path]) -> str: + """Explicit ``-f`` wins; otherwise infer from the ``--out`` suffix; default yaml.""" + if explicit_fmt: + return explicit_fmt.lower() + if out_file is not None: + return _FORMAT_BY_SUFFIX.get(out_file.suffix.lower(), "yaml") + return "yaml" + + +def _non_strict_problems(bundle: Mapping[str, Any]) -> List[str]: + """Run every export-time check and collect failures instead of raising. + + Mirrors what strict mode enforces: the ODPS product schema, each per-port + ODCS schema, and the optional ``vowl`` second pass. """ - Export FLUID contract to ODPS format. + from fluid_build.providers.odcs import validation as odcs_validation + from fluid_build.providers.odps_standard import validation as odps_validation - Args: - args.contract: Path to FLUID contract file - args.version: ODPS specification version (default: 4.1) - args.out: Output file path or '-' for stdout - args.env: Optional environment overlay - args.validate: Validate output against schema (default: true) - args.pretty: Pretty-print JSON output (default: true) + problems: List[str] = [] + try: + odps_validation.validate(bundle.get("product") or {}) + except Exception as exc: # noqa: BLE001 - reported, never raised + problems.append(f"ODPS product: {exc}") + for contract_id, odcs in (bundle.get("contracts") or {}).items(): + try: + odcs_validation.validate(odcs) + except Exception as exc: # noqa: BLE001 + problems.append(f"ODCS contract {contract_id}: {exc}") + try: + odcs_validation.validate_via_vowl(odcs) + except Exception as exc: # noqa: BLE001 + problems.append(f"ODCS contract {contract_id}: {exc}") + return problems - Returns: - 0 on success, non-zero on error + +def _export_bitol( + args: argparse.Namespace, + contract: Dict[str, Any], + logger: logging.Logger, + *, + api_version: Optional[str], +) -> int: + """Bitol ODPS export — 1 ODPS doc + N sibling ODCS contracts. + + ``api_version=None`` lets :class:`BitolOdpsProvider` resolve the version + itself (``ODPS_API_VERSION`` > imported source apiVersion > v1.0.0). """ - from fluid_build.cli.bootstrap import build_provider, load_contract_with_overlay + from fluid_build.providers.odps_standard import BitolOdpsProvider + from fluid_build.providers.odps_standard.io import write_output - version = getattr(args, "version", DEFAULT_VERSION) - version_info = get_version_info(version) + out = _str_arg(args, "out") or "-" + out_dir = _str_arg(args, "out_dir") + validate_strict = getattr(args, "validate_strict", True) + out_file = Path(out) if out != "-" else None - logger.debug( - "opds_export_start", - extra={ - "contract": args.contract, - "version": version, - "spec": version_info["spec_url"], - "output": getattr(args, "out", "-"), - }, - ) + if out_file is not None and out_dir: + console_error( + "--out and --out-dir are mutually exclusive: use --out FILE for the product " + "doc (per-port ODCS contracts are written next to it) or --out-dir DIR." + ) + return 2 + + fmt = _resolve_bitol_format(_str_arg(args, "format"), out_file) - # Load contract try: - contract = load_contract_with_overlay(args.contract, getattr(args, "env", None), logger) + provider = BitolOdpsProvider(api_version=api_version) + provider.strict_validation = bool(validate_strict) + # Render in memory first so every target path can be checked before + # anything touches the filesystem. + bundle = provider.render(contract, out=None, out_dir=None, fmt=fmt) except Exception as e: - logger.error("contract_load_failed", extra={"error": str(e)}) - console_error(f"Error loading contract: {e}") + logger.error("opds_export_failed", extra={"error": str(e), "api_version": api_version}) + console_error(f"Error exporting to Bitol ODPS: {e}") return 1 - # Build OPDS provider + product = bundle.get("product") or {} + contracts = bundle.get("contracts") or {} + written_version = product.get("apiVersion") or getattr(provider, "api_version", "") + + if not validate_strict: + for problem in _non_strict_problems(bundle): + _warn(logger, f"validation (non-strict): {problem}") + + if out_file is None and not out_dir: + # Dump product doc to stdout. Use JSON for stdout regardless of fmt + # so the output is unambiguous in pipelines. + _write_stdout(json.dumps(product, indent=2, ensure_ascii=False)) + return 0 + + stems = [("contractId", contract_id) for contract_id in contracts] + if out_dir: + # The provider names the product file after its id (or name). + stems.append(("product id", product.get("id") or product.get("name") or "product")) + for label, stem in stems: + reason = _unsafe_file_stem(stem) + if reason: + console_error( + f"Refusing to write Bitol ODPS export: {label} {stem!r} cannot be used " + f"as a file name because {reason}." + ) + return 1 + + if out_dir: + try: + # Canonical layout, written by the provider itself. + provider._write_bundle(product, contracts, Path(out_dir), fmt) + except Exception as e: + logger.error("opds_export_write_failed", extra={"error": str(e)}) + console_error(f"Error writing Bitol ODPS export: {e}") + return 1 + cprint( + f"✓ Exported Bitol ODPS {written_version}: 1 product + " + f"{len(contracts)} ODCS contract(s) → {out_dir}", + soft_wrap=True, + markup=False, + ) + return 0 + + # --out FILE: the product goes to FILE and every per-port ODCS contract is + # written next to it as .odcs., so the bundle stays + # importable (the resolver probes exactly these sibling names). + ext = out_file.suffix.lstrip(".").lower() + if _FORMAT_BY_SUFFIX.get(f".{ext}") != fmt: + ext = fmt + contract_paths = [out_file.parent / f"{contract_id}.odcs.{ext}" for contract_id in contracts] + try: + write_output(dict(product), out_file, fmt) + for contract_path, odcs in zip(contract_paths, contracts.values()): + write_output(dict(odcs), contract_path, fmt) + except Exception as e: + logger.error("opds_export_write_failed", extra={"error": str(e)}) + console_error(f"Error writing Bitol ODPS export: {e}") + return 1 + + cprint( + f"✓ Exported Bitol ODPS {written_version}: 1 product + " + f"{len(contract_paths)} ODCS contract(s)" + ) + # Paths are printed unwrapped and without markup parsing so they stay + # copy-pasteable. + cprint(f" product: {out_file}", soft_wrap=True, markup=False) + for contract_path in contract_paths: + cprint(f" contract: {contract_path}", soft_wrap=True, markup=False) + return 0 + + +def _export_odpi_v41( + args: argparse.Namespace, contract: Dict[str, Any], logger: logging.Logger +) -> int: + """Legacy ODPI v4.1 export path (single JSON document).""" + from fluid_build.cli.bootstrap import build_provider + + # --out-dir and -f/--format only mean something for the Bitol bundle + # layout; ODPI v4.1 is always a single JSON document. Refuse them instead + # of silently writing JSON somewhere else. + if _str_arg(args, "out_dir"): + console_error( + "--out-dir is not supported with --spec odpi-4.1 (ODPI v4.1 is a single JSON " + "document): use --out FILE, or omit it to print to stdout." + ) + return 2 + explicit_fmt = _str_arg(args, "format") + if explicit_fmt and explicit_fmt.lower() != "json": + console_error( + f"--format {explicit_fmt} is not supported with --spec odpi-4.1: ODPI v4.1 is " + "always written as JSON (omit -f/--format or use -f json)." + ) + return 2 + + version_info = get_version_info("4.1") try: provider = build_provider("odps", None, None, logger) except Exception as e: logger.error("provider_build_failed", extra={"error": str(e)}) - console_error(f"Error building OPDS provider: {e}") + console_error(f"Error building ODPI provider: {e}") return 1 - # Set version-specific configuration - provider.opds_version = version + provider.opds_version = "4.1" provider.opds_spec_url = version_info["spec_url"] provider.opds_schema_url = version_info["schema_url"] - # Render to OPDS try: result = provider.render(contract, out=getattr(args, "out", "-"), fmt="opds") - - # Output handling out_path = getattr(args, "out", "-") if out_path == "-": - # Print to stdout with optional pretty printing if getattr(args, "pretty", True): - cprint(json.dumps(result, indent=2, ensure_ascii=False)) + _write_stdout(json.dumps(result, indent=2, ensure_ascii=False)) else: - cprint(json.dumps(result, ensure_ascii=False)) + _write_stdout(json.dumps(result, ensure_ascii=False)) else: - logger.debug( - "opds_export_success", - extra={ - "output": out_path, - "version": version, - "size_bytes": Path(out_path).stat().st_size if Path(out_path).exists() else 0, - }, - ) - cprint(f"✓ Exported to OPDS v{version}: {out_path}") + cprint(f"✓ Exported to ODPI v4.1: {out_path}", soft_wrap=True, markup=False) cprint(f" Specification: {version_info['spec_url']}") - return 0 - except Exception as e: logger.error("opds_export_failed", extra={"error": str(e)}) - console_error(f"Error exporting to OPDS: {e}") + console_error(f"Error exporting to ODPI: {e}") return 1 +def cmd_opds_import(args: argparse.Namespace, logger: logging.Logger) -> int: + """Import an ODPS / ODCS file or directory → one FLUID contract skeleton. + + The *inputs* are schema-validated; the emitted FLUID is a skeleton that + may need edits (fluidVersion/kind/id/name, expose kind, binding + format/location) before ``fluid validate`` / ``plan`` accept it. + + Input dispatch: + - Path ends in ``.odcs.yaml``/``.odcs.yml``/``.odcs.json`` → + :meth:`OdcsProvider.import_contract` (single-expose FLUID). + - Path is a file with ``kind: DataProduct`` (or ``.odps.yaml``) → + :meth:`BitolOdpsProvider.import_contract` with resolver. + - Path is a directory → :meth:`BitolOdpsProvider.import_directory`. + + ``--spec odpi-4.1`` is reserved for the legacy provider, which is + export-only and rejects import with a clear error. + """ + spec = resolve_spec(args) + if spec == SPEC_ODPI_4_1: + console_error( + "ODPI v4.1 is export-only; import is supported for the Bitol specs " + f"({', '.join(BITOL_SPEC_API_VERSIONS)})" + ) + return 2 + + in_path = Path(args.path) + if not in_path.exists(): + console_error(f"Input path not found: {in_path}") + return 1 + + allow_remote = not getattr(args, "no_remote", False) + lenient = bool(getattr(args, "lenient", False)) + out = getattr(args, "out", None) + out_file = ( + Path(out) if isinstance(out, (str, os.PathLike)) and str(out) not in ("", "-") else None + ) + # Same rule as export: an explicit -f/--format wins, else the -o suffix + # decides (.json → json, .yaml/.yml → yaml), else yaml. + fmt = _resolve_bitol_format(_str_arg(args, "format"), out_file) + + try: + fluid = _dispatch_import(in_path, allow_remote=allow_remote, lenient=lenient) + except Exception as e: + logger.error("opds_import_failed", extra={"error": str(e), "path": str(in_path)}) + console_error(f"Error importing: {e}") + return 1 + + if out_file is not None: + out_path = out_file + out_path.parent.mkdir(parents=True, exist_ok=True) + if fmt == "json": + out_path.write_text(json.dumps(fluid, indent=2, ensure_ascii=False)) + else: + import yaml + + out_path.write_text(yaml.dump(fluid, default_flow_style=False, sort_keys=False)) + # Paths are printed unwrapped and without markup parsing so they stay + # copy-pasteable. + cprint(f"✓ Imported {in_path} → {out_path}", soft_wrap=True, markup=False) + else: + if fmt == "json": + _write_stdout(json.dumps(fluid, indent=2, ensure_ascii=False)) + else: + import yaml + + _write_stdout(yaml.dump(fluid, default_flow_style=False, sort_keys=False)) + return 0 + + +def _dispatch_import(path: Path, *, allow_remote: bool, lenient: bool) -> Dict[str, Any]: + """Choose the right import method based on the input path's shape/content.""" + if path.is_dir(): + from fluid_build.providers.odps_standard import BitolOdpsProvider + + return BitolOdpsProvider().import_directory( + path, allow_remote=allow_remote, lenient=lenient + ) + + # Single-file dispatch + suffixes = {s.lower() for s in path.suffixes} + name_lower = path.name.lower() + + if ".odcs" in suffixes or name_lower.endswith((".odcs.yaml", ".odcs.yml", ".odcs.json")): + from fluid_build.providers.odcs import OdcsProvider + + return OdcsProvider().import_contract(path, lenient=lenient) + + if ".odps" in suffixes or name_lower.endswith((".odps.yaml", ".odps.yml", ".odps.json")): + from fluid_build.providers.odps_standard import BitolOdpsProvider + + return BitolOdpsProvider().import_contract(path, allow_remote=allow_remote, lenient=lenient) + + # Last resort: sniff the file's ``kind`` field + from fluid_build.providers.odcs.io import read_input + + data = read_input(path) + if isinstance(data, dict) and "fluidVersion" in data: + # FLUID contracts also declare ``kind: DataProduct``. + raise ValueError( + f"{path} is a FLUID contract, not a Bitol ODPS/ODCS document; " + "use `fluid opds export` to turn it into ODPS." + ) + kind = data.get("kind") if isinstance(data, dict) else None + if kind == "DataProduct": + from fluid_build.providers.odps_standard import BitolOdpsProvider + + return BitolOdpsProvider().import_contract(path, allow_remote=allow_remote, lenient=lenient) + if kind == "DataContract": + from fluid_build.providers.odcs import OdcsProvider + + return OdcsProvider().import_contract(path, lenient=lenient) + + raise ValueError( + f"Cannot determine input type for {path}: " + f"expected a directory, *.odps.yaml, *.odcs.yaml, or a file with " + f"kind: DataProduct or kind: DataContract" + ) + + +def _load_document(path: Path) -> Any: + """Load a YAML or JSON document. + + ``.yaml``/``.yml`` files are parsed as YAML. Anything else is parsed as + JSON first (so JSON errors read exactly as before); when that fails a + YAML parse is attempted and accepted only if it yields a mapping. + """ + import yaml + + text = path.read_text(encoding="utf-8") + if path.suffix.lower() in (".yaml", ".yml"): + data = yaml.safe_load(text) + if not isinstance(data, dict): + raise ValueError(f"{path} did not parse as a mapping") + return data + try: + return json.loads(text) + except json.JSONDecodeError as json_err: + try: + data = yaml.safe_load(text) + except yaml.YAMLError: + raise json_err from None + if isinstance(data, dict): + return data + raise json_err from None + + +def _is_bitol_document(doc: Any) -> bool: + return isinstance(doc, dict) and "apiVersion" in doc and doc.get("kind") == "DataProduct" + + +def _validate_bitol( + path: Path, + doc: Any, + logger: logging.Logger, + *, + spec: Optional[str], +) -> int: + """Validate a Bitol ODPS product with :meth:`BitolOdpsProvider.validate_product`. + + The schema follows the document's own ``apiVersion`` unless an explicit + Bitol ``--spec`` pins it. + """ + from fluid_build.providers.base import ProviderError + from fluid_build.providers.odps_standard import BitolOdpsProvider + from fluid_build.providers.odps_standard import validation as odps_validation + + if not isinstance(doc, dict): + console_error(f"✗ Bitol ODPS validation failed: {path} is not a mapping") + return 1 + + if spec in BITOL_SPEC_API_VERSIONS: + schema_version = BITOL_SPEC_API_VERSIONS[spec] + else: + schema_version = odps_validation.resolve_schema_version(doc) + + logger.info( + "opds_validate_start", + extra={"file": str(path), "spec": spec or "auto", "schema_version": schema_version}, + ) + try: + if spec in BITOL_SPEC_API_VERSIONS: + odps_validation.validate(doc, odps_validation.load_schema(schema_version)) + else: + BitolOdpsProvider().validate_product(doc) + except ProviderError as e: + console_error(f"✗ Bitol ODPS validation failed (schema {schema_version})") + console_error(f" - {e}") + logger.error("opds_invalid", extra={"file": str(path), "error": str(e)}) + return 1 + + cprint(f"✓ Bitol ODPS file is valid (schema {schema_version})") + cprint(f" Data Product: {doc.get('id')}") + cprint(f" Name: {doc.get('name')}") + cprint(f" apiVersion: {doc.get('apiVersion')}") + cprint(f" Specification: {BITOL_SPEC_URL}") + logger.info("opds_valid", extra={"file": str(path), "schema_version": schema_version}) + return 0 + + def cmd_opds_validate(args: argparse.Namespace, logger: logging.Logger) -> int: """ - Validate an OPDS file against the specification schema. + Validate an ODPS file (YAML or JSON) against the specification schema. Args: - args.file: Path to OPDS JSON file - args.version: OPDS specification version to validate against + args.file: Path to the ODPS file (YAML or JSON) + args.spec: ``bitol-1.0.0`` / ``bitol-1.1.0`` / ``odpi-4.1``; when + omitted the document is auto-detected — ``apiVersion`` plus + ``kind: DataProduct`` means Bitol, anything else is ODPI v4.1. + args.version: Legacy ODPI version selector (forces the ODPI path) args.full_schema: Use full JSON schema validation (default: true) Returns: 0 if valid, non-zero if invalid """ - version = getattr(args, "version", DEFAULT_VERSION) + spec = _str_arg(args, "spec") + if spec is not None and spec not in SUPPORTED_SPECS: + spec = None + legacy_version = _str_arg(args, "version") + version = legacy_version or DEFAULT_VERSION version_info = get_version_info(version) use_full_schema = getattr(args, "full_schema", True) + # Load OPDS file + try: + path = Path(args.file) + if not path.exists(): + console_error(f"Error: File not found: {args.file}") + return 1 + + opds_data = _load_document(path) + except Exception as e: + console_error(f"Error loading OPDS file: {e}") + return 1 + + if spec in BITOL_SPEC_API_VERSIONS or ( + spec is None and legacy_version is None and _is_bitol_document(opds_data) + ): + return _validate_bitol(path, opds_data, logger, spec=spec) + logger.info( "opds_validate_start", extra={ @@ -191,16 +654,7 @@ def cmd_opds_validate(args: argparse.Namespace, logger: logging.Logger) -> int: }, ) - # Load OPDS file try: - path = Path(args.file) - if not path.exists(): - console_error(f"Error: File not found: {args.file}") - return 1 - - with open(path, encoding="utf-8") as f: - opds_data = json.load(f) - # Check if this is a wrapped format (from render()) if "artifacts" in opds_data and isinstance(opds_data["artifacts"], dict): logger.info("opds_validate_unwrap", extra={"format": "wrapped"}) @@ -285,21 +739,40 @@ def cmd_opds_info(args: argparse.Namespace, logger: logging.Logger) -> int: Display information about OPDS specification versions. Args: - args.version: Optional specific version to show info for + args.spec: Optional specification (``bitol-1.0.0`` / ``bitol-1.1.0`` / + ``odpi-4.1``) to show info for + args.version: Optional specific ODPI version to show info for (legacy) args.json: Output in JSON format Returns: 0 on success """ - if hasattr(args, "version") and args.version: + spec = _str_arg(args, "spec") + if spec in BITOL_SPECS: + spec_info = BITOL_SPECS[spec] + if getattr(args, "json", False): + _write_stdout(json.dumps({spec: spec_info}, indent=2)) + else: + cprint(f"Bitol ODPS {spec_info['apiVersion']} (--spec {spec})") + cprint("=" * 60) + cprint(f"Description: {spec_info['description']}") + cprint(f"Status: {spec_info['status']}") + cprint(f"Spec URL: {spec_info['spec_url']}") + cprint(f"Schema: vendored odps-product-{spec_info['apiVersion']}.json") + if spec_info.get("default"): + cprint("Default: Yes (fluid odps export)") + return 0 + + version = "4.1" if spec == SPEC_ODPI_4_1 else getattr(args, "version", None) + if version: # Show info for specific version try: - version_info = get_version_info(args.version) + version_info = get_version_info(version) if getattr(args, "json", False): - cprint(json.dumps({args.version: version_info}, indent=2)) + _write_stdout(json.dumps({version: version_info}, indent=2)) else: - cprint(f"OPDS Version {args.version}") + cprint(f"OPDS Version {version}") cprint("=" * 60) cprint(f"Description: {version_info['description']}") cprint(f"Status: {version_info['status']}") @@ -313,7 +786,7 @@ def cmd_opds_info(args: argparse.Namespace, logger: logging.Logger) -> int: else: # Show all versions if getattr(args, "json", False): - cprint(json.dumps(ODPS_VERSIONS, indent=2)) + _write_stdout(json.dumps(ODPS_VERSIONS, indent=2)) else: cprint("ODPS (Open Data Product Specification) Versions") cprint("=" * 60) @@ -328,9 +801,21 @@ def cmd_opds_info(args: argparse.Namespace, logger: logging.Logger) -> int: cprint(f" Schema: {info['schema_url']}") cprint() + cprint("Bitol ODPS (Open Data Product Standard) Specifications") + cprint("=" * 60) + cprint() + for spec_name, spec_info in BITOL_SPECS.items(): + default_marker = " (default for export)" if spec_info.get("default") else "" + cprint(f"--spec {spec_name}{default_marker}") + cprint(f" {spec_info['description']}") + cprint(f" Spec: {spec_info['spec_url']}") + cprint() + cprint("Usage:") - cprint(" fluid odps export contract.yaml --version 4.1") - cprint(" fluid odps validate output.json --version 4.1") + cprint(f" fluid odps export contract.yaml --out-dir out/ # {SPEC_BITOL_1_0_0}") + cprint(" fluid odps export contract.yaml --spec odpi-4.1 --out output.json") + cprint(" fluid odps validate output.json # auto-detects the spec") + cprint(" fluid odps info --spec odpi-4.1") return 0 @@ -338,18 +823,29 @@ def cmd_opds_info(args: argparse.Namespace, logger: logging.Logger) -> int: def register(subparsers: argparse._SubParsersAction) -> None: """Register ODPS commands with the CLI.""" - # Main ODPS command group + # Main ODPS command group. ``opds`` is registered as an alias so users + # can spell the command either way; both go through the --spec dispatcher. odps = subparsers.add_parser( "odps", - help="Export and validate ODPS (Open Data Product Specification) format", + aliases=["opds"], + help="Bitol ODPS v1.0.0 (default) or ODPI v4.1 — export, import, validate", description=""" - Work with ODPS (Open Data Product Specification) format. - - The Open Data Product Specification is a vendor-neutral, open-source standard - for describing data products. This command supports exporting FLUID contracts - to ODPS format and validating ODPS files. - - Official Specification: https://github.com/Open-Data-Product-Initiative + Work with the ODPS family of standards. + + Two specifications are supported, dispatched via --spec: + bitol-1.0.0 Bitol Open Data Product Standard v1.0.0 (default). + bitol-1.1.0 Bitol Open Data Product Standard v1.1.0. + Bidirectional. Emits 1 ODPS doc + N sibling ODCS + contracts. Import accepts a file, a directory, or + a lone ODCS file. + odpi-4.1 Open Data Product Initiative v4.1 (Linux Foundation). + Export-only single JSON document. + + validate auto-detects Bitol vs ODPI when --spec is omitted. + + Official Specifications: + Bitol ODPS: https://github.com/bitol-io/open-data-product-standard + ODPI v4.1: https://github.com/Open-Data-Product-Initiative/v4.1 """, ) @@ -360,63 +856,178 @@ def register(subparsers: argparse._SubParsersAction) -> None: "export", help="Export FLUID contract to ODPS format", description=""" - Export a FLUID contract to ODPS (Open Data Product Specification) JSON format. - - Supports multiple OPDS specification versions for future compatibility. - Output can be written to a file or stdout for pipeline integration. + Export a FLUID contract to ODPS. + + Bitol specs (default): --out-dir DIR writes the ODPS doc plus one + .odcs. file per output port; --out FILE writes the + ODPS doc to FILE and the ODCS files next to it; with neither, the + ODPS doc is printed to stdout as JSON. With --spec odpi-4.1 a single + ODPI v4.1 JSON document is written to --out (default: stdout). """, ) export.add_argument("contract", help="Path to FLUID contract file (YAML/JSON)") + export.add_argument( + "--spec", + default=None, + choices=list(SUPPORTED_SPECS), + help=( + f"Target specification (default: Bitol ODPS, {DEFAULT_SPEC} for fresh exports). " + f"{SPEC_BITOL_1_0_0} / {SPEC_BITOL_1_1_0} emit 1 ODPS doc + N sibling " + f"ODCS contracts declaring apiVersion v1.0.0 / v1.1.0; when omitted the " + f"version follows ODPS_API_VERSION, else the imported document's own " + f"apiVersion, else v1.0.0. " + f"{SPEC_ODPI_4_1} emits a single ODPI v4.1 JSON document." + ), + ) + # Deprecated alias — kept for back-compat. Hidden from --help. export.add_argument( "--version", - default=DEFAULT_VERSION, + default=None, choices=list(ODPS_VERSIONS.keys()), - help=f"ODPS specification version (default: {DEFAULT_VERSION})", + help=argparse.SUPPRESS, ) export.add_argument( - "--out", default="-", help="Output file path, or '-' for stdout (default: stdout)" + "--out", + default="-", + help=( + "Output file path, or '-' for stdout (default: stdout). For the Bitol specs " + "the per-port ODCS contracts are written next to it as .odcs.." + ), + ) + export.add_argument( + "--out-dir", + dest="out_dir", + default=None, + help=( + "For the Bitol specs: directory where the ODPS doc + per-port " + "ODCS files are written. Mutually exclusive with --out (use one or the other). " + "Rejected with --spec odpi-4.1." + ), + ) + export.add_argument( + "--format", + "-f", + default=None, + choices=["yaml", "json"], + help=( + "Bitol specs only: output format for file/dir writes (default: inferred from " + "the --out suffix — .json → json, .yaml/.yml → yaml — else yaml). Stdout " + "always uses JSON. ODPI v4.1 is always JSON; -f yaml is rejected with " + "--spec odpi-4.1." + ), ) export.add_argument("--env", help="Environment name for overlay application") export.add_argument( - "--validate", + "--validate-strict", + dest="validate_strict", action="store_true", default=True, - help="Validate output against OPDS schema (default: true)", + help="Validate the emitted ODPS + per-port ODCS docs against vendored schemas (default: true).", ) export.add_argument( - "--no-validate", dest="validate", action="store_false", help="Skip validation of output" + "--no-validate-strict", + dest="validate_strict", + action="store_false", + help=( + "Still validate, but report schema problems as warnings on stderr " + "instead of failing the export." + ), ) export.add_argument( "--pretty", action="store_true", default=True, - help="Pretty-print JSON output (default: true)", + help="(ODPI only) Pretty-print JSON output (default: true)", ) export.add_argument( "--compact", dest="pretty", action="store_false", - help="Compact JSON output (no indentation)", + help="(ODPI only) Compact JSON output (no indentation).", ) export.set_defaults(func=cmd_opds_export) + # odps import + importp = odps_sub.add_parser( + "import", + help="Import an ODPS / ODCS file or directory → one FLUID contract", + description=( + "Import a Bitol ODPS product file, a directory containing the ODPS " + "doc + sibling ODCS files (or just ODCS files), or a single ODCS " + "contract file. The inputs are schema-validated; the output is one FLUID " + "contract skeleton that may need edits (fluidVersion/kind/id/name, expose " + "kind, binding format/location) before `fluid validate` / `fluid plan` pass." + ), + ) + importp.add_argument( + "path", + help="Input path — a .odps.yaml file, a directory, or a .odcs.yaml file.", + ) + importp.add_argument( + "--spec", + default=None, + choices=list(SUPPORTED_SPECS), + help=( + f"Specification (default: {DEFAULT_SPEC}). Either Bitol spec accepts " + f"v1.0.0 and v1.1.0 documents on import; {SPEC_ODPI_4_1} is export-only." + ), + ) + importp.add_argument( + "-o", "--out", default=None, help="Output FLUID file path (default: stdout)" + ) + importp.add_argument( + "-f", + "--format", + default=None, + choices=["yaml", "json"], + help=( + "Output format (default: inferred from the -o suffix — .json → json, " + ".yaml/.yml → yaml — else yaml; stdout defaults to yaml)." + ), + ) + importp.add_argument( + "--no-remote", + dest="no_remote", + action="store_true", + help="Disable http(s) fetch when resolving contractId references.", + ) + importp.add_argument( + "--lenient", + action="store_true", + help=( + "Downgrade output-port resolution failures, and schema-invalid ODCS " + "inputs, to warnings (input ports are always lenient)." + ), + ) + importp.set_defaults(func=cmd_opds_import) + # odps validate validate = odps_sub.add_parser( "validate", help="Validate an ODPS file against specification", description=""" - Validate an ODPS JSON file against the official specification schema. - - Performs structural validation and checks required fields according to - the ODPS specification version. + Validate an ODPS file (YAML or JSON) against its specification schema. + + With --spec omitted the document is auto-detected: one declaring + apiVersion and kind: DataProduct is validated as Bitol ODPS (schema + chosen from its apiVersion); anything else as ODPI v4.1. """, ) - validate.add_argument("file", help="Path to ODPS JSON file") + validate.add_argument("file", help="Path to ODPS file (YAML or JSON)") + validate.add_argument( + "--spec", + default=None, + choices=list(SUPPORTED_SPECS), + help=( + "Specification to validate against (default: auto-detect). An explicit " + "Bitol spec pins the schema version." + ), + ) validate.add_argument( "--version", - default=DEFAULT_VERSION, + default=None, choices=list(ODPS_VERSIONS.keys()), - help=f"ODPS version to validate against (default: {DEFAULT_VERSION})", + help=f"ODPI version to validate against (forces ODPI; default: {DEFAULT_VERSION})", ) validate.add_argument( "--full-schema", @@ -438,10 +1049,16 @@ def register(subparsers: argparse._SubParsersAction) -> None: help="Display ODPS specification version information", description=""" Show information about supported ODPS specification versions. - + Displays specification URLs, schema references, and version status. """, ) + info.add_argument( + "--spec", + default=None, + choices=list(SUPPORTED_SPECS), + help="Show info for one specification only", + ) info.add_argument( "--version", choices=list(ODPS_VERSIONS.keys()), help="Show info for specific version only" ) diff --git a/fluid_build/cli/publish.py b/fluid_build/cli/publish.py index d9ca1211..8032aad7 100644 --- a/fluid_build/cli/publish.py +++ b/fluid_build/cli/publish.py @@ -38,6 +38,7 @@ import argparse import asyncio import logging +import warnings from pathlib import Path from typing import List, Optional @@ -54,7 +55,13 @@ from ..config_manager import FluidConfig from ..loader import load_contract -from ..providers.catalogs import PublishResult, get_catalog_provider +from ..providers.catalogs import ( + BaseCatalogProvider, + CatalogTarget, + PublishResult, + get_catalog_provider, +) +from ..providers.catalogs.base import _fluid_identity_error, _publish_declares_target from ..providers.common import metrics_collector COMMAND = "publish" @@ -132,9 +139,14 @@ def register(subparsers: argparse._SubParsersAction): catalog_group = p.add_argument_group("Catalog Selection") catalog_group.add_argument( "--catalog", + "--target", "-c", default="fluid-command-center", - help="Target catalog name (default: fluid-command-center)", + help=( + "Target catalog name (default: fluid-command-center). " + "``--target`` is accepted as an alias for ecosystem tooling " + "(e.g. snowflake-biz-lab's Taskfile) that uses that flag name." + ), ) catalog_group.add_argument( "--list-catalogs", action="store_true", help="List configured catalogs and exit" @@ -223,6 +235,14 @@ async def publish_contract( error=f"Failed to load contract: {e}", ) + if not isinstance(contract, dict): + return PublishResult( + success=False, + catalog_id=catalog_name, + asset_id=str(contract_path), + error=_fluid_identity_error(contract), + ) + # Get catalog config catalog_config = config.get_catalog_config(catalog_name) if not catalog_config: @@ -252,11 +272,142 @@ async def publish_contract( error=f"Failed to create catalog provider: {e}", ) - # Map contract to asset + # Build publish target from the FLUID dict — the raw contract stays the + # payload, the target just names it and carries the byte-preserving + # contract YAML for catalogs that store the original file. try: - asset = provider.map_contract_to_asset(contract) - # Attach raw contract YAML so catalogs can store the full file - asset.contract_yaml = contract_path.read_text(encoding="utf-8") + contract_yaml = contract_path.read_text(encoding="utf-8") + except Exception: + contract_yaml = None + target = CatalogTarget.from_fluid(contract, contract_yaml=contract_yaml) + + # Third-party catalog plugins written against the pre-0.7.9 API + # (``publish(asset)`` / ``update(asset)`` / ``validate_asset(asset)``) + # keep working for one release through the old CatalogAsset flow. + if _uses_legacy_asset_api(provider): + return await _publish_contract_legacy( + provider, + contract, + contract_path=contract_path, + contract_yaml=contract_yaml, + catalog_name=catalog_name, + dry_run=dry_run, + verify_only=verify_only, + skip_health_check=skip_health_check, + verbose=verbose, + ) + + # Contract fragments, overlays and config files are not publishable — + # 0.7.8 rejected them while mapping; don't upsert them as ``unknown``. + identity_error = _fluid_identity_error(contract) + if identity_error: + return PublishResult( + success=False, + catalog_id=catalog_name, + asset_id=str(contract_path), + error=identity_error, + ) + + if verbose: + logger.info(f"📦 Publishing contract {target.contract_name} (id: {target.contract_id})") + + # Verify-only mode + if verify_only: + exists = await provider.verify(target.contract_id) + return PublishResult( + success=exists, + catalog_id=catalog_name, + asset_id=target.contract_id, + error=None if exists else "Asset not found in catalog", + details={"verified": exists, "operation": "verify"}, + ) + + # Dry-run mode + if dry_run: + is_valid, error_msg = provider.validate_target(contract, target) + return PublishResult( + success=is_valid, + catalog_id=catalog_name, + asset_id=target.contract_id, + error=error_msg, + details={"dry_run": True, "valid": is_valid}, + ) + + # Health check (unless skipped) + if not skip_health_check: + if verbose: + logger.info("🏥 Checking catalog health...") + + is_healthy = await provider.health_check() + if not is_healthy: + return PublishResult( + success=False, + catalog_id=catalog_name, + asset_id=target.contract_id, + error="Catalog health check failed - endpoint not accessible", + ) + + # Publish! + if verbose: + logger.info(f"🚀 Publishing to {catalog_name}...") + + result = await provider.publish(contract, target) + return result + + +def _uses_legacy_asset_api(provider: object) -> bool: + """True for a catalog provider still implementing the 0.7.8 API. + + Detection is positive for the current API: a provider is new-style only + when its ``publish`` declares ``(fluid, target)`` (second parameter + named ``target`` or annotated ``CatalogTarget``). Everything else — the + 0.7.8 ``publish(asset)`` signature with or without extra optional + parameters / ``*args`` / ``**kwargs``, mocks, and duck-typed plugins that + don't subclass :class:`BaseCatalogProvider` — takes the CatalogAsset + route. A duck-typed object is treated as new-style only when it also + provides ``validate_target`` (the legacy route would need + ``map_contract_to_asset``, which such an object doesn't have). + """ + cls = type(provider) + if isinstance(provider, BaseCatalogProvider): + return cls._uses_legacy_asset_api() + declares_new_api = _publish_declares_target(cls) and callable( + getattr(provider, "validate_target", None) + ) + return not declares_new_api + + +async def _publish_contract_legacy( + provider: BaseCatalogProvider, + contract: dict, + *, + contract_path: Path, + contract_yaml: Optional[str], + catalog_name: str, + dry_run: bool, + verify_only: bool, + skip_health_check: bool, + verbose: bool, +) -> PublishResult: + """Run the pre-0.7.9 CatalogAsset publish flow for a legacy plugin. + + Mirrors the old ``publish_contract`` exactly: ``map_contract_to_asset`` + (the plugin's override when present) → ``validate_asset`` for + ``--dry-run`` → health check → ``publish(asset)``. + """ + warnings.warn( + f"Catalog provider {type(provider).__name__} implements the deprecated " + "publish(asset) API; implement publish(fluid, target) instead. " + "Legacy CatalogAsset providers will stop working in a future release.", + DeprecationWarning, + stacklevel=2, + ) + try: + with warnings.catch_warnings(): + # The single deprecation notice above is enough. + warnings.simplefilter("ignore", DeprecationWarning) + asset = provider.map_contract_to_asset(contract) + asset.contract_yaml = contract_yaml except Exception as e: return PublishResult( success=False, @@ -268,7 +419,6 @@ async def publish_contract( if verbose: logger.info(f"📦 Mapped contract to asset: {asset.name} (ID: {asset.id})") - # Verify-only mode if verify_only: exists = await provider.verify(asset.id) return PublishResult( @@ -279,7 +429,6 @@ async def publish_contract( details={"verified": exists, "operation": "verify"}, ) - # Dry-run mode if dry_run: is_valid, error_msg = provider.validate_asset(asset) return PublishResult( @@ -290,13 +439,10 @@ async def publish_contract( details={"dry_run": True, "valid": is_valid}, ) - # Health check (unless skipped) if not skip_health_check: if verbose: logger.info("🏥 Checking catalog health...") - - is_healthy = await provider.health_check() - if not is_healthy: + if not await provider.health_check(): return PublishResult( success=False, catalog_id=catalog_name, @@ -304,12 +450,10 @@ async def publish_contract( error="Catalog health check failed - endpoint not accessible", ) - # Publish! if verbose: logger.info(f"🚀 Publishing to {catalog_name}...") - result = await provider.publish(asset) - return result + return await provider.publish(asset) # type: ignore[call-arg] def format_results( diff --git a/fluid_build/config_manager.py b/fluid_build/config_manager.py index 886cf5b6..5f60350d 100644 --- a/fluid_build/config_manager.py +++ b/fluid_build/config_manager.py @@ -18,16 +18,25 @@ Provides hierarchical configuration loading from multiple sources: 1. Default configuration (built-in) 2. System-wide configuration (/etc/fluid/config.yaml or C:\\ProgramData\\fluid\\config.yaml) -3. User configuration (~/.fluidrc.yaml or ~/.config/fluid/config.yaml) +3. User configuration: ~/.fluid/config.yaml (lowest-priority user layer), + then the first found of ~/.fluidrc.yaml, ~/.fluidrc or + ~/.config/fluid/config.yaml merged on top of it 4. Project configuration (.fluidrc.yaml or fluid.config.yaml in current directory) 5. Environment variables (FLUID_*) 6. Command-line arguments (highest priority) + +String values may reference environment variables as ``${VAR}``. The +placeholders are stored as written and expanded when a value is read +(``get``/``get_section``/``get_catalog_config``/``to_dict``), so +``save_user_config()`` never writes resolved secrets back to disk. """ from __future__ import annotations +import copy import logging import os +import re from pathlib import Path from typing import Any, Dict, Optional @@ -37,6 +46,20 @@ LOGGER = logging.getLogger("fluid.config") +# ``${VAR}`` references inside config string values. +_ENV_REF_RE = re.compile(r"\$\{([A-Za-z_][A-Za-z0-9_]*)\}") + + +def _has_env_ref(value: Any) -> bool: + """True when *value* (recursively) contains a ``${VAR}`` reference.""" + if isinstance(value, str): + return _ENV_REF_RE.search(value) is not None + if isinstance(value, dict): + return any(_has_env_ref(v) for v in value.values()) + if isinstance(value, list): + return any(_has_env_ref(v) for v in value) + return False + # Default configuration DEFAULT_CONFIG = { @@ -104,7 +127,10 @@ def __init__(self): def _load_defaults(self) -> None: """Load default configuration.""" - self._config = DEFAULT_CONFIG.copy() + # Deep copy: ``_deep_merge`` mutates nested dicts in place, and a + # shallow copy would leak file config into the module-level defaults + # (and from there into every later FluidConfig in the process). + self._config = copy.deepcopy(DEFAULT_CONFIG) LOGGER.debug("Loaded default configuration") def _load_system_config(self) -> None: @@ -124,7 +150,13 @@ def _load_user_config(self) -> None: """Load user configuration.""" home = Path.home() - # Try multiple locations + # ``~/.fluid/config.yaml`` (the path ``fluid publish --help`` documents + # and the snowflake-biz-lab bootstrap writes) is an additional, + # lowest-priority user layer. It is merged first so it never hides + # the legacy user files below, which keep their original first-found + # order and are deep-merged on top of it. + self._merge_config_file(home / ".fluid" / "config.yaml", "user") + user_configs = [ home / ".fluidrc.yaml", home / ".fluidrc", @@ -153,6 +185,24 @@ def _load_project_config(self) -> None: self._merge_config_file(config_path, "project") break + def _expand_env_in_config(self, value: Any) -> Any: + """Recursively expand ``${VAR}`` references in string values. + + Mirrors ``os.path.expandvars`` semantics — undefined vars stay literal + so misconfiguration is visible. Returns new containers; the input is + never modified. + """ + if isinstance(value, str): + return _ENV_REF_RE.sub( + lambda m: os.environ.get(m.group(1), m.group(0)), + value, + ) + if isinstance(value, dict): + return {k: self._expand_env_in_config(v) for k, v in value.items()} + if isinstance(value, list): + return [self._expand_env_in_config(v) for v in value] + return value + def _load_env_vars(self) -> None: """Load configuration from environment variables.""" env_mappings = { @@ -210,6 +260,9 @@ def _merge_config_file(self, path: Path, source: str) -> None: with open(path) as f: file_config = yaml.safe_load(f) or {} + # Shell-style ``${VAR}`` references are kept verbatim here and + # expanded on read (see ``get``), so ``save_user_config`` writes the + # placeholders back instead of the resolved secrets. self._deep_merge(self._config, file_config) LOGGER.debug(f"Loaded {source} configuration from {path}") @@ -287,6 +340,11 @@ def get(self, path: str, default: Any = None) -> Any: else: return default + # ``${VAR}`` placeholders are expanded on read (into a copy) so the + # stored config — and anything ``save_user_config`` writes — keeps + # the placeholder rather than the resolved secret. + if _has_env_ref(value): + return self._expand_env_in_config(value) return value def set(self, path: str, value: Any) -> None: @@ -331,7 +389,10 @@ def get_catalog_config(self, catalog_name: Optional[str] = None) -> Dict[str, An catalogs = self.get_section("catalogs") if catalog_name: - catalog_config = catalogs.get(catalog_name, {}) + # Work on a copy so the env-var overrides below (which can carry + # secrets such as FLUID_API_KEY) never leak into the stored + # config that ``save_user_config`` writes. + catalog_config = copy.deepcopy(catalogs.get(catalog_name, {})) # Apply environment variable overrides if catalog_name == "fluid-command-center": @@ -354,9 +415,9 @@ def to_dict(self) -> Dict[str, Any]: Get the complete configuration as a dictionary. Returns: - Complete configuration dictionary + Complete configuration dictionary (``${VAR}`` references expanded) """ - return self._config.copy() + return self._expand_env_in_config(self._config) def update_from_args(self, **kwargs) -> None: """ diff --git a/fluid_build/features.yaml b/fluid_build/features.yaml index 6c4db899..27d5fd47 100644 --- a/fluid_build/features.yaml +++ b/fluid_build/features.yaml @@ -110,62 +110,64 @@ providers: resource_types: 9 architecture: "Full BaseProvider with 5-phase planning" + # ODPI v4.1 — Open Data Product Initiative (Linux Foundation). Export-only. + # Distinct from Bitol Open Data Product Standard v1.0.0 below. odps: # MANUAL CONTROL - status: alpha - note: "Experimental - MaxCompute support in early stages" + status: beta + note: "Open Data Product Initiative v4.1 (Linux Foundation) exporter" blockers: - - "Very early stage - needs foundational work" - - "Increase test coverage from 30%" - - "Complete basic resource types" - target_release: "future" - + - "Refactor to use BaseProvider pattern" + - "Implement proper plan() and apply() methods" + - "Add basic error handling and retry logic" + target_release: "1.1.0" + # AUTO-UPDATED test_coverage: 30 code_size: 1351 last_tested: "2026-01-20T00:00:00Z" - + # Provider metadata docs_complete: false resource_types: 2 - architecture: "Legacy pattern (needs refactor to BaseProvider)" - blockers: - - "Refactor to use BaseProvider pattern" - - "Implement proper plan() and apply() methods" - - "Add basic error handling and retry logic" - - "Write README with setup instructions" - - "Add at least 1 integration test" - target_release: "1.1.0" - + architecture: "ODPI v4.1 exporter (FLUID → ODPI JSON)" + + # Bitol Open Data Product Standard v1.0.0 / v1.1.0 — bidirectional provider with + # contractId resolution to per-port ODCS contracts. The key is the package + # directory (fluid_build/providers/odps_standard/), which is what the maturity + # scripts and build-manifest.yaml use; the provider registers as ``odps_bitol`` + # (plus the legacy ``odps-standard`` alias). odps_standard: # MANUAL CONTROL status: beta - note: "ODPS v1.0.0 export provider - production ready, well tested" + note: "Bitol ODPS v1.0.0 / v1.1.0 bidirectional, lossless provider (registry name: odps_bitol) with contractId resolver; validation is version-selected from apiVersion" blockers: - "Complete documentation with real-world examples" - - "Add API integration tests (when backend available)" + - "Add property-based round-trip tests" target_release: "0.9.0" - + # AUTO-UPDATED - test_coverage: 100 # 25/25 tests passing - code_size: 479 - last_tested: "2026-01-20T00:00:00Z" - + test_coverage: 100 # 22 legacy tests + Phase 3 round-trip smoke + code_size: 1450 # provider + mappers + resolver + validation + last_tested: "2026-05-23T00:00:00Z" + # Provider metadata docs_complete: true - resource_types: 0 # Export-only provider - architecture: "Export provider (FLUID → ODPS Standard v1.0.0)" + resource_types: 0 # Conversion provider + architecture: "Bidirectional (FLUID ↔ Bitol ODPS v1.0.0 / v1.1.0 + per-port ODCS); verbatim stash + native overlay for lossless round-trip" features: - - "Export FLUID contracts to Bitol.io ODPS format" - - "Entropy Data marketplace integration" - - "PII detection and classification" - - "Team/owner extraction" - - "Unicode and special character support" + - "Export 1 ODPS doc + N sibling .odcs.yaml files" + - "Linking invariant: port.contractId == odcs.id" + - "Three import entry points: file, directory, lone ODCS" + - "ContractResolver with local + http(s) probing and caching" + - "Strict JSON Schema validation by default" + - "Verbatim round-trip via metadata.odps_passthrough.* namespace" + - "Legacy OdpsStandardProvider kept as back-compat alias" odcs: # MANUAL CONTROL status: beta - note: "ODCS v3.1.0 bidirectional provider - production ready, comprehensive testing" + note: "ODCS v3.1.0 / v3.2.0 bidirectional, lossless provider - validated against verbatim upstream Bitol examples; validation is version-selected from apiVersion" blockers: - "Add more real-world round-trip test cases" - "Complete documentation with examples" @@ -179,7 +181,7 @@ providers: # Provider metadata docs_complete: true resource_types: 0 # Conversion provider - architecture: "Bidirectional conversion (FLUID ↔ ODCS v3.1.0)" + architecture: "Bidirectional conversion (FLUID ↔ ODCS v3.1.0 / v3.2.0); verbatim stash + native overlay for lossless round-trip" features: - "Import ODCS contracts to FLUID" - "Export FLUID contracts to ODCS" @@ -375,7 +377,7 @@ release_history: profile: beta highlights: - "ODPS Standard v1.0.0 export provider (100% coverage)" - - "ODCS v3.1.0 bidirectional conversion (100% coverage)" + - "ODCS v3.1.0 / v3.2.0 bidirectional conversion (100% coverage)" - "Data Mesh Manager publisher (100% coverage)" - "76 comprehensive tests across all standards providers" providers: [local, gcp, odps_standard, odcs, datamesh_manager] diff --git a/fluid_build/providers/catalogs/__init__.py b/fluid_build/providers/catalogs/__init__.py index 67f32e67..305c73e1 100644 --- a/fluid_build/providers/catalogs/__init__.py +++ b/fluid_build/providers/catalogs/__init__.py @@ -14,7 +14,7 @@ """Catalog provider registry""" -from .base import BaseCatalogProvider, CatalogAsset, PublishResult +from .base import BaseCatalogProvider, CatalogAsset, CatalogTarget, PublishResult from .fluid_cc import FluidCommandCenterProvider # Lazy-import optional catalog backends ─ don't crash if deps are missing @@ -62,6 +62,7 @@ def get_catalog_provider(catalog_type: str, config: dict) -> BaseCatalogProvider __all__ = [ "BaseCatalogProvider", "CatalogAsset", + "CatalogTarget", "PublishResult", "FluidCommandCenterProvider", "CATALOG_PROVIDERS", diff --git a/fluid_build/providers/catalogs/base.py b/fluid_build/providers/catalogs/base.py index 42eb27ee..a642b5b5 100644 --- a/fluid_build/providers/catalogs/base.py +++ b/fluid_build/providers/catalogs/base.py @@ -12,46 +12,108 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Base catalog provider for marketplace integrations +"""Base catalog provider for marketplace integrations. -Similar to market.py's BaseCatalogConnector but focused on WRITE operations (publish, update, delete). +The primary API is :meth:`BaseCatalogProvider.publish`, which accepts a raw +FLUID contract dict plus a thin :class:`CatalogTarget` descriptor. The old +:class:`CatalogAsset` DTO is kept as a deprecated builder for callers that +still construct one by hand. """ +import copy +import dataclasses +import inspect import logging +import warnings from abc import ABC, abstractmethod from dataclasses import dataclass, field from datetime import datetime -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Optional, Tuple + + +@dataclass +class CatalogTarget: + """Thin publish descriptor — the FLUID dict is the payload, this carries + identity + optional per-catalog hints. + + Providers are expected to read structural information (exposes, schema, + owner, tags, etc.) from the FLUID dict passed alongside this descriptor. + The descriptor itself only names the target and offers a couple of + convenience fields that would otherwise force every provider to + re-derive them. + """ + + contract_id: str + contract_name: str = "" + contract_yaml: Optional[str] = None + catalog_hints: Dict[str, Any] = field(default_factory=dict) + # Set only on the deprecated CatalogAsset paths: the asset the caller + # passed (or a 0.7.8 ``map_contract_to_asset`` override produced), so the + # built-in providers can build their payload from its fields exactly as + # 0.7.8 did. Not part of the public constructor. + _legacy_asset: Optional["CatalogAsset"] = field( + default=None, init=False, repr=False, compare=False + ) + + @classmethod + def from_fluid( + cls, + fluid: Dict[str, Any], + *, + contract_yaml: Optional[str] = None, + catalog_hints: Optional[Dict[str, Any]] = None, + ) -> "CatalogTarget": + """Build a target descriptor from a FLUID dict. + + ``contract_id`` falls back to the top-level ``name`` and then to + ``metadata.name`` so a FLUID document that omits the id (rare, but + legal) still produces a usable descriptor. + """ + metadata = fluid.get("metadata") or {} + contract_id = fluid.get("id") or fluid.get("name") or metadata.get("name") or "unknown" + contract_name = fluid.get("name") or metadata.get("name") or fluid.get("id") or "unknown" + return cls( + contract_id=str(contract_id), + contract_name=str(contract_name), + contract_yaml=contract_yaml, + catalog_hints=dict(catalog_hints or {}), + ) @dataclass class CatalogAsset: - """Normalized asset representation for catalog publishing + """DEPRECATED: flat asset DTO retained for one release for back-compat. - This is the standardized format that all catalog providers accept. - Contract data is mapped to this format before publishing. + New code should call :meth:`BaseCatalogProvider.publish` with the raw + FLUID dict + a :class:`CatalogTarget`. This class remains so external + callers that build a CatalogAsset by hand continue to work through the + :meth:`BaseCatalogProvider.publish_asset` shim. """ - id: str # contract.id - name: str # contract.name - description: str # contract.description - type: str # contract.kind (e.g., 'DataProduct', 'DataStream') - domain: str # contract.domain - owner: str # contract.metadata.owner.team - owner_email: str # contract.metadata.owner.email - layer: str # contract.metadata.layer (Bronze/Silver/Gold) - tags: List[str] # contract.metadata.tags - version: str # contract.version - platform: str # gcp, aws, snowflake - location: Dict[str, Any] # Platform-specific location info - schema: Optional[List[Dict[str, Any]]] = None # From exposes[0].contract.schema - sensitivity: str = "internal" # internal, public, confidential - contract_yaml: Optional[str] = None # Raw YAML content of the contract file + id: str + name: str + description: str + type: str + domain: str + owner: str + owner_email: str + layer: str + tags: List[str] + version: str + platform: str + location: Dict[str, Any] + schema: Optional[List[Dict[str, Any]]] = None + sensitivity: str = "internal" + contract_yaml: Optional[str] = None + # The full FLUID dict — set by :meth:`BaseCatalogProvider.map_contract_to_asset` + # so the deprecated ``publish_asset`` shim can round-trip the original + # payload rather than re-synthesising it from the flat fields. + raw_contract: Optional[Dict[str, Any]] = None @dataclass class PublishResult: - """Result of catalog publish operation""" + """Result of a catalog publish operation.""" success: bool catalog_id: str @@ -63,161 +125,546 @@ class PublishResult: class BaseCatalogProvider(ABC): - """Base class for all catalog integrations - - Provides common functionality: - - Contract to asset mapping - - Authentication setup - - Error handling patterns - - Logging + """Base class for all catalog integrations. - Subclasses implement catalog-specific operations. + Subclasses implement :meth:`publish`, :meth:`update`, :meth:`verify`. + The base offers a default :meth:`validate_target` and back-compat shims + (``publish_asset``, ``update_asset``, ``map_contract_to_asset``, + ``validate_asset``) for the deprecated CatalogAsset flow. """ - name: str = "base" # Override in subclasses + name: str = "base" def __init__(self, config: Dict[str, Any]): self.config = config self.endpoint = config.get("endpoint", config.get("url", "")) self.auth = config.get("auth", {}) self.logger = logging.getLogger(f"{__name__}.{self.__class__.__name__}") - - # Resilience settings self.max_retries = config.get("max_retries", 3) self.timeout = config.get("timeout", 30.0) + # -- primary API -------------------------------------------------------- + @abstractmethod - async def publish(self, asset: CatalogAsset) -> PublishResult: - """Publish asset to catalog + async def publish( + self, fluid: Dict[str, Any], target: Optional[CatalogTarget] = None + ) -> PublishResult: + """Publish a FLUID contract to the catalog. Args: - asset: Normalized asset to publish + fluid: The raw FLUID contract dict — providers should treat this + as the source of truth for schema, output ports, metadata, + owner, etc. + target: Identity + per-catalog hints for this publish call. + Defaults to :meth:`CatalogTarget.from_fluid`. Returns: - PublishResult with success/failure details - """ - pass + PublishResult with success/failure details. - @abstractmethod - async def update(self, asset: CatalogAsset) -> PublishResult: - """Update existing asset in catalog - - Args: - asset: Normalized asset with updated information + Implementations should start with + ``fluid, target = self._coerce_publish_args(fluid, target, "publish")`` + so callers still using the deprecated ``publish(asset)`` form (a + single :class:`CatalogAsset`) keep working for one release. - Returns: - PublishResult with success/failure details + The second parameter must be named ``target`` (or be annotated + :class:`CatalogTarget`): that is how ``fluid publish`` tells a + provider implementing this API from one still implementing the + 0.7.8 ``publish(asset)`` API. """ - pass + ... @abstractmethod - async def verify(self, asset_id: str) -> bool: - """Verify asset exists in catalog - - Args: - asset_id: Unique identifier of the asset + async def update( + self, fluid: Dict[str, Any], target: Optional[CatalogTarget] = None + ) -> PublishResult: + """Update an existing asset in the catalog. - Returns: - True if asset exists, False otherwise + Same contract as :meth:`publish`; most providers treat PUT as + idempotent and simply delegate to publish. """ - pass - - async def delete(self, asset_id: str) -> bool: - """Delete asset from catalog (optional) + ... - Args: - asset_id: Unique identifier of the asset + @abstractmethod + async def verify( + self, contract_id: Optional[str] = None, *, asset_id: Optional[str] = None + ) -> bool: + """Verify an asset with this contract id exists in the catalog. - Returns: - True if deleted, False otherwise + ``asset_id=`` is the deprecated 0.7.8 keyword for ``contract_id``; + implementations should resolve both with :func:`_coerce_contract_id`. """ + ... + + async def delete( + self, contract_id: Optional[str] = None, *, asset_id: Optional[str] = None + ) -> bool: + """Delete an asset from the catalog (optional).""" + _coerce_contract_id(contract_id, asset_id, "delete") self.logger.warning(f"Delete not implemented for {self.name}") return False - def map_contract_to_asset(self, contract: Dict[str, Any]) -> CatalogAsset: - """Map FLUID contract to catalog asset + def validate_target( + self, fluid: Dict[str, Any], target: CatalogTarget + ) -> Tuple[bool, Optional[str]]: + """Validate a (fluid, target) pair before publishing. + + The FLUID dict must identify a contract (``id``, ``name`` or + ``metadata.name``) — anything else (a contract fragment, an overlay, + a config file) is rejected, as 0.7.8 effectively did. A missing + ``domain`` or owner is *not* an error — the payload builders fall + back to ``"general"`` / ``"unknown"`` exactly as the 0.7.8 + ``map_contract_to_asset`` + ``validate_asset`` flow did, so + schema-valid contracts that omit them still publish. + + A subclass that still overrides the deprecated + :meth:`map_contract_to_asset` or :meth:`validate_asset` hooks keeps + gating publishes: ``validate_asset`` is consulted with the asset the + (possibly overridden) mapper produces. + """ + if not target.contract_id: + return False, "contract id is required" + if not target.contract_name: + return False, "contract name is required" + identity_error = _fluid_identity_error(fluid) + if identity_error: + return False, identity_error + if self._overrides_legacy_mapper() or ( + type(self).validate_asset is not BaseCatalogProvider.validate_asset + ): + # A (possibly overridden) 0.7.8 mapper may raise on contracts it + # cannot map; report that per contract, as 0.7.8 and the real + # publish path do, instead of aborting a whole --dry-run batch. + try: + asset = self._legacy_asset_from_fluid(fluid, target) + except Exception as exc: + return False, f"Failed to map contract to asset: {exc}" + return self.validate_asset(asset) + return True, None - This is a common mapping that works for most contracts. - Subclasses can override for custom mapping logic. + # -- deprecated-API compatibility helpers ------------------------------- - Args: - contract: FLUID contract dictionary + @classmethod + def _uses_legacy_asset_api(cls) -> bool: + """True unless ``cls.publish`` positively declares the current + ``publish(fluid, target)`` API (see :func:`_publish_declares_target`). - Returns: - CatalogAsset instance + Anything else — the 0.7.8 ``publish(asset)`` signature, with or + without extra optional parameters or ``*args`` / ``**kwargs`` — is a + third-party catalog plugin written against 0.7.8; ``fluid publish`` + routes it through ``map_contract_to_asset`` → ``validate_asset`` → + ``publish(asset)``. """ - metadata = contract.get("metadata", {}) - owner = metadata.get("owner", {}) - exposes = contract.get("exposes", []) - - # Extract platform and location from binding - platform = "unknown" - location = {} - schema = None - sensitivity = "internal" - - if exposes: - first_expose = exposes[0] - binding = first_expose.get("binding", {}) - platform = binding.get("platform", "unknown") - location = binding.get("location", {}) - - # Extract schema - contract_spec = first_expose.get("contract", {}) - schema = contract_spec.get("schema") - - # Extract sensitivity - sensitivity = first_expose.get("sensitivity", "internal") - - return CatalogAsset( - id=contract.get("id", contract.get("name", "unknown")), - name=contract["name"], - description=contract.get("description", ""), - type=contract.get("kind", "DataProduct").lower(), - domain=contract.get("domain", "general"), - owner=owner.get("team", owner.get("name", "unknown")), - owner_email=owner.get("email", ""), - layer=metadata.get("layer", "Bronze"), - tags=metadata.get("tags", []), - version=contract.get("version", "1.0.0"), - platform=platform, - location=location, - schema=schema, - sensitivity=sensitivity, + return not _publish_declares_target(cls) + + def _overrides_legacy_mapper(self) -> bool: + return type(self).map_contract_to_asset is not BaseCatalogProvider.map_contract_to_asset + + def _legacy_asset_from_fluid( + self, fluid: Dict[str, Any], target: Optional[CatalogTarget] = None + ) -> CatalogAsset: + """Build the flat asset a legacy hook expects, honouring a subclass's + own :meth:`map_contract_to_asset` override (without re-emitting the + base deprecation warning).""" + if self._overrides_legacy_mapper(): + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + asset = self.map_contract_to_asset(fluid) + else: + asset = _build_catalog_asset_from_fluid(fluid) + if target is not None and target.contract_yaml and not asset.contract_yaml: + asset.contract_yaml = target.contract_yaml + return asset + + def _with_legacy_asset(self, fluid: Dict[str, Any], target: CatalogTarget) -> CatalogTarget: + """Attach the CatalogAsset a built-in provider must publish from. + + Returns *target* unchanged when it already carries the caller's + asset (deprecated ``publish(asset)`` / ``publish_asset``) or when + the provider does not override :meth:`map_contract_to_asset`. + Otherwise — a 0.7.8 subclass customising the mapping — the override + is run (with a :class:`DeprecationWarning`) and a copy of *target* + carrying its asset is returned, so the customisation reaches the + published payload as it did in 0.7.8. Exceptions from the override + propagate to the caller. + """ + if target._legacy_asset is not None or not self._overrides_legacy_mapper(): + return target + warnings.warn( + f"{type(self).__name__}.map_contract_to_asset() overrides are deprecated " + "and are honoured for one release only; build the payload from the " + "FLUID dict in publish(fluid, target) instead.", + DeprecationWarning, + stacklevel=3, ) + asset = self._legacy_asset_from_fluid(fluid, target) + new_target = dataclasses.replace(target) + new_target._legacy_asset = asset + return new_target + + def _coerce_publish_args( + self, + fluid: Any, + target: Optional[CatalogTarget], + method: str = "publish", + asset: Optional[CatalogAsset] = None, + ) -> Tuple[Dict[str, Any], CatalogTarget]: + """Normalise ``publish``/``update`` arguments. + + Accepts the current ``(fluid, target)`` form (``target`` defaults to + :meth:`CatalogTarget.from_fluid`) and the deprecated 0.7.8 forms + ``(asset)`` / ``(asset=asset)``, which emit a + :class:`DeprecationWarning` and are converted to the equivalent + ``(fluid, target)`` pair (the target remembers the asset, see + :attr:`CatalogTarget._legacy_asset`). + """ + if asset is not None: + if fluid is not None: + raise TypeError(f"{method}() got values for both 'fluid' and 'asset'") + fluid = asset + if fluid is None: + raise TypeError(f"{method}() missing 1 required positional argument: 'fluid'") + if isinstance(fluid, CatalogAsset): + warnings.warn( + f"{method}(CatalogAsset) is deprecated; call " + f"{method}(fluid, CatalogTarget.from_fluid(fluid)) with the raw " + "FLUID dict instead. The CatalogAsset form will be removed in a " + "future release.", + DeprecationWarning, + stacklevel=3, + ) + legacy_fluid, legacy_target = _fluid_target_from_asset(fluid) + if target is not None and target.catalog_hints: + legacy_target.catalog_hints = dict(target.catalog_hints) + return legacy_fluid, legacy_target + if target is None: + target = CatalogTarget.from_fluid(fluid) + return fluid, target - def validate_asset(self, asset: CatalogAsset) -> tuple[bool, Optional[str]]: - """Validate asset before publishing + async def health_check(self) -> bool: + """Check whether the catalog endpoint is reachable.""" + try: + return True + except Exception as e: + self.logger.error(f"Health check failed: {e}") + return False - Args: - asset: Asset to validate + # -- deprecated CatalogAsset shim -------------------------------------- - Returns: - Tuple of (is_valid, error_message) + def map_contract_to_asset(self, contract: Dict[str, Any]) -> CatalogAsset: + """DEPRECATED: build a flat CatalogAsset from a FLUID contract. + + Prefer :meth:`CatalogTarget.from_fluid` and :meth:`publish` — the + flat asset representation loses multi-port detail and forces every + provider to reconstruct the FLUID dict. """ + warnings.warn( + "map_contract_to_asset() and CatalogAsset are deprecated; " + "call publish(fluid, CatalogTarget.from_fluid(fluid)) instead. " + "This shim will be removed in a future release.", + DeprecationWarning, + stacklevel=2, + ) + return _build_catalog_asset_from_fluid(contract) + + def validate_asset(self, asset: CatalogAsset) -> Tuple[bool, Optional[str]]: + """DEPRECATED: validate a flat CatalogAsset.""" if not asset.name: return False, "Asset name is required" - if not asset.id: return False, "Asset ID is required" - if not asset.owner: return False, "Asset owner is required" - if not asset.domain: return False, "Asset domain is required" - return True, None - async def health_check(self) -> bool: - """Check if catalog is accessible + async def publish_asset(self, asset: CatalogAsset) -> PublishResult: + """DEPRECATED: publish via the flat CatalogAsset DTO. - Returns: - True if healthy, False otherwise + Reconstructs a ``(fluid, target)`` pair from the asset — the + asset's own fields win over its ``raw_contract`` — and forwards to + :meth:`publish`. Emits a DeprecationWarning. """ - try: - # Default implementation - subclasses should override - return True - except Exception as e: - self.logger.error(f"Health check failed: {e}") - return False + warnings.warn( + "publish_asset(CatalogAsset) is deprecated; " + "call publish(fluid, target) with the raw FLUID dict instead.", + DeprecationWarning, + stacklevel=2, + ) + if self._uses_legacy_asset_api(): + # Provider still implements the old ``publish(asset)`` signature. + return await self.publish(asset) # type: ignore[call-arg] + fluid, target = _fluid_target_from_asset(asset) + return await self.publish(fluid, target) + + async def update_asset(self, asset: CatalogAsset) -> PublishResult: + """DEPRECATED: update via the flat CatalogAsset DTO.""" + warnings.warn( + "update_asset(CatalogAsset) is deprecated; " + "call update(fluid, target) with the raw FLUID dict instead.", + DeprecationWarning, + stacklevel=2, + ) + if self._uses_legacy_asset_api(): + return await self.update(asset) # type: ignore[call-arg] + fluid, target = _fluid_target_from_asset(asset) + return await self.update(fluid, target) + + +# --------------------------------------------------------------------------- +# Module-level helpers used by both the deprecated CatalogAsset shim and +# any provider that still wants a flat summary for logging / rendering. +# --------------------------------------------------------------------------- + + +def _publish_declares_target(cls: Any) -> bool: + """True when ``cls.publish`` positively declares ``publish(fluid, target)``. + + The second parameter after ``self`` must be a positional parameter + named ``target`` or annotated :class:`CatalogTarget` (string + annotations, e.g. under ``from __future__ import annotations``, count). + Callables whose signature cannot be read (mocks, builtins) do not. + """ + publish = getattr(cls, "publish", None) + if publish is None or not inspect.isfunction(inspect.unwrap(publish)): + return False + try: + params = list(inspect.signature(publish).parameters.values()) + except (TypeError, ValueError): + return False + params = params[1:] # drop ``self`` + if len(params) < 2: + return False + second = params[1] + if second.kind not in ( + inspect.Parameter.POSITIONAL_ONLY, + inspect.Parameter.POSITIONAL_OR_KEYWORD, + ): + return False + if second.name == "target": + return True + annotation = second.annotation + if annotation is inspect.Parameter.empty: + return False + return annotation is CatalogTarget or "CatalogTarget" in str(annotation) + + +def _coerce_contract_id(contract_id: Optional[str], asset_id: Optional[str], method: str) -> str: + """Resolve ``verify`` / ``delete``'s id, accepting the 0.7.8 ``asset_id=`` + keyword (deprecated) alongside ``contract_id``.""" + if asset_id is not None: + warnings.warn( + f"{method}(asset_id=...) is deprecated; pass the contract id " + f"positionally or as {method}(contract_id=...).", + DeprecationWarning, + stacklevel=3, + ) + if contract_id is not None and contract_id != asset_id: + raise TypeError(f"{method}() got different values for 'contract_id' and 'asset_id'") + return asset_id + if contract_id is None: + raise TypeError(f"{method}() missing 1 required positional argument: 'contract_id'") + return contract_id + + +def _fluid_identity_error(fluid: Any) -> Optional[str]: + """Return an error when *fluid* is not a FLUID contract, else ``None``. + + A contract must be a mapping with a ``name`` (top level, as the FLUID + schema requires, or ``metadata.name``). Contract fragments, overlays and + config files lack one — even a fragment that carries its own ``id``; + 0.7.8 rejected them (``Failed to map contract to asset: 'name'``) and + publishing them would upsert a stray catalog asset. + """ + if not isinstance(fluid, dict): + return ( + "not a FLUID contract: expected a mapping at the top level, " + f"got {type(fluid).__name__}" + ) + metadata = fluid.get("metadata") + meta_name = metadata.get("name") if isinstance(metadata, dict) else None + if not (fluid.get("name") or meta_name): + return ( + "not a FLUID contract: 'name' is required (or 'metadata.name'); " + "contract fragments, overlays and config files cannot be published" + ) + return None + + +def _build_catalog_asset_from_fluid(contract: Dict[str, Any]) -> CatalogAsset: + """Flatten a FLUID contract into the legacy :class:`CatalogAsset` shape. + + Kept module-private so it can be reused by test helpers and the + deprecation shim without going through the public warning path. + """ + metadata = contract.get("metadata") or {} + owner = metadata.get("owner") or contract.get("owner") or {} + exposes = contract.get("exposes") or [] + + platform = "unknown" + location: Dict[str, Any] = {} + schema: Optional[List[Dict[str, Any]]] = None + sensitivity = "internal" + + if exposes: + first_expose = exposes[0] + binding = first_expose.get("binding") or {} + platform = binding.get("platform", "unknown") + location = binding.get("location") or {} + contract_spec = first_expose.get("contract") or {} + schema = contract_spec.get("schema") + sensitivity = first_expose.get("sensitivity", "internal") + + return CatalogAsset( + id=contract.get("id", contract.get("name", "unknown")), + name=contract.get("name") or contract.get("id") or "unknown", + description=contract.get("description", ""), + type=(contract.get("kind") or "DataProduct").lower(), + domain=contract.get("domain") or metadata.get("domain") or "general", + owner=owner.get("team", owner.get("name", "unknown")), + owner_email=owner.get("email", ""), + layer=metadata.get("layer", "Bronze"), + tags=metadata.get("tags", []), + version=contract.get("version", "1.0.0"), + platform=platform, + location=location, + schema=schema, + sensitivity=sensitivity, + raw_contract=dict(contract), + ) + + +def _fluid_target_from_asset(asset: CatalogAsset) -> Tuple[Dict[str, Any], CatalogTarget]: + """Convert a deprecated :class:`CatalogAsset` to a ``(fluid, target)`` pair. + + The returned target remembers *asset* (``target._legacy_asset``) so the + built-in providers can publish exactly the fields the caller set. + """ + target = CatalogTarget( + contract_id=asset.id, + contract_name=asset.name, + contract_yaml=asset.contract_yaml, + ) + target._legacy_asset = asset + return _fluid_from_asset(asset), target + + +def _fluid_from_asset(asset: CatalogAsset) -> Dict[str, Any]: + """The FLUID dict equivalent of a deprecated :class:`CatalogAsset`. + + With a ``raw_contract`` (set by :meth:`BaseCatalogProvider.map_contract_to_asset`) + a copy of it is returned with every flat field of the asset applied on + top, so edits a 0.7.8 caller made to the asset after mapping it win — + the asset is what that caller publishes — while the multi-port detail + of the original contract is kept. Without one the dict is rebuilt from + the flat fields (:func:`_minimal_fluid_from_asset`). + """ + raw = asset.raw_contract + if not raw: + return _minimal_fluid_from_asset(asset) + + baseline = _build_catalog_asset_from_fluid(raw) + fluid: Dict[str, Any] = copy.deepcopy(dict(raw)) + raw_meta = fluid.get("metadata") + meta: Dict[str, Any] = dict(raw_meta) if isinstance(raw_meta, dict) else {} + fluid["metadata"] = meta + + fluid["id"] = asset.id + fluid["name"] = meta["name"] = asset.name + fluid["description"] = meta["description"] = asset.description + fluid["domain"] = meta["domain"] = asset.domain + fluid["version"] = meta["version"] = asset.version + meta["layer"] = asset.layer + meta["tags"] = list(asset.tags or []) + if "tags" in fluid: + fluid["tags"] = list(asset.tags or []) + if asset.type != baseline.type: + fluid["kind"] = asset.type + + for holder in (fluid, meta): + owner = holder.get("owner") + if isinstance(owner, dict): + holder["owner"] = {**owner, "team": asset.owner, "email": asset.owner_email} + if not isinstance(meta.get("owner"), dict) and not isinstance(fluid.get("owner"), dict): + meta["owner"] = {"team": asset.owner, "email": asset.owner_email} + + expose_edits = { + "platform": asset.platform != baseline.platform, + "location": asset.location != baseline.location, + "schema": asset.schema != baseline.schema, + "sensitivity": asset.sensitivity != baseline.sensitivity, + } + if any(expose_edits.values()): + exposes = list(fluid.get("exposes") or []) + first: Dict[str, Any] = dict(exposes[0]) if exposes else {"id": asset.id} + binding = dict(first.get("binding") or {}) + if expose_edits["platform"]: + binding["platform"] = asset.platform + if expose_edits["location"]: + binding["location"] = asset.location + if binding: + first["binding"] = binding + if expose_edits["schema"]: + first["contract"] = {**(first.get("contract") or {}), "schema": asset.schema} + if expose_edits["sensitivity"]: + first["sensitivity"] = asset.sensitivity + if exposes: + exposes[0] = first + else: + exposes = [first] + fluid["exposes"] = exposes + return fluid + + +def _minimal_fluid_from_asset(asset: CatalogAsset) -> Dict[str, Any]: + """Reconstruct a minimal FLUID dict from a flat CatalogAsset. + + Used only when a deprecated caller invokes :meth:`publish_asset` with an + asset whose ``raw_contract`` was never populated (older external + integrations, hand-built assets in tests). Not lossless — the whole + reason for the redesign is to avoid this path. + """ + fluid: Dict[str, Any] = { + "id": asset.id, + "name": asset.name, + "description": asset.description, + "kind": (asset.type or "DataProduct").title().replace(" ", ""), + "metadata": { + "name": asset.name, + "description": asset.description, + "domain": asset.domain, + "version": asset.version, + "tags": list(asset.tags or []), + "layer": asset.layer, + "status": "active", + "owner": { + "team": asset.owner, + "email": asset.owner_email, + }, + }, + "owner": { + "team": asset.owner, + "email": asset.owner_email, + }, + "domain": asset.domain, + "version": asset.version, + } + has_expose_detail = ( + bool(asset.location) + or asset.platform != "unknown" + or asset.schema is not None + or (bool(asset.sensitivity) and asset.sensitivity != "internal") + ) + if has_expose_detail: + expose: Dict[str, Any] = { + "id": asset.id, + "binding": {"platform": asset.platform}, + } + if asset.location: + expose["binding"]["location"] = asset.location + if asset.schema is not None: + expose["contract"] = {"schema": asset.schema} + if asset.sensitivity: + expose["sensitivity"] = asset.sensitivity + fluid["exposes"] = [expose] + return fluid diff --git a/fluid_build/providers/catalogs/datamesh_manager.py b/fluid_build/providers/catalogs/datamesh_manager.py index 266b6deb..2fc89b49 100644 --- a/fluid_build/providers/catalogs/datamesh_manager.py +++ b/fluid_build/providers/catalogs/datamesh_manager.py @@ -23,9 +23,17 @@ from __future__ import annotations import logging -from typing import Any, Dict - -from .base import BaseCatalogProvider, CatalogAsset, PublishResult +from typing import Any, Dict, Mapping, Optional + +from .base import ( + BaseCatalogProvider, + CatalogAsset, + CatalogTarget, + PublishResult, + _coerce_contract_id, + _fluid_from_asset, + _fluid_identity_error, +) LOG = logging.getLogger(__name__) @@ -49,15 +57,75 @@ def __init__(self, config: Dict[str, Any]) -> None: # -- BaseCatalogProvider interface -------------------------------------- - async def publish(self, asset: CatalogAsset) -> PublishResult: - """Publish *asset* as a data product to Entropy Data.""" - fluid = self._asset_to_fluid(asset) + def _spec_selection(self, target: CatalogTarget) -> Dict[str, Optional[str]]: + """Collect the data-product-spec opt-outs for this publish call. + + Per-call ``target.catalog_hints`` win over the catalog config + (``catalogs.datamesh-manager.data_product_spec`` / ``provider_hint``). + When neither is set both values stay ``None`` so + :meth:`DataMeshManagerProvider._resolve_data_product_specification` + applies the same order as ``fluid dmm publish``: explicit spec → + provider hint → ``DMM_DATA_PRODUCT_SPEC`` → ODPS default. + """ + hints = target.catalog_hints or {} + spec = ( + hints.get("data_product_specification") + or hints.get("data_product_spec") + or self.config.get("data_product_specification") + or self.config.get("data_product_spec") + ) + hint = hints.get("provider_hint") or self.config.get("provider_hint") + return { + "data_product_specification": str(spec) if spec else None, + "provider_hint": str(hint) if hint else None, + } + + async def publish( + self, + fluid: Any = None, + target: Optional[CatalogTarget] = None, + *, + asset: Optional[CatalogAsset] = None, + ) -> PublishResult: + """Publish *fluid* as a data product to Entropy Data. + + The FLUID dict is forwarded to the underlying provider (a copy with + the 0.7.8 catalog-path defaults applied, see + :func:`_catalog_compat_fluid`) so every output port is preserved — + one per-port ODCS contract is emitted instead of the single + redundant wrapper the old flat-asset path used to build. + + Passing a deprecated :class:`CatalogAsset` (the 0.7.8 + ``publish(asset)`` / ``publish(asset=asset)`` API) still works for + one release and emits a :class:`DeprecationWarning`; the asset's own + fields are published. A subclass's ``map_contract_to_asset`` + override is honoured the same way. + """ + fluid, target = self._coerce_publish_args(fluid, target, "publish", asset=asset) try: - result = self._provider.apply(fluid, publish_contract=True) + target = self._with_legacy_asset(fluid, target) + legacy_asset = target._legacy_asset + if legacy_asset is not None: + payload_fluid = self._asset_to_fluid(legacy_asset) + else: + identity_error = _fluid_identity_error(fluid) + if identity_error: + return PublishResult( + success=False, + catalog_id=self.name, + asset_id=target.contract_id, + error=f"Validation failed: {identity_error}", + ) + payload_fluid = fluid + result = self._provider.apply( + _catalog_compat_fluid(payload_fluid), + publish_contract=True, + **self._spec_selection(target), + ) return PublishResult( success=True, catalog_id=self.name, - asset_id=asset.id, + asset_id=target.contract_id, catalog_url=result.get("url"), details=result, ) @@ -65,24 +133,37 @@ async def publish(self, asset: CatalogAsset) -> PublishResult: return PublishResult( success=False, catalog_id=self.name, - asset_id=asset.id, + asset_id=target.contract_id, error=str(exc), ) - async def update(self, asset: CatalogAsset) -> PublishResult: + async def update( + self, + fluid: Any = None, + target: Optional[CatalogTarget] = None, + *, + asset: Optional[CatalogAsset] = None, + ) -> PublishResult: # PUT is idempotent — publish == update - return await self.publish(asset) + fluid, target = self._coerce_publish_args(fluid, target, "update", asset=asset) + return await self.publish(fluid, target) - async def verify(self, asset_id: str) -> bool: + async def verify( + self, contract_id: Optional[str] = None, *, asset_id: Optional[str] = None + ) -> bool: + contract_id = _coerce_contract_id(contract_id, asset_id, "verify") try: - self._provider.verify(asset_id) + self._provider.verify(contract_id) return True except Exception: return False - async def delete(self, asset_id: str) -> bool: + async def delete( + self, contract_id: Optional[str] = None, *, asset_id: Optional[str] = None + ) -> bool: + contract_id = _coerce_contract_id(contract_id, asset_id, "delete") try: - return self._provider.delete(asset_id) + return self._provider.delete(contract_id) except Exception: return False @@ -97,7 +178,16 @@ async def health_check(self) -> bool: @staticmethod def _asset_to_fluid(asset: CatalogAsset) -> Dict[str, Any]: - """Convert a CatalogAsset back to a minimal FLUID dict.""" + """Convert a deprecated CatalogAsset to the FLUID dict to publish. + + An asset produced by ``map_contract_to_asset`` (it carries the + original ``raw_contract``) keeps every output port of that contract, + with the asset's own fields applied on top. A hand-built asset is + converted exactly as 0.7.8 did. + """ + if asset.raw_contract: + return _fluid_from_asset(asset) + fluid: Dict[str, Any] = { "id": asset.id, "name": asset.name, @@ -132,3 +222,28 @@ def _asset_to_fluid(asset: CatalogAsset) -> Dict[str, Any]: fluid["exposes"] = [expose] return fluid + + +def _catalog_compat_fluid(fluid: Mapping[str, Any]) -> Dict[str, Any]: + """Apply the 0.7.8 ``fluid publish --catalog datamesh-manager`` defaults. + + 0.7.8 rebuilt the FLUID dict from the flat CatalogAsset before calling + the provider, which always sent ``metadata.status: active`` and took + ``metadata.domain`` / ``metadata.version`` from the top-level FLUID + fields (defaulting to ``general`` / ``1.0.0``). Forwarding the raw + contract would instead publish every product as ``draft`` (moving live + products back to draft on each republish) and drop ``custom.domain`` / + ``custom.version`` from the DPS payload. A copy is returned; values the + contract declares in ``metadata`` win. + """ + out: Dict[str, Any] = dict(fluid) + raw_meta = fluid.get("metadata") + meta: Dict[str, Any] = dict(raw_meta) if isinstance(raw_meta, Mapping) else {} + if not meta.get("status"): + meta["status"] = "active" + if not meta.get("domain"): + meta["domain"] = fluid.get("domain") or "general" + if not meta.get("version"): + meta["version"] = fluid.get("version") or "1.0.0" + out["metadata"] = meta + return out diff --git a/fluid_build/providers/catalogs/fluid_cc/provider.py b/fluid_build/providers/catalogs/fluid_cc/provider.py index d767f404..b1708d89 100644 --- a/fluid_build/providers/catalogs/fluid_cc/provider.py +++ b/fluid_build/providers/catalogs/fluid_cc/provider.py @@ -12,9 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Native FLUID Command Center integration with resilience patterns +"""Native FLUID Command Center integration with resilience patterns. + +Publishes FLUID contracts as assets to the FLUID Command Center API. +Reads the raw FLUID dict directly — no flat-asset intermediate — so +multi-port contracts round-trip losslessly through the ``metadata`` payload. -This provider publishes FLUID contracts as assets to the FLUID Command Center API. Includes: - Circuit breaker for fault tolerance - Retry with exponential backoff @@ -25,82 +28,94 @@ import asyncio import time -from typing import Any, Dict, Optional +from typing import Any, Dict, List, Optional import httpx from ...common import CircuitBreaker, get_auth_headers, metrics_collector -from ..base import BaseCatalogProvider, CatalogAsset, PublishResult +from ..base import ( + BaseCatalogProvider, + CatalogAsset, + CatalogTarget, + PublishResult, + _coerce_contract_id, +) class FluidCommandCenterProvider(BaseCatalogProvider): - """Native integration with FLUID Command Center - - Leverages patterns from market.py: - - Circuit breaker for publish failures - - Retry with exponential backoff - - Health checking before operations - - Metrics collection - """ + """Native integration with FLUID Command Center.""" name = "fluid_cc" def __init__(self, config: Dict[str, Any]): super().__init__(config) - # Initialize circuit breaker self.circuit_breaker = CircuitBreaker( failure_threshold=config.get("circuit_breaker_threshold", 3), recovery_timeout=config.get("circuit_breaker_timeout", 60), expected_exception=httpx.HTTPError, ) - async def publish(self, asset: CatalogAsset) -> PublishResult: - """Publish to FLUID Command Center API with retry logic and upsert - - Workflow: - 1. Pre-publish health check - 2. Validate asset - 3. Search for existing asset by contract ID - 4. Create new or update existing (upsert) - 5. Retry with exponential backoff on failure - 6. Record metrics + async def publish( + self, + fluid: Optional[Dict[str, Any]] = None, + target: Optional[CatalogTarget] = None, + *, + asset: Optional[CatalogAsset] = None, + ) -> PublishResult: + """Publish a FLUID contract with retry logic and upsert. + + The deprecated ``publish(asset)`` / ``publish(asset=asset)`` forms + (a :class:`~fluid_build.providers.catalogs.base.CatalogAsset`) still + work for one release and emit a :class:`DeprecationWarning`; the + asset's fields are published exactly as 0.7.8 did. A subclass's + ``map_contract_to_asset`` override is honoured the same way. """ + fluid, target = self._coerce_publish_args(fluid, target, "publish", asset=asset) start_time = time.time() metrics_collector.record_publish_request(self.name) - # Pre-publish health check if not await self.health_check(): - result = PublishResult( + metrics_collector.record_publish_failure(self.name, "health_check_failed") + return PublishResult( success=False, catalog_id=self.name, - asset_id=asset.id, + asset_id=target.contract_id, error="Catalog health check failed - endpoint not accessible", ) - metrics_collector.record_publish_failure(self.name, "health_check_failed") - return result - # Validate asset - is_valid, error_msg = self.validate_asset(asset) + try: + target = self._with_legacy_asset(fluid, target) + except Exception as e: + metrics_collector.record_validation_error(str(e)) + return PublishResult( + success=False, + catalog_id=self.name, + asset_id=target.contract_id, + error=f"Failed to map contract to asset: {e}", + ) + + legacy_asset = target._legacy_asset + if legacy_asset is not None: + # 0.7.8 validated the asset it was about to publish. + is_valid, error_msg = self.validate_asset(legacy_asset) + else: + is_valid, error_msg = self.validate_target(fluid, target) if not is_valid: - result = PublishResult( + metrics_collector.record_validation_error(error_msg) + return PublishResult( success=False, catalog_id=self.name, - asset_id=asset.id, + asset_id=target.contract_id, error=f"Validation failed: {error_msg}", ) - metrics_collector.record_validation_error(error_msg) - return result - # Retry with exponential backoff for attempt in range(self.max_retries): try: - result = await self.circuit_breaker.call(self._publish_impl, asset) - + result = await self.circuit_breaker.call(self._publish_impl, fluid, target) latency = time.time() - start_time metrics_collector.record_publish_success(self.name, latency) - # Update circuit breaker stats cb_state = self.circuit_breaker.get_state() metrics_collector.update_circuit_breaker_stats( self.name, @@ -110,7 +125,7 @@ async def publish(self, asset: CatalogAsset) -> PublishResult: ) self.logger.info( - f"✅ Published {asset.name} to Command Center " + f"✅ Published {target.contract_name} to Command Center " f"(attempt {attempt + 1}/{self.max_retries}, {latency:.2f}s)" ) return result @@ -118,7 +133,6 @@ async def publish(self, asset: CatalogAsset) -> PublishResult: except Exception as e: if attempt < self.max_retries - 1: delay = self.config.get("retry_delay", 1.0) * (2**attempt) - # Log more details for debugging error_details = str(e) if hasattr(e, "response"): try: @@ -142,48 +156,30 @@ async def publish(self, asset: CatalogAsset) -> PublishResult: ) metrics_collector.record_publish_failure(self.name, str(type(e).__name__)) return PublishResult( - success=False, catalog_id=self.name, asset_id=asset.id, error=error_msg + success=False, + catalog_id=self.name, + asset_id=target.contract_id, + error=error_msg, ) - async def _publish_impl(self, asset: CatalogAsset) -> PublishResult: - """Internal publish implementation (wrapped by circuit breaker)""" - - # Map CatalogAsset to Command Center API format - # Note: owner_id will be overridden by the backend from authenticated user - # but it's required by the Pydantic model, so we send a placeholder - asset_data = { - "name": asset.name, - "description": asset.description, - "type": asset.type, - "owner_id": "placeholder", # Will be replaced by backend from auth token - "tags": asset.tags, - "version": asset.version, - "is_public": asset.sensitivity in ["public", "internal"], - "metadata": { - "fluid_contract_id": asset.id, # Track contract ID for upsert - "domain": asset.domain, - "layer": asset.layer, - "platform": asset.platform, - "location": asset.location, - "schema": asset.schema, - "owner": asset.owner, - "owner_email": asset.owner_email, - "sensitivity": asset.sensitivity, - }, - } - - # Include the full contract YAML if available - if asset.contract_yaml: + async def _publish_impl(self, fluid: Dict[str, Any], target: CatalogTarget) -> PublishResult: + """Internal publish implementation (wrapped by circuit breaker).""" + legacy_asset = target._legacy_asset + if legacy_asset is not None: + asset_data = _build_cc_asset_payload_from_asset(legacy_asset, fluid) + contract_yaml = legacy_asset.contract_yaml + else: + asset_data = _build_cc_asset_payload(fluid, target) + contract_yaml = target.contract_yaml + + if contract_yaml: import hashlib - asset_data["contract_yaml"] = asset.contract_yaml - asset_data["contract_hash"] = hashlib.sha256( - asset.contract_yaml.encode("utf-8") - ).hexdigest() + asset_data["contract_yaml"] = contract_yaml + asset_data["contract_hash"] = hashlib.sha256(contract_yaml.encode("utf-8")).hexdigest() headers = get_auth_headers(self.endpoint, self.auth) - # Debug: Log what we're sending import json as json_lib self.logger.info( @@ -192,13 +188,11 @@ async def _publish_impl(self, asset: CatalogAsset) -> PublishResult: self.logger.debug(f"Full asset_data: {json_lib.dumps(asset_data, indent=2, default=str)}") async with httpx.AsyncClient(timeout=self.timeout) as client: - # Check if asset already exists (by contract ID in metadata) - existing = await self._find_by_contract_id(client, headers, asset.id) + existing = await self._find_by_contract_id(client, headers, target.contract_id) if existing: - # Update existing asset (PATCH) self.logger.info( - f"Updating existing asset: {existing['id']} for contract {asset.id}" + f"Updating existing asset: {existing['id']} for contract {target.contract_id}" ) response = await client.patch( f"{self.endpoint}/api/v1/assets/{existing['id']}", @@ -206,10 +200,11 @@ async def _publish_impl(self, asset: CatalogAsset) -> PublishResult: headers=headers, ) else: - # Create new asset (POST) - self.logger.info(f"Creating new asset for contract: {asset.id}") + self.logger.info(f"Creating new asset for contract: {target.contract_id}") response = await client.post( - f"{self.endpoint}/api/v1/assets", json=asset_data, headers=headers + f"{self.endpoint}/api/v1/assets", + json=asset_data, + headers=headers, ) response.raise_for_status() @@ -223,20 +218,15 @@ async def _publish_impl(self, asset: CatalogAsset) -> PublishResult: details={ "operation": "update" if existing else "create", "api_asset_id": result_data["id"], - "contract_id": asset.id, + "contract_id": target.contract_id, }, ) async def _find_by_contract_id( self, client: httpx.AsyncClient, headers: Dict[str, str], contract_id: str ) -> Optional[Dict[str, Any]]: - """Find asset by fluid_contract_id in metadata - - This enables upsert behavior - we can update existing assets - rather than creating duplicates. - """ + """Find asset by fluid_contract_id in metadata (enables upsert).""" try: - # Use the dedicated fluid_contract_id filter parameter response = await client.get( f"{self.endpoint}/api/v1/assets", params={"fluid_contract_id": contract_id, "limit": 1}, @@ -247,7 +237,6 @@ async def _find_by_contract_id( results = response.json() assets = results.get("items", results.get("assets", [])) - if assets: return assets[0] @@ -262,7 +251,6 @@ async def _find_by_contract_id( results = response.json() assets = results.get("items", results.get("assets", [])) - for asset in assets: metadata = asset.get("metadata", {}) if metadata.get("fluid_contract_id") == contract_id: @@ -274,32 +262,141 @@ async def _find_by_contract_id( self.logger.warning(f"Error searching for existing asset: {e}") return None - async def update(self, asset: CatalogAsset) -> PublishResult: - """Update existing asset (delegates to publish for upsert logic)""" - return await self.publish(asset) - - async def verify(self, asset_id: str) -> bool: - """Verify asset exists in catalog""" + async def update( + self, + fluid: Optional[Dict[str, Any]] = None, + target: Optional[CatalogTarget] = None, + *, + asset: Optional[CatalogAsset] = None, + ) -> PublishResult: + """Update existing asset (delegates to publish for upsert logic).""" + fluid, target = self._coerce_publish_args(fluid, target, "update", asset=asset) + return await self.publish(fluid, target) + + async def verify( + self, contract_id: Optional[str] = None, *, asset_id: Optional[str] = None + ) -> bool: + """Verify asset exists in catalog (``asset_id=`` is deprecated).""" + contract_id = _coerce_contract_id(contract_id, asset_id, "verify") try: headers = get_auth_headers(self.endpoint, self.auth) async with httpx.AsyncClient(timeout=10.0) as client: - # Try to find by contract ID - existing = await self._find_by_contract_id(client, headers, asset_id) + existing = await self._find_by_contract_id(client, headers, contract_id) return existing is not None except Exception as e: self.logger.error(f"Verification failed: {e}") return False async def health_check(self) -> bool: - """Check if Command Center API is accessible""" + """Check if Command Center API is accessible.""" try: headers = get_auth_headers(self.endpoint, self.auth) async with httpx.AsyncClient(timeout=5.0) as client: - # Try to ping the API (GET /api/v1/assets with limit=1) response = await client.get( - f"{self.endpoint}/api/v1/assets", params={"limit": 1}, headers=headers + f"{self.endpoint}/api/v1/assets", + params={"limit": 1}, + headers=headers, ) return response.status_code == 200 except Exception as e: self.logger.warning(f"Health check failed: {e}") return False + + +# --------------------------------------------------------------------------- +# Payload builder — kept module-level so the publish path stays a thin +# wrapper around a pure function (easy to unit-test independently). +# --------------------------------------------------------------------------- + + +def _build_cc_asset_payload(fluid: Dict[str, Any], target: CatalogTarget) -> Dict[str, Any]: + """Flatten a FLUID contract into the Command Center asset payload. + + Multi-port contracts survive as ``metadata.exposes`` (a compact + per-port summary) rather than being collapsed to the first port. + """ + metadata = fluid.get("metadata") or {} + owner = metadata.get("owner") or fluid.get("owner") or {} + exposes = fluid.get("exposes") or [] + + tags = metadata.get("tags") or fluid.get("tags") or [] + version = fluid.get("version") or metadata.get("version") or "1.0.0" + description = fluid.get("description") or metadata.get("description") or "" + domain = fluid.get("domain") or metadata.get("domain") or "general" + layer = metadata.get("layer") or "Bronze" + kind = fluid.get("kind") or "DataProduct" + + first_expose = exposes[0] if exposes else {} + first_binding = first_expose.get("binding") or {} + first_platform = first_binding.get("platform", "unknown") + first_location = first_binding.get("location") or {} + first_sensitivity = first_expose.get("sensitivity", "internal") + first_schema = (first_expose.get("contract") or {}).get("schema") + + return { + "name": target.contract_name, + "description": description, + "type": kind.lower(), + "owner_id": "placeholder", # backend replaces from auth token + "tags": list(tags), + "version": version, + "is_public": first_sensitivity in ("public", "internal"), + "metadata": { + "fluid_contract_id": target.contract_id, + "domain": domain, + "layer": layer, + "platform": first_platform, + "location": first_location, + "schema": first_schema, + "owner": owner.get("team", owner.get("name", "unknown")), + "owner_email": owner.get("email", ""), + "sensitivity": first_sensitivity, + "exposes": _port_summary(exposes), + }, + } + + +def _port_summary(exposes: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + summary: List[Dict[str, Any]] = [] + for expose in exposes: + binding = expose.get("binding") or {} + contract_spec = expose.get("contract") or {} + summary.append( + { + "id": expose.get("id"), + "platform": binding.get("platform"), + "location": binding.get("location"), + "sensitivity": expose.get("sensitivity"), + "schema_fields": len(contract_spec.get("schema") or []), + } + ) + return summary + + +def _build_cc_asset_payload_from_asset( + asset: CatalogAsset, fluid: Dict[str, Any] +) -> Dict[str, Any]: + """The deprecated CatalogAsset route: the 0.7.8 payload, built from the + asset's own fields, plus the additive ``metadata.exposes`` summary of + the equivalent FLUID dict.""" + return { + "name": asset.name, + "description": asset.description, + "type": asset.type, + "owner_id": "placeholder", # backend replaces from auth token + "tags": asset.tags, + "version": asset.version, + "is_public": asset.sensitivity in ("public", "internal"), + "metadata": { + "fluid_contract_id": asset.id, + "domain": asset.domain, + "layer": asset.layer, + "platform": asset.platform, + "location": asset.location, + "schema": asset.schema, + "owner": asset.owner, + "owner_email": asset.owner_email, + "sensitivity": asset.sensitivity, + "exposes": _port_summary(fluid.get("exposes") or []), + }, + } diff --git a/fluid_build/providers/datamesh_manager/README.md b/fluid_build/providers/datamesh_manager/README.md index 1d649532..9e0dd7a6 100644 --- a/fluid_build/providers/datamesh_manager/README.md +++ b/fluid_build/providers/datamesh_manager/README.md @@ -71,17 +71,50 @@ fluid dmm publish contract.yaml --api-key "your-key" ## FLUID → Entropy Data Mapping -| FLUID Field | Entropy Data Field | +### Default: Bitol ODPS + +Data products are sent as **Bitol Open Data Product Standard (ODPS)** +documents (`apiVersion: v1.0.0`, `kind: DataProduct`) by default — current +Entropy Data / Entropy CE organisations accept only this type. + +| FLUID Field | Entropy Data (ODPS) Field | |---|---| -| `id` / `metadata.id` | `info.id` | -| `metadata.name` | `info.name` | +| `id` / `metadata.id` | `id` | +| `name` / `metadata.name` | `name` | +| `description` | `description.purpose` | +| `metadata.status` | `status` | +| `domain` | `domain` | +| `metadata.owner` / `owner.team` | `team.name` (the team id, slugged from the owner team — e.g. `Customer Platform` → `customer-platform` — the same id the auto-created team gets; `--team-id` overrides it) + `team.members[]` | +| `expects[]` | `inputPorts[]` | +| `exposes[]` | `outputPorts[]` (each with `contractId` = `{productId}.{exposeId}`) | +| `metadata.tags` / `tags` | `tags[]` | + +### Legacy: Data Product Specification 0.0.1 (opt-out) + +Organisations that still expect the legacy DPS 0.0.1 shape can opt out of ODPS +with `--data-product-spec 0.0.1`, `DMM_DATA_PRODUCT_SPEC=0.0.1`, or (for +`fluid publish --catalog datamesh-manager`) `data_product_spec: "0.0.1"` / +`provider_hint: dps` in the catalog config. Resolution order: explicit spec → +provider hint (`odps`/`opds` → ODPS, `dps` → 0.0.1) → `DMM_DATA_PRODUCT_SPEC` +→ ODPS. The token `dps` means `0.0.1` wherever a spec value is accepted +(`--data-product-spec dps`, `DMM_DATA_PRODUCT_SPEC=dps`, `data_product_spec: dps`). + +```bash +fluid dmm publish contract.fluid.yaml --data-product-spec 0.0.1 +``` + +| FLUID Field | Entropy Data (DPS 0.0.1) Field | +|---|---| +| — | `dataProductSpecification: "0.0.1"` | +| `id` / `metadata.id` | `id` | +| `metadata.name` | `info.title` | | `metadata.description` | `info.description` | | `metadata.status` (production→active) | `info.status` | -| `owner.team` | `teamId` + `info.owner` | +| `owner.team` | `teamId` + `info.owner` (`--team-id` overrides `teamId`) | | `metadata.archetype` | `info.archetype` | | `metadata.maturity` | `info.maturity` | | `expects[]` | `inputPorts[]` | -| `exposes[]` | `outputPorts[]` | +| `exposes[]` | `outputPorts[]` (each with `dataContractId`) | | `metadata.tags` | `tags[]` | | `metadata.domain`, `.version`, etc. | `custom{}` | @@ -105,11 +138,17 @@ catalogs: auth: api_key: ${DMM_API_KEY} enabled: true + # data_product_spec: "0.0.1" # optional: legacy DPS instead of ODPS # Then publish fluid publish contract.fluid.yaml --catalog datamesh-manager ``` +As in 0.7.8, the catalog path publishes a contract without `metadata.status` +as `active`, and takes a missing `metadata.domain` / `metadata.version` from the +top-level `domain` / `version` (defaulting to `general` / `1.0.0`). Values the +contract declares under `metadata` win. + ## API Reference - **Swagger**: https://api.entropy-data.com/swagger/index.html diff --git a/fluid_build/providers/datamesh_manager/datamesh_manager.py b/fluid_build/providers/datamesh_manager/datamesh_manager.py index 96a62fbc..7fc0d1e4 100644 --- a/fluid_build/providers/datamesh_manager/datamesh_manager.py +++ b/fluid_build/providers/datamesh_manager/datamesh_manager.py @@ -28,8 +28,12 @@ Environment Variables --------------------- -DMM_API_KEY (required) API key for Entropy Data. -DMM_API_URL (optional) Base URL, default ``https://api.entropy-data.com``. +DMM_API_KEY (required) API key for Entropy Data. +DMM_API_URL (optional) Base URL, default ``https://api.entropy-data.com``. +DMM_DATA_PRODUCT_SPEC (optional) Default data product specification when + neither ``--data-product-spec`` nor a + provider hint is given: ``odps`` (default) + or ``0.0.1`` for the legacy DPS shape. """ from __future__ import annotations @@ -161,6 +165,7 @@ def plan( data_product_specification=self._resolve_data_product_specification( data_product_specification, provider_hint=provider_hint, + fluid=c, ), ) actions.append( @@ -214,6 +219,7 @@ def apply(self, contract: Any, out: Any = None, fmt: str = "yaml", **kw: Any) -> data_product_specification=self._resolve_data_product_specification( data_product_specification, provider_hint=provider_hint, + fluid=c, ), validate_generated_contracts=validate_generated_contracts, validation_mode=validation_mode, @@ -392,7 +398,12 @@ def _publish_one( team_obj = dp.get("team") if not isinstance(team_obj, dict): team_obj = {} - team_obj.setdefault("name", tid) + # ``team.name`` must be the team id ``_ensure_team`` creates — + # ``--team-id`` or the slug derived from the owner (as 0.7.8 + # sent). The ODPS mapper fills ``team.name`` with the owner's + # display name (e.g. "Customer Platform"), which would assign + # the product to a team that was never created. + team_obj["name"] = tid dp["team"] = team_obj else: dp["teamId"] = tid @@ -672,14 +683,17 @@ def _to_data_product( ) -> Dict[str, Any]: """Map a FLUID contract to the Entropy Data *DataProduct* shape. - Conforms to Data Product Specification v0.0.1 by default. + ``data_product_specification`` is the value already resolved by + :meth:`_resolve_data_product_specification` (ODPS unless overridden). + ODPS delegates to :meth:`_to_data_product_odps`; anything else is + emitted as the legacy Data Product Specification v0.0.1 shape. Reference: ``PUT /api/dataproducts/{id}`` - Schema requires: + DPS schema requires: - ``id`` at root level - ``info.title`` (not ``info.name``) - ``info.owner`` (team id) - - ``dataProductSpecification`` at root (defaults to ``"0.0.1"``) + - ``dataProductSpecification`` at root (``"0.0.1"`` when unset) """ if self._is_odps_spec(data_product_specification): return self._to_data_product_odps(fluid) @@ -816,22 +830,54 @@ def _resolve_data_product_specification( value: Optional[str], *, provider_hint: Optional[str] = None, + fluid: Optional[Mapping[str, Any]] = None, ) -> str: """Resolve outgoing dataProductSpecification. Resolution order: - 1) explicit value - 2) ODPS provider hint (``odps``/``opds``) - 3) default DPS specification + 1) explicit value (``--data-product-spec`` / ``data_product_specification=``) + 2) provider hint: ``odps``/``opds`` → ODPS, ``dps`` → legacy DPS 0.0.1 + 3) ``DMM_DATA_PRODUCT_SPEC`` environment variable + 4) default: ODPS + + The token ``dps`` (any case) means DPS 0.0.1 wherever a value is + accepted — explicit value, catalog config, provider hint or env var + — since ``"dps"`` itself is not a valid ``dataProductSpecification``. + + ODPS is the default because current Entropy Data / Entropy CE + organisations accept only ``odps`` and reject the legacy DPS shape + with HTTP 400 ("Specification type 'dps' is not supported"). Pass + ``--data-product-spec 0.0.1`` (or ``provider_hint="dps"``, or set + ``DMM_DATA_PRODUCT_SPEC=0.0.1``) to keep sending the legacy Data + Product Specification. + + ``fluid`` is accepted for call-site compatibility and is not used: + the contract's ``kind`` does not influence the outgoing spec (every + schema-valid FLUID data product declares ``kind: DataProduct``, so a + ``kind`` heuristic would make the env-var opt-out unreachable). """ - if value: - return str(value).strip() + explicit = str(value or "").strip() + if explicit: + return self._normalize_spec_token(explicit) hint = str(provider_hint or "").strip().lower() if hint in {"odps", "opds"}: return self.DATA_PRODUCT_SPEC_ODPS + if hint == "dps": + return self.DATA_PRODUCT_SPEC_DPS - return self.DATA_PRODUCT_SPEC_DPS + env_value = os.getenv("DMM_DATA_PRODUCT_SPEC", "").strip() + if env_value: + return self._normalize_spec_token(env_value) + + return self.DATA_PRODUCT_SPEC_ODPS + + @classmethod + def _normalize_spec_token(cls, value: str) -> str: + """Map the ``dps`` token to the DPS version it stands for.""" + if value.lower() == "dps": + return cls.DATA_PRODUCT_SPEC_DPS + return value # ---- port mapping ----------------------------------------------------- @@ -1624,6 +1670,11 @@ def _ensure_team(self, fluid: Mapping[str, Any], team_id: str) -> None: team: Dict[str, Any] = { "id": team_id, "name": owner.get("name") or owner.get("team") or team_id, + # ``type`` is required by Entropy CE 2.0.x — older versions + # accept and ignore it, so this is safe across versions. Caller + # can override via owner["team_type"] when a custom value (e.g. + # ``business-unit``) is needed. + "type": owner.get("team_type", "internal"), } if owner.get("email"): team["contactEmail"] = owner["email"] diff --git a/fluid_build/providers/odcs/__init__.py b/fluid_build/providers/odcs/__init__.py index 0835b30b..790d4c51 100644 --- a/fluid_build/providers/odcs/__init__.py +++ b/fluid_build/providers/odcs/__init__.py @@ -22,6 +22,9 @@ ODCS Specification: https://github.com/bitol-io/open-data-contract-standard """ +from fluid_build.providers import register_provider + from .odcs import OdcsProvider +register_provider("odcs", OdcsProvider) __all__ = ["OdcsProvider"] diff --git a/fluid_build/providers/odcs/io.py b/fluid_build/providers/odcs/io.py new file mode 100644 index 00000000..c0d05490 --- /dev/null +++ b/fluid_build/providers/odcs/io.py @@ -0,0 +1,120 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +"""Filesystem helpers for the ODCS provider — read YAML/JSON, write YAML/JSON. + +Contracts are often untrusted (a bundle someone sent, a remote +``contractId``), so YAML is loaded through :func:`load_yaml`, which refuses +documents whose aliases would expand into an enormous tree ("billion +laughs"): PyYAML shares aliased objects, but validation, deep copies and +every mapper walk the *expanded* tree, so a few hundred bytes could cost +minutes and gigabytes. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import IO, Any, Dict, List, Optional, Tuple, Union + +from fluid_build.providers.base import ProviderError + +# Nodes alias expansion may add on top of a document's own nodes. Real +# contracts use aliases sparingly (if at all); the cap only stops documents +# whose expanded size is out of all proportion to what was written. +MAX_ALIAS_EXPANSION = 1_000_000 + + +def read_input(path: Union[str, Path]) -> Dict[str, Any]: + input_path = Path(path) + with open(input_path) as f: + if input_path.suffix in (".yaml", ".yml"): + return load_yaml(f, source=str(input_path)) + return json.load(f) + + +def load_yaml(stream: Union[str, bytes, IO[Any]], *, source: Optional[str] = None) -> Any: + """``yaml.safe_load`` that refuses alias bombs. + + The document is composed into a node graph first (aliases are shared + nodes there, so this is linear in the input) and its *expanded* node + count is computed without expanding anything. A document whose aliases + would add more than :data:`MAX_ALIAS_EXPANSION` nodes, or that is + recursive, raises :class:`ProviderError` before any Python object is + built. + """ + import yaml + + loader = yaml.SafeLoader(stream) + try: + node = loader.get_single_node() + if node is None: + return None + _check_alias_expansion(node, source) + return loader.construct_document(node) + finally: + loader.dispose() + + +def _check_alias_expansion(root: Any, source: Optional[str]) -> None: + import yaml + + where = f" in {source}" if source else "" + sizes: Dict[int, int] = {} # id(node) -> expanded size of its subtree + in_progress = set() + # Iterative post-order walk: (node, children-already-pushed). + stack: List[Tuple[Any, bool]] = [(root, False)] + unique = 0 + while stack: + node, expanded = stack.pop() + key = id(node) + if not expanded: + if key in sizes: + continue + if key in in_progress: + raise ProviderError(f"Refusing recursive YAML alias{where}") + in_progress.add(key) + unique += 1 + stack.append((node, True)) + for child in _children(node, yaml): + if id(child) in in_progress: + raise ProviderError(f"Refusing recursive YAML alias{where}") + if id(child) not in sizes: + stack.append((child, False)) + continue + total = 1 + sum(sizes[id(child)] for child in _children(node, yaml)) + sizes[key] = total + in_progress.discard(key) + if total - unique > MAX_ALIAS_EXPANSION: + raise ProviderError( + f"Refusing YAML document{where}: its aliases expand to more than " + f"{MAX_ALIAS_EXPANSION:,} extra nodes (possible alias bomb)" + ) + + +def _children(node: Any, yaml: Any) -> List[Any]: + if isinstance(node, yaml.SequenceNode): + return list(node.value) + if isinstance(node, yaml.MappingNode): + out: List[Any] = [] + for key_node, value_node in node.value: + out.append(key_node) + out.append(value_node) + return out + return [] + + +def write_output(data: Dict[str, Any], path: Union[Path, str], fmt: str) -> None: + output_path = Path(path) + output_path.parent.mkdir(parents=True, exist_ok=True) + with open(output_path, "w") as f: + if fmt == "yaml": + import yaml + + yaml.dump(data, f, default_flow_style=False, sort_keys=False) + else: + json.dump(data, f, indent=2) diff --git a/fluid_build/providers/odcs/mappers/__init__.py b/fluid_build/providers/odcs/mappers/__init__.py new file mode 100644 index 00000000..83523699 --- /dev/null +++ b/fluid_build/providers/odcs/mappers/__init__.py @@ -0,0 +1,30 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +"""Ordered mapper pipeline. + +Order matters: +- ``metadata`` first — establishes the FLUID skeleton (exposes/expects lists) + and seeds the metadata block. +- ``team`` — populates ``owner``. +- ``schema`` — fills ``exposes[]`` from ODCS SchemaObjects (needed before sla). +- ``servers`` — fills ``expects[]`` from ODCS servers. +- ``sla`` — attaches qos to ``exposes[0]`` (depends on schema). +- ``quality`` — contract-level only (property/object level handled in schema). +- ``downgrade`` — export only, last: rewrites replayed v3.2.0-only content + when an imported v3.2.0 document is exported at an older ``apiVersion``. + +On export the same order is fine: each mapper writes its own slice of the +ODCS dict; no inter-mapper dependencies. +""" + +from . import downgrade, metadata, quality, schema, servers, sla, team # noqa: F401 + +IMPORT_PIPELINE = [metadata, team, schema, servers, sla, quality] +EXPORT_PIPELINE = [metadata, team, schema, servers, sla, quality, downgrade] + +__all__ = ["IMPORT_PIPELINE", "EXPORT_PIPELINE"] diff --git a/fluid_build/providers/odcs/mappers/base.py b/fluid_build/providers/odcs/mappers/base.py new file mode 100644 index 00000000..2f2573e5 --- /dev/null +++ b/fluid_build/providers/odcs/mappers/base.py @@ -0,0 +1,193 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +"""Section mapper protocol, contexts, and the round-trip pass-through namespace. + +Every ODCS section maps to one module under :mod:`fluid_build.providers.odcs.mappers` +exposing a pair of pure functions: :func:`to_fluid` (ODCS → FLUID) and +:func:`to_odcs` (FLUID → ODCS). + +Lossless round-trip is achieved with a **verbatim stash + native overlay** +pattern rather than per-field pass-through lists: + +- On import, each ODCS section (top-level doc, schema object, property) is + stashed *verbatim* under ``[STASH_KEY]`` and only the FLUID-native + fields are projected out. +- On export, the stash is the base layer; native fields are overlaid **only + where they differ from what the stash would have projected** + (:func:`overlay_native`). Untouched fields — including ones added by a + newer spec version, explicit ``false`` booleans, and empty lists — pass + through byte-for-byte with zero per-field code. +- A section with no stash (fresh FLUID) is synthesised as before. + +Pass-through data lives under a single key per level so the round-trip +surface is auditable in one place. +""" + +from __future__ import annotations + +import copy +import logging +from collections.abc import Mapping, MutableMapping +from dataclasses import dataclass, field +from typing import Any, Dict, Iterable, Optional + +from ..validation import DEFAULT_API_VERSION, normalize_api_version + +PASSTHROUGH_KEY = "odcs_passthrough" +STASH_KEY = "_verbatim" + + +def metadata_passthrough(fluid: MutableMapping[str, Any]) -> Dict[str, Any]: + """Get or create the contract-level pass-through bucket under ``metadata``.""" + metadata = fluid.setdefault("metadata", {}) + if not isinstance(metadata, dict): + metadata = dict(metadata) + fluid["metadata"] = metadata + return metadata.setdefault(PASSTHROUGH_KEY, {}) + + +def get_metadata_passthrough(fluid: Mapping[str, Any]) -> Mapping[str, Any]: + metadata = fluid.get("metadata") or {} + if not isinstance(metadata, Mapping): + return {} + pt = metadata.get(PASSTHROUGH_KEY) + return pt if isinstance(pt, Mapping) else {} + + +def expose_passthrough(expose: MutableMapping[str, Any]) -> Dict[str, Any]: + """Get or create an expose's pass-through bucket.""" + return expose.setdefault(PASSTHROUGH_KEY, {}) + + +def get_expose_passthrough(expose: Mapping[str, Any]) -> Mapping[str, Any]: + pt = expose.get(PASSTHROUGH_KEY) + return pt if isinstance(pt, Mapping) else {} + + +def field_passthrough(fld: MutableMapping[str, Any]) -> Dict[str, Any]: + """Get or create a field's pass-through bucket.""" + return fld.setdefault(PASSTHROUGH_KEY, {}) + + +def get_field_passthrough(fld: Mapping[str, Any]) -> Mapping[str, Any]: + pt = fld.get(PASSTHROUGH_KEY) + return pt if isinstance(pt, Mapping) else {} + + +# ---- verbatim stash + native overlay --------------------------------------- + + +def stash(bucket: MutableMapping[str, Any], section: Mapping[str, Any]) -> None: + """Record ``section`` verbatim in ``bucket`` for lossless re-export. + + The stash is a deep copy: it must never share nested objects with the + caller's input or with the FLUID-native projection, otherwise a YAML + dump of the imported FLUID emits anchors/aliases and an edit to one + side silently rewrites the other. + """ + bucket[STASH_KEY] = copy.deepcopy(dict(section)) + + +def get_stash(bucket: Mapping[str, Any]) -> Optional[Mapping[str, Any]]: + """The verbatim section stashed on import, or ``None`` for fresh FLUID.""" + s = bucket.get(STASH_KEY) + return s if isinstance(s, Mapping) else None + + +def overlay_native( + base: Mapping[str, Any], + native_then: Mapping[str, Any], + native_now: Mapping[str, Any], + *, + keys: Optional[Iterable[str]] = None, +) -> Dict[str, Any]: + """Three-way merge: ``base`` + (``native_now`` − ``native_then``). + + ``native_then`` is what the FLUID projection of ``base`` maps back to; + ``native_now`` is the same projection of the *current* FLUID. Only keys + whose value actually changed are written over ``base`` — so a value the + user never touched keeps its exact original form (absent stays absent, + ``false`` stays ``false``, ``[]`` stays ``[]``), while a genuine edit in + FLUID-land wins. A key edited to ``None`` is removed. + + Both projections must be *total* (always emit every native key, using + ``None`` for "not set") so the comparison is meaningful. + + The result is a deep copy, so mutating a rendered document never + reaches back into the stash (or the FLUID it came from). + """ + out = copy.deepcopy(dict(base)) + for k in keys if keys is not None else (set(native_then) | set(native_now)): + then_v = native_then.get(k) + now_v = native_now.get(k) + if now_v == then_v: + continue + if now_v is None: + out.pop(k, None) + else: + out[k] = copy.deepcopy(now_v) + return out + + +@dataclass +class ImportCtx: + """Mutable context threaded through ODCS → FLUID mappers.""" + + odcs: Mapping[str, Any] + fluid: MutableMapping[str, Any] + logger: logging.Logger + options: Dict[str, Any] = field(default_factory=dict) + + +@dataclass +class ExportCtx: + """Mutable context threaded through FLUID → ODCS mappers. + + ``options["api_version"]`` — when set, the explicit ODCS version to emit + (CLI flag / constructor / env). When ``None`` the metadata mapper + preserves an imported document's own ``apiVersion`` and otherwise falls + back to :data:`fluid_build.providers.odcs.validation.DEFAULT_API_VERSION`. + """ + + fluid: Mapping[str, Any] + odcs: MutableMapping[str, Any] + logger: logging.Logger + options: Dict[str, Any] = field(default_factory=dict) + + +def target_api_version(ctx: ExportCtx) -> str: + """The ODCS ``apiVersion`` this export targets. + + Same precedence as the metadata mapper: an explicit + ``options["api_version"]``, else the imported document's own + ``apiVersion`` (verbatim stash), else :data:`DEFAULT_API_VERSION`. Once + the metadata mapper has run, the value it emitted wins. + """ + emitted = ctx.odcs.get("apiVersion") if isinstance(ctx.odcs, Mapping) else None + if emitted: + return str(emitted) + explicit = ctx.options.get("api_version") + if explicit: + return normalize_api_version(explicit) or DEFAULT_API_VERSION + st = get_stash(get_metadata_passthrough(ctx.fluid)) + if st and st.get("apiVersion"): + return str(st["apiVersion"]) + return DEFAULT_API_VERSION + + +def fluid_id(fluid: Mapping[str, Any]) -> Optional[str]: + """Resolve a FLUID contract's id from any of the supported locations.""" + if "id" in fluid: + return fluid["id"] + contract = fluid.get("contract") + if isinstance(contract, Mapping) and contract.get("id"): + return contract["id"] + metadata = fluid.get("metadata") + if isinstance(metadata, Mapping) and metadata.get("id"): + return metadata["id"] + return None diff --git a/fluid_build/providers/odcs/mappers/downgrade.py b/fluid_build/providers/odcs/mappers/downgrade.py new file mode 100644 index 00000000..39360280 --- /dev/null +++ b/fluid_build/providers/odcs/mappers/downgrade.py @@ -0,0 +1,255 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +"""Downgrade replayed ODCS v3.2.0 content for an older target ``apiVersion``. + +The section mappers replay an imported document's verbatim stash. When that +document declared v3.2.0 (or newer) and the export targets an older version +(an explicit ``api_version`` / ``ODCS_API_VERSION`` pin), the replayed content +can carry constructs the declared version does not allow. This last export +step rewrites them so the output validates against the version it declares, +the way the ODPS side handles its v1.1.0-only keys: + +- ``logicalType: map`` → ``object`` and ``vector`` → ``array``; the ``map`` + shape and vector ``logicalTypeOptions`` move to ``customProperties``; +- v3.2.0-only keys on the contract (``context``), SchemaObjects + (``context``, ``synonyms``, ``deprecated``) and properties (``enum``, + ``semanticType``, ``synonyms``, ``deprecated``) move to the element's + ``customProperties``; a relationship ``id`` moves to the relationship's; +- keys the older version has no place for are dropped: ``vendor`` on custom + properties, ``customProperties`` / ``authoritativeDefinitions`` on SLA + entries; +- servers are checked against the target schema: a ``${VAR}`` string port + and any key the server type does not allow move to the server's + ``customProperties`` (a numeric string port becomes an integer); an engine + type the target lacks, or a typed server left without a key its type + requires, becomes ``custom`` with the original type kept as the + ``serverType`` custom property. + +Every change is logged as one warning. Fresh FLUID never needs this: the +mappers already gate v3.2.0-only output on the target version. +""" + +from __future__ import annotations + +import copy +from collections.abc import Mapping, MutableMapping +from typing import Any, Dict, List, Optional, Set + +from ..validation import api_version_tuple, load_schema +from .base import ExportCtx, ImportCtx, get_metadata_passthrough, get_stash + +_V320 = (3, 2, 0) + +_TOP_LEVEL_KEYS = ("context",) +_OBJECT_KEYS = ("context", "synonyms", "deprecated") +_PROPERTY_KEYS = ("enum", "semanticType", "synonyms", "deprecated") +_SLA_DROP_KEYS = ("customProperties", "authoritativeDefinitions") + + +def to_fluid(ctx: ImportCtx) -> None: # pragma: no cover - export-only step + return None + + +def to_odcs(ctx: ExportCtx) -> None: + target = ctx.odcs.get("apiVersion") + target_t = api_version_tuple(target) + if not target_t or target_t >= _V320: + return + st = get_stash(get_metadata_passthrough(ctx.fluid)) + source = st.get("apiVersion") if st else None + source_t = api_version_tuple(source) + if not source_t or source_t < _V320: + return + changes = downgrade_v320(ctx.odcs, str(target)) + if changes: + ctx.logger.warning( + "ODCS export at %s of a %s contract: downgraded v3.2.0-only content (%s)", + target, + source, + "; ".join(changes), + ) + + +def downgrade_v320(doc: MutableMapping[str, Any], target: str) -> List[str]: + """Rewrite ``doc`` in place so it is valid for ``target`` (< v3.2.0). + + Returns a description of every change made. + """ + changes: List[str] = [] + _move_to_custom(doc, _TOP_LEVEL_KEYS, "contract", changes) + for i, obj in enumerate(doc.get("schema") or []): + if isinstance(obj, MutableMapping): + _downgrade_element(obj, f"schema[{i}]", _OBJECT_KEYS, changes) + for i, entry in enumerate(doc.get("slaProperties") or []): + if isinstance(entry, MutableMapping): + dropped = [k for k in _SLA_DROP_KEYS if entry.pop(k, None) is not None] + if dropped: + changes.append(f"slaProperties[{i}]: dropped {', '.join(dropped)}") + _downgrade_servers(doc, target, changes) + _strip_custom_property_vendor(doc, "", changes) + return changes + + +# ----- schema elements ------------------------------------------------------- + + +def _downgrade_element( + el: MutableMapping[str, Any], where: str, keys: Any, changes: List[str] +) -> None: + logical = el.get("logicalType") + if logical == "map": + el["logicalType"] = "object" + moved = ["map"] if "map" in el else [] + if "logicalTypeOptions" in el: + moved.append("logicalTypeOptions") + _move_to_custom(el, moved, where, changes, note=False) + changes.append(f"{where}: logicalType map -> object") + elif logical == "vector": + el["logicalType"] = "array" + _move_to_custom(el, ["logicalTypeOptions"], where, changes, note=False) + changes.append(f"{where}: logicalType vector -> array") + _move_to_custom(el, keys, where, changes) + for j, rel in enumerate(el.get("relationships") or []): + if isinstance(rel, MutableMapping): + _move_to_custom(rel, ("id",), f"{where}.relationships[{j}]", changes) + for j, prop in enumerate(el.get("properties") or []): + if isinstance(prop, MutableMapping): + _downgrade_element(prop, f"{where}.properties[{j}]", _PROPERTY_KEYS, changes) + items = el.get("items") + if isinstance(items, MutableMapping): + _downgrade_element(items, f"{where}.items", _PROPERTY_KEYS, changes) + + +def _move_to_custom( + el: MutableMapping[str, Any], + keys: Any, + where: str, + changes: List[str], + *, + note: bool = True, +) -> None: + """Move ``keys`` of ``el`` into its ``customProperties`` list. + + An existing custom property of the same name wins (its value is kept and + the moved value dropped); ``None`` values are dropped. + """ + present = [k for k in keys if k in el] + if not present: + return + custom = el.get("customProperties") + custom_list: List[Any] = list(custom) if isinstance(custom, list) else [] + names = {c.get("property") for c in custom_list if isinstance(c, Mapping)} + for key in present: + value = el.pop(key) + if key not in names and value is not None: + custom_list.append({"property": key, "value": copy.deepcopy(value)}) + names.add(key) + if custom_list: + el["customProperties"] = custom_list + if note: + changes.append(f"{where}: {', '.join(present)} -> customProperties") + + +def _strip_custom_property_vendor(node: Any, where: str, changes: List[str]) -> None: + if isinstance(node, MutableMapping): + for key, value in node.items(): + path = f"{where}.{key}" if where else str(key) + if key == "customProperties" and isinstance(value, list): + hit = False + for entry in value: + if isinstance(entry, MutableMapping) and "vendor" in entry: + entry.pop("vendor") + hit = True + if hit: + changes.append(f"{path}: dropped vendor") + continue # custom property values are free-form + _strip_custom_property_vendor(value, path, changes) + elif isinstance(node, list): + for i, value in enumerate(node): + _strip_custom_property_vendor(value, f"{where}[{i}]", changes) + + +# ----- servers --------------------------------------------------------------- + + +def _downgrade_servers(doc: MutableMapping[str, Any], target: str, changes: List[str]) -> None: + servers = doc.get("servers") + if not isinstance(servers, list) or not servers: + return + try: + schema = load_schema(target) + except Exception: # unknown target: nothing to check against + schema = None + if not schema: + return + defs = schema.get("$defs") or {} + server_def = defs.get("Server") or {} + base_keys: Set[str] = set((server_def.get("properties") or {}).keys()) + type_enum = ((server_def.get("properties") or {}).get("type") or {}).get("enum") or [] + sources = defs.get("ServerSource") or {} + by_type: Dict[str, Mapping[str, Any]] = {} + for cond in server_def.get("allOf") or []: + const = (((cond.get("if") or {}).get("properties") or {}).get("type") or {}).get("const") + ref = (cond.get("then") or {}).get("$ref") or "" + source = sources.get(ref.rsplit("/", 1)[-1]) if ref else None + if const and isinstance(source, Mapping): + by_type[const] = source + + def fit(server: MutableMapping[str, Any], where: str) -> Mapping[str, Any]: + source = by_type.get(str(server.get("type"))) or {} + source_props: Mapping[str, Any] = source.get("properties") or {} + _coerce_port(server, source_props.get("port"), where, changes) + allowed = base_keys | set(source_props) + _move_to_custom(server, [k for k in server if k not in allowed], where, changes) + return source + + for i, server in enumerate(servers): + if not isinstance(server, MutableMapping): + continue + where = f"servers[{i}]" + stype = server.get("type") + if stype is not None and type_enum and stype not in type_enum: + _to_custom(server, where, changes) + source = fit(server, where) + missing = [k for k in source.get("required") or [] if k not in server] + if missing and server.get("type") != "custom": + # A required key could not be kept (e.g. a ``${VAR}`` port): + # the typed server cannot be expressed, the custom one can. + _to_custom(server, where, changes) + fit(server, where) + + +def _to_custom(server: MutableMapping[str, Any], where: str, changes: List[str]) -> None: + """Turn ``server`` into a ``custom`` one, keeping its type as ``serverType``.""" + stype = server.get("type") + server["type"] = "custom" + custom = server.get("customProperties") + custom_list: List[Any] = list(custom) if isinstance(custom, list) else [] + if not any(isinstance(c, Mapping) and c.get("property") == "serverType" for c in custom_list): + custom_list.append({"property": "serverType", "value": stype}) + server["customProperties"] = custom_list + changes.append(f"{where}: type {stype} -> custom") + + +def _coerce_port( + server: MutableMapping[str, Any], + port_schema: Optional[Mapping[str, Any]], + where: str, + changes: List[str], +) -> None: + """v3.2.0 allows a string port (``${DB_PORT}``); older versions an integer.""" + port = server.get("port") + if not isinstance(port, str) or not isinstance(port_schema, Mapping): + return + if port_schema.get("type") != "integer": + return + if port.strip().isdigit(): + server["port"] = int(port.strip()) + changes.append(f"{where}: port {port!r} -> {server['port']}") + else: + _move_to_custom(server, ("port",), where, changes) diff --git a/fluid_build/providers/odcs/mappers/metadata.py b/fluid_build/providers/odcs/mappers/metadata.py new file mode 100644 index 00000000..c6f2077c --- /dev/null +++ b/fluid_build/providers/odcs/mappers/metadata.py @@ -0,0 +1,209 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +"""Contract-level metadata mapper. + +Owns the top-level ODCS scalars and descriptive blocks (``apiVersion``, +``kind``, ``id``, ``version``, ``status``, ``name``, ``description``, ``tags``, +``domain``, ``tenant``, ``dataProduct``, ``customProperties``, +``authoritativeDefinitions``, ``support``, ``price``, ``roles``, +``slaDefaultElement``, ``contractCreatedTs``, and the v3.2.0 ``context``). +The section keys owned by sibling mappers (``schema``, ``servers``, +``slaProperties``, ``team``, ``quality``) are excluded from this mapper's +verbatim stash. + +``apiVersion`` on export resolves in this order: + +1. ``ExportCtx.options["api_version"]`` — explicit (CLI / constructor / env). +2. The imported document's own ``apiVersion`` (verbatim stash) — a v3.0.0 + contract re-exports as v3.0.0, a v3.2.0 one as v3.2.0. +3. :data:`~fluid_build.providers.odcs.validation.DEFAULT_API_VERSION`. +""" + +from __future__ import annotations + +import copy +from collections.abc import Mapping +from typing import Any, Dict, Optional + +from fluid_build.providers.base import ProviderError + +from ..validation import DEFAULT_API_VERSION, normalize_api_version +from .base import ( + ExportCtx, + ImportCtx, + fluid_id, + get_metadata_passthrough, + get_stash, + metadata_passthrough, + overlay_native, + stash, +) +from .types import fluid_to_odcs_status, odcs_to_fluid_status + +ODCS_KIND = "DataContract" + +# Top-level keys owned by sibling mappers — never stashed here. +SECTION_KEYS = frozenset({"schema", "servers", "slaProperties", "team", "quality"}) + + +# ----- ODCS → FLUID -------------------------------------------------------- + + +def _project_metadata(odcs: Mapping[str, Any]) -> Dict[str, Any]: + """Pure projection of the top-level ODCS scalars into FLUID ``metadata``.""" + metadata: Dict[str, Any] = { + "version": odcs.get("version", "1.0.0"), + "name": odcs.get("name", odcs.get("id")), + "status": odcs_to_fluid_status(odcs.get("status", "active")), + } + description = odcs.get("description") + if isinstance(description, Mapping): + if description.get("purpose"): + metadata["description"] = description["purpose"] + elif isinstance(description, str) and description: + metadata["description"] = description + if odcs.get("tags"): + metadata["tags"] = list(odcs["tags"]) + for key in ("domain", "tenant", "dataProduct"): + if odcs.get(key): + metadata[key] = odcs[key] + return metadata + + +def to_fluid(ctx: ImportCtx) -> None: + odcs = ctx.odcs + fluid = ctx.fluid + + metadata = fluid.setdefault("metadata", {}) + metadata.update(copy.deepcopy(_project_metadata(odcs))) + + fluid.setdefault("contract", {})["id"] = odcs.get("id") + fluid.setdefault("exposes", []) + fluid.setdefault("expects", []) + + stash( + metadata_passthrough(fluid), + {k: v for k, v in odcs.items() if k not in SECTION_KEYS}, + ) + + +# ----- FLUID → ODCS -------------------------------------------------------- + + +def _native_metadata( + metadata: Mapping[str, Any], + contract_id: Optional[str], + raw_status: Any, + root: Optional[Mapping[str, Any]] = None, +) -> Dict[str, Any]: + """Total projection of FLUID-native top-level fields to ODCS keys. + + ``metadata.*`` wins; ``root`` (the FLUID document) is the fallback for + ``name`` / ``description`` / ``tags`` / ``domain``, which schema-valid + FLUID 0.7.x keeps at the top level (its ``metadata`` block does not + allow them). + """ + root = root or {} + + def pick(key: str) -> Any: + value = metadata.get(key) + if value: + return value + # FLUID declares these as a string (``tags``: a list) at the root; + # anything else there is not the FLUID field. + fallback = root.get(key) + expected = list if key == "tags" else str + return fallback if isinstance(fallback, expected) else None + + tags = pick("tags") + description = pick("description") + return { + "version": metadata.get("version", "1.0.0"), + "id": contract_id, + "status": fluid_to_odcs_status(raw_status), + "name": pick("name") or None, + "description": ({"purpose": copy.deepcopy(description)} if description else None), + "tags": list(tags) if tags else None, + "domain": copy.deepcopy(pick("domain")) or None, + "tenant": copy.deepcopy(metadata.get("tenant")) or None, + "dataProduct": copy.deepcopy(metadata.get("dataProduct")) or None, + } + + +def _resolve_api_version(ctx: ExportCtx, st: Optional[Mapping[str, Any]]) -> str: + explicit = ctx.options.get("api_version") + if explicit: + return normalize_api_version(explicit) or DEFAULT_API_VERSION + if st and st.get("apiVersion"): + return str(st["apiVersion"]) + return DEFAULT_API_VERSION + + +def _raw_status(fluid: Mapping[str, Any], metadata: Mapping[str, Any]) -> Any: + if fluid.get("_scoped_status"): + return fluid["_scoped_status"] + if metadata.get("status"): + return metadata["status"] + lifecycle = fluid.get("lifecycle") + if isinstance(lifecycle, Mapping) and lifecycle.get("state"): + return lifecycle["state"] + return "active" + + +def to_odcs(ctx: ExportCtx) -> None: + fluid = ctx.fluid + odcs = ctx.odcs + metadata = fluid.get("metadata") or {} + + contract_id = fluid.get("_scoped_id") or fluid_id(fluid) + if not contract_id: + raise ProviderError( + "Contract missing required 'id' field. " + "Expected one of: fluid['id'], fluid['contract']['id'], or fluid['metadata']['id']" + ) + raw_status = _raw_status(fluid, metadata) + + st = get_stash(get_metadata_passthrough(fluid)) + now = _native_metadata(metadata, contract_id, raw_status, fluid) + + if st is not None: + then = _native_metadata( + _project_metadata(st), st.get("id"), _project_metadata(st)["status"] + ) + merged = overlay_native(st, then, now) + # ``description`` is an object in ODCS but a string in FLUID (its + # ``purpose``): an edit or removal of the purpose must not discard + # the stashed limitations/usage/authoritativeDefinitions. + stashed_desc = st.get("description") + if now["description"] != then["description"] and isinstance(stashed_desc, Mapping): + if now["description"] is not None: + merged["description"] = { + **copy.deepcopy(dict(stashed_desc)), + **now["description"], + } + else: + rest = {k: copy.deepcopy(v) for k, v in stashed_desc.items() if k != "purpose"} + if rest: + merged["description"] = rest + else: + merged.pop("description", None) + else: + merged = {k: v for k, v in now.items() if v is not None} + + merged["apiVersion"] = _resolve_api_version(ctx, st) + merged["kind"] = ODCS_KIND + + # Canonical head ordering for readable YAML; remaining keys keep their + # stashed order (dict comparison is order-insensitive either way). + head = ("version", "apiVersion", "kind", "id", "status", "name") + for key in head: + if key in merged: + odcs[key] = merged[key] + for key, value in merged.items(): + if key not in head: + odcs[key] = value diff --git a/fluid_build/providers/odcs/mappers/quality.py b/fluid_build/providers/odcs/mappers/quality.py new file mode 100644 index 00000000..0f92be26 --- /dev/null +++ b/fluid_build/providers/odcs/mappers/quality.py @@ -0,0 +1,234 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +"""Quality mapper. + +Three levels of quality survive round-trip: + +1. **Property-level** — emitted under ``schemaObject.properties[i].quality[]``. + For fresh FLUID, generated from ``field.required``, primary-key tags and + ``field.validations``, followed by any user-provided ``field.quality`` + items normalised to ODCS. When the FLUID field was itself imported from + ODCS it carries a verbatim stash and the schema mapper replays its + original ``quality`` list as-is — no auto-checks added — so the + round-trip stays lossless. ``include_quality_checks=False`` + (``--no-quality`` / ``ODCS_INCLUDE_QUALITY=false``) suppresses all of it. + +2. **Object-level** — ``schemaObject.quality[]``. Pure pass-through via the + schema object's verbatim stash (handled in :mod:`.schema`); also + suppressed by ``include_quality_checks=False``, as is quality nested in + a replayed property's ``properties`` / ``items``. + +3. **Contract-level** — top-level ``odcs["quality"]`` (non-spec extension + emitted by FLUID). Read into ``fluid.quality`` / pass-through and replayed + verbatim on export. +""" + +from __future__ import annotations + +import copy +from collections.abc import Mapping +from typing import Any, Dict, List, Optional + +from .base import ( + ExportCtx, + ImportCtx, + get_metadata_passthrough, + metadata_passthrough, +) + +# ----- ODCS → FLUID -------------------------------------------------------- + + +def to_fluid(ctx: ImportCtx) -> None: + """Contract-level quality only — property/object level are handled by schema.""" + contract_quality = ctx.odcs.get("quality") + if contract_quality is None: + return + if isinstance(contract_quality, Mapping): + ctx.fluid["quality"] = copy.deepcopy(dict(contract_quality)) + else: + metadata_passthrough(ctx.fluid)["contract_quality_raw"] = copy.deepcopy(contract_quality) + + +# ----- FLUID → ODCS (contract level) -------------------------------------- + + +def to_odcs(ctx: ExportCtx) -> None: + if not ctx.options.get("include_quality_checks", True): + return + + pt = get_metadata_passthrough(ctx.fluid) + if "contract_quality_raw" in pt: + ctx.odcs["quality"] = copy.deepcopy(pt["contract_quality_raw"]) + return + + spec = ctx.fluid.get("quality") + if not spec: + return + if isinstance(spec, Mapping): + ctx.odcs["quality"] = { + "type": spec.get("type", "custom"), + "specification": spec.get("specification", ""), + } + + +# ----- FLUID → ODCS (property level, called from schema mapper) ----------- + + +def to_odcs_property(fld: Mapping[str, Any]) -> Optional[List[Dict[str, Any]]]: + """Return the ``quality[]`` array for a *fresh* ODCS SchemaProperty. + + Auto-checks are synthesised from ``required``, primary-key tags and + ``field.validations``; any user-provided ``field.quality`` items are + appended after them, normalised to ODCS (see :func:`_normalise_custom`). + + Imported fields never reach this function: they carry a verbatim stash + and are re-emitted by the schema mapper's overlay, which replays the + original ``quality`` list untouched. + """ + quality_checks: List[Dict[str, Any]] = [] + name = fld.get("name", "unknown") + tags = fld.get("tags") or [] + is_primary_key = "primary-key" in tags or "primaryKey" in tags + + if fld.get("required"): + quality_checks.append( + { + "type": "library", + "metric": "nullValues", + "mustBe": 0, + "dimension": "completeness", + "description": f"Field '{name}' must not contain null values", + } + ) + + if is_primary_key: + quality_checks.append( + { + "type": "library", + "metric": "duplicateValues", + "mustBe": 0, + "dimension": "uniqueness", + "description": f"Primary key field '{name}' must contain only unique values", + } + ) + if not fld.get("required"): + quality_checks.append( + { + "type": "library", + "metric": "nullValues", + "mustBe": 0, + "dimension": "completeness", + "description": (f"Primary key field '{name}' must not contain null values"), + } + ) + + validations = fld.get("validations") or [] + if isinstance(validations, Mapping): + validations = [{"type": k, "value": v} for k, v in validations.items()] + if not isinstance(validations, list): + validations = [] + + for validation in validations: + if not isinstance(validation, Mapping): + continue + check = _validation_to_quality(validation, name, is_primary_key, fld) + if check is not None: + quality_checks.append(check) + + custom_quality = fld.get("quality") + if isinstance(custom_quality, list): + for item in custom_quality: + normalised = _normalise_custom(item) + if normalised is not None: + quality_checks.append(normalised) + + return quality_checks or None + + +# ODCS ``DataQuality.type`` values. +_ODCS_QUALITY_TYPES = frozenset({"library", "text", "sql", "custom"}) + + +def _normalise_custom(item: Any) -> Optional[Dict[str, Any]]: + """A user-supplied ``field.quality`` item as an ODCS quality check. + + ODCS-shaped items (a known ``type``, or any ``type`` + ``metric``) are + kept; a bare ``{description}`` becomes a ``text`` check; anything else + has no ODCS meaning and is dropped. + """ + if not isinstance(item, Mapping): + return None + qtype = item.get("type") + if qtype in _ODCS_QUALITY_TYPES or ("type" in item and "metric" in item): + return copy.deepcopy(dict(item)) + if "description" in item: + return {"type": "text", "description": copy.deepcopy(item["description"])} + return None + + +def _validation_to_quality( + validation: Mapping[str, Any], + name: str, + is_primary_key: bool, + fld: Mapping[str, Any], +) -> Optional[Dict[str, Any]]: + val_type = validation.get("type", "") + val_value = validation.get("value") + val_values = validation.get("values") + + if val_type in ("pattern", "regex") and val_value: + return { + "type": "text", + "description": f"Field '{name}' must match pattern: {val_value}", + } + if val_type == "min_length" and val_value is not None: + return { + "type": "text", + "description": f"Field '{name}' must have minimum length of {val_value}", + } + if val_type == "max_length" and val_value is not None: + return { + "type": "text", + "description": f"Field '{name}' must have maximum length of {val_value}", + } + if val_type == "min_value" and val_value is not None: + return { + "type": "text", + "description": (f"Field '{name}' must be greater than or equal to {val_value}"), + } + if val_type == "max_value" and val_value is not None: + return { + "type": "text", + "description": (f"Field '{name}' must be less than or equal to {val_value}"), + } + if val_type in ("allowed_values", "enum") and val_values: + values_str = ", ".join(str(v) for v in val_values[:5]) + if len(val_values) > 5: + values_str += f", ... ({len(val_values)} total)" + return { + "type": "text", + "description": f"Field '{name}' must be one of: {values_str}", + } + if val_type == "not_null" and val_value and not fld.get("required"): + return { + "type": "library", + "metric": "nullValues", + "mustBe": 0, + "dimension": "completeness", + "description": f"Field '{name}' must not contain null values", + } + if val_type == "unique" and val_value and not is_primary_key: + return { + "type": "library", + "metric": "duplicateValues", + "mustBe": 0, + "dimension": "uniqueness", + "description": f"Field '{name}' must contain only unique values", + } + return None diff --git a/fluid_build/providers/odcs/mappers/schema.py b/fluid_build/providers/odcs/mappers/schema.py new file mode 100644 index 00000000..51035a63 --- /dev/null +++ b/fluid_build/providers/odcs/mappers/schema.py @@ -0,0 +1,379 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +"""Schema mapper. + +ODCS ``schema`` is an array of :class:`SchemaObject` whose ``properties[]`` +carry the field definitions. Each SchemaObject becomes one FLUID expose; each +SchemaProperty becomes one FLUID field. + +Lossless by construction (see :mod:`.base`): on import every SchemaObject and +SchemaProperty is stashed verbatim and only the FLUID-native fields are +projected out; on export the stash is the base layer and native fields are +overlaid only where they changed. Every other ODCS field — ``physicalType``, +``relationships``, ``transform*``, ``enum``, ``semanticType``, ``synonyms``, +``context``, ``deprecated``, ``id`` and whatever the next spec version adds — +round-trips with no per-field code here. + +Fresh FLUID (no stash) is synthesised: ``physicalType`` from the provider's +type table, ``quality`` auto-checks via :mod:`.quality`. +""" + +from __future__ import annotations + +import copy +import logging +from collections.abc import Mapping +from typing import Any, Dict, List, Optional + +from .base import ( + ExportCtx, + ImportCtx, + expose_passthrough, + field_passthrough, + get_expose_passthrough, + get_field_passthrough, + get_metadata_passthrough, + get_stash, + metadata_passthrough, + overlay_native, + stash, + target_api_version, +) +from .servers import binding_for_object +from .types import fluid_to_logical, fluid_to_physical, logical_to_fluid + +LOG = logging.getLogger(__name__) + +_PK_TAGS = ("primary-key", "primaryKey") + +# Contract-level pass-through flag: the imported document had no ``schema``. +SCHEMA_ABSENT_KEY = "schema_absent" + + +# ----- ODCS → FLUID -------------------------------------------------------- + + +def to_fluid(ctx: ImportCtx) -> None: + if "schema" not in ctx.odcs: + # Recorded so re-export does not synthesise ``schema: []``. + metadata_passthrough(ctx.fluid)[SCHEMA_ABSENT_KEY] = True + schema = ctx.odcs.get("schema") or [] + if not isinstance(schema, list): + return + exposes = ctx.fluid.setdefault("exposes", []) + for schema_object in schema: + if isinstance(schema_object, Mapping): + exposes.append(_object_to_expose(schema_object, ctx.odcs)) + + +def _project_object(schema_object: Mapping[str, Any], odcs: Mapping[str, Any]) -> Dict[str, Any]: + """Pure FLUID projection of a SchemaObject (no stash, no properties).""" + expose_id = schema_object.get("name") or odcs.get("id") or "default" + expose: Dict[str, Any] = { + "id": expose_id, + "exposeId": expose_id, + "version": odcs.get("version", "1.0.0"), + } + if schema_object.get("description"): + expose["description"] = copy.deepcopy(schema_object["description"]) + # ``physicalType`` (table/view/topic) says what the object is, not which + # platform hosts it — the binding comes from the servers, and only when + # they identify one unambiguously. + binding = binding_for_object(schema_object, odcs.get("servers") or []) + if binding: + expose["binding"] = binding + return expose + + +def _object_to_expose(schema_object: Mapping[str, Any], odcs: Mapping[str, Any]) -> Dict[str, Any]: + expose = _project_object(schema_object, odcs) + expose["contract"] = { + "schema": [ + _property_to_field(p) + for p in schema_object.get("properties") or [] + if isinstance(p, Mapping) + ] + } + # Properties are stashed per-field; keep the object stash free of them, + # but record that the key was present (as ``properties: []``) so an + # object with an explicit empty list round-trips exactly. + stash( + expose_passthrough(expose), + { + k: ([] if k == "properties" else v) + for k, v in schema_object.items() + if k != "properties" or isinstance(v, list) + }, + ) + return expose + + +def _project_property(prop: Mapping[str, Any]) -> Dict[str, Any]: + """Pure FLUID projection of a SchemaProperty (presence-preserving).""" + fld: Dict[str, Any] = { + "name": prop.get("name", "unknown"), + "type": logical_to_fluid(prop.get("logicalType", "string")), + } + if prop.get("description"): + fld["description"] = copy.deepcopy(prop["description"]) + if prop.get("classification"): + fld["classification"] = copy.deepcopy(prop["classification"]) + + # required (v3.x); legacy isNullable accepted + if "required" in prop: + fld["required"] = bool(prop["required"]) + elif "isNullable" in prop: + fld["required"] = not bool(prop["isNullable"]) + else: + fld["required"] = False + + if "tags" in prop: + fld["tags"] = copy.deepcopy(list(prop["tags"] or [])) + + # ``quality`` is always present on an imported field: the verbatim list, + # or ``[]`` meaning "the source had none — don't synthesise on export". + quality = prop.get("quality") + fld["quality"] = copy.deepcopy(list(quality)) if isinstance(quality, list) else [] + return fld + + +def _property_to_field(prop: Mapping[str, Any]) -> Dict[str, Any]: + fld = _project_property(prop) + stash(field_passthrough(fld), prop) + return fld + + +# ----- FLUID → ODCS -------------------------------------------------------- + + +def to_odcs(ctx: ExportCtx) -> None: + """Emit ``odcs.schema`` from FLUID exposes (an array). + + An imported document that had no ``schema`` key gets none back unless + FLUID now produces SchemaObjects. + """ + schema = list( + _extract_schema( + ctx.fluid, + include_quality=ctx.options.get("include_quality_checks", True), + api_version=target_api_version(ctx), + ) + ) + if not schema and get_metadata_passthrough(ctx.fluid).get(SCHEMA_ABSENT_KEY): + return + ctx.odcs["schema"] = schema + + +def _extract_schema( + fluid: Mapping[str, Any], + include_quality: bool = True, + api_version: Optional[str] = None, +) -> List[Dict[str, Any]]: + odcs_schema: List[Dict[str, Any]] = [] + for expose in fluid.get("exposes") or []: + if not isinstance(expose, Mapping): + continue + + # Field list — FLUID 0.7.x at expose.contract.schema, 0.5.x at expose.schema.fields + contract_schema: List[Mapping[str, Any]] = [] + contract = expose.get("contract") + if isinstance(contract, Mapping) and isinstance(contract.get("schema"), list): + contract_schema = contract["schema"] + else: + schema_obj = expose.get("schema") + if isinstance(schema_obj, Mapping) and isinstance(schema_obj.get("fields"), list): + contract_schema = schema_obj["fields"] + + # An imported SchemaObject is re-emitted even without properties + # (they are optional in ODCS); fresh exposes without fields are + # skipped as before. + st = get_stash(get_expose_passthrough(expose)) + if not contract_schema and st is None: + continue + + provider = _expose_provider(expose) + properties = [ + _field_to_property( + fld, provider, include_quality=include_quality, api_version=api_version + ) + for fld in contract_schema + if isinstance(fld, Mapping) + ] + if not properties and st is None: + continue + + expose_id = expose.get("exposeId") or expose.get("id") or "dataset" + if st is not None: + then = _native_object(_project_object(st, {"id": expose_id})) + now = _native_object(expose) + schema_object = overlay_native(st, then, now) + if properties or "properties" in st: + schema_object["properties"] = properties + else: + schema_object.pop("properties", None) + if not include_quality: + # Object-level quality is replayed from the stash only. + schema_object.pop("quality", None) + else: + schema_object = { + "name": expose_id, + "logicalType": "object", + "physicalType": _platform_to_physical_type(provider), + "properties": properties, + } + if expose.get("description"): + schema_object["description"] = copy.deepcopy(expose["description"]) + odcs_schema.append(schema_object) + return odcs_schema + + +def _native_object(expose: Mapping[str, Any]) -> Dict[str, Any]: + """Total projection of an expose's FLUID-native fields to ODCS keys.""" + return { + "name": expose.get("exposeId") or expose.get("id"), + "description": expose.get("description") or None, + } + + +def _native_property(fld: Mapping[str, Any], api_version: Optional[str] = None) -> Dict[str, Any]: + """Total projection of a field's FLUID-native fields to ODCS keys.""" + tags = fld.get("tags") + quality = fld.get("quality") + return { + "name": fld.get("name", "unknown"), + "logicalType": fluid_to_logical(fld.get("type", "string"), api_version), + "description": fld.get("description") or None, + "classification": fld.get("classification") or None, + "required": bool(fld.get("required")), + "tags": list(tags) if tags is not None else None, + "quality": list(quality) if isinstance(quality, list) else None, + "primaryKey": True if tags and any(t in _PK_TAGS for t in tags) else None, + } + + +# Property keys that are only valid alongside one particular logicalType. +_TYPE_BOUND_KEYS = {"items": "array", "properties": "object", "map": "map"} + +# Most general ODCS v3.2.0 ``map`` shape — FLUID has no way to say more. +_DEFAULT_MAP_SHAPE = {"key": {"logicalType": "string"}, "value": {"logicalType": "string"}} + + +def _field_to_property( + fld: Mapping[str, Any], + provider: Optional[str], + include_quality: bool = True, + api_version: Optional[str] = None, +) -> Dict[str, Any]: + st = get_stash(get_field_passthrough(fld)) + if st is not None: + then = _native_property(_project_property(st), api_version) + now = _native_property(fld, api_version) + prop = overlay_native(st, then, now) + if now["logicalType"] != then["logicalType"]: + _drop_stale_type_details(prop, fld, provider, api_version) + if not include_quality: + _strip_quality(prop) + return prop + + # ---- fresh FLUID: synthesise --------------------------------------- + prop = { + "name": fld.get("name", "unknown"), + "logicalType": fluid_to_logical(fld.get("type", "string"), api_version), + } + if prop["logicalType"] == "map": + # ODCS v3.2.0 requires the key/value shape alongside ``map``. + prop["map"] = copy.deepcopy(_DEFAULT_MAP_SHAPE) + if provider: + phys = fluid_to_physical(fld.get("type", "string"), provider, api_version) + if phys: + prop["physicalType"] = phys + if fld.get("description"): + prop["description"] = copy.deepcopy(fld["description"]) + # Downstream consumers of fresh exports expect an explicit ``required``. + prop["required"] = bool(fld.get("required")) + if fld.get("classification"): + prop["classification"] = copy.deepcopy(fld["classification"]) + if fld.get("tags"): + prop["tags"] = list(fld["tags"]) + + if include_quality: + from . import quality as quality_mapper # local import to avoid cycle + + qchecks = quality_mapper.to_odcs_property(fld) + if qchecks: + prop["quality"] = qchecks + + if fld.get("tags") and any(t in _PK_TAGS for t in fld["tags"]): + prop["primaryKey"] = True + return prop + + +def _strip_quality(prop: Dict[str, Any]) -> None: + """Remove ``quality`` from a replayed property and everything nested in it + (object ``properties``, array ``items``, map ``key``/``value``).""" + prop.pop("quality", None) + props = prop.get("properties") + nested: List[Any] = list(props) if isinstance(props, list) else [] + if isinstance(prop.get("items"), dict): + nested.append(prop["items"]) + if isinstance(prop.get("map"), dict): + nested.extend(v for k, v in prop["map"].items() if k in ("key", "value")) + for child in nested: + if isinstance(child, dict): + _strip_quality(child) + + +def _drop_stale_type_details( + prop: Dict[str, Any], + fld: Mapping[str, Any], + provider: Optional[str], + api_version: Optional[str], +) -> None: + """The FLUID type of an imported field changed: drop stashed type details. + + ``physicalType`` and ``logicalTypeOptions`` describe the *old* type (the + schema rejects options that don't fit the new one), so they are removed + — ``physicalType`` is re-synthesised when the platform is known — + together with any ``items`` / ``properties`` / ``map`` block the new + logicalType does not allow. + """ + dropped = [k for k in ("physicalType", "logicalTypeOptions") if k in prop] + prop.pop("physicalType", None) + prop.pop("logicalTypeOptions", None) + for key, logical in _TYPE_BOUND_KEYS.items(): + if key in prop and prop.get("logicalType") != logical: + prop.pop(key) + dropped.append(key) + if provider: + phys = fluid_to_physical(fld.get("type", "string"), provider, api_version) + if phys: + prop["physicalType"] = phys + if prop.get("logicalType") == "map" and "map" not in prop: + prop["map"] = copy.deepcopy(_DEFAULT_MAP_SHAPE) + if dropped: + LOG.warning( + "ODCS export: type of field %r changed to %r; dropped stale %s", + fld.get("name"), + prop.get("logicalType"), + ", ".join(dropped), + ) + + +def _expose_provider(expose: Mapping[str, Any]) -> Optional[str]: + binding = expose.get("binding") + if isinstance(binding, Mapping): + prov = binding.get("platform") or binding.get("provider") + if prov: + return prov + return expose.get("provider") + + +def _platform_to_physical_type(provider: Optional[str]) -> str: + if provider and provider.lower() == "kafka": + return "topic" + return "table" diff --git a/fluid_build/providers/odcs/mappers/servers.py b/fluid_build/providers/odcs/mappers/servers.py new file mode 100644 index 00000000..8671f7cb --- /dev/null +++ b/fluid_build/providers/odcs/mappers/servers.py @@ -0,0 +1,540 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +"""ODCS servers ↔ FLUID expects / exposes binding. + +ODCS ``servers[]`` carry one entry per source/sink. FLUID models the same +information in two places: each ``expose.binding`` (where data is produced) +and each ``expect`` (where data is consumed). On export we read from both; +on import every server becomes an ``expects[]`` entry, and an expose's +``binding`` is derived from the server that unambiguously backs its +SchemaObject (see :func:`binding_for_object`). + +Round-trip: the imported ``servers`` list is stashed verbatim. On export it +is replayed as-is unless the FLUID ``expects`` (or a derived expose +``binding``) changed since import; then only the edited servers are +rebuilt, keeping every stashed key FLUID does not model. +""" + +from __future__ import annotations + +import copy +import re +from collections.abc import Mapping +from typing import Any, Dict, List, Optional, Sequence, Tuple + +from .base import ( + ExportCtx, + ImportCtx, + get_expose_passthrough, + get_metadata_passthrough, + get_stash, + metadata_passthrough, + overlay_native, + target_api_version, +) +from .types import provider_to_server_type, server_type_to_provider + +# Binding-level FLUID keys that ODCS carries on the server itself. FLUID's +# schema requires ``binding.format`` and forbids it in ``binding.location``, +# so the binding is the only place to find it. +_BINDING_LEVEL_KEYS = ("format", "delimiter") + +# Server keys that are not type-specific — kept when a server's type changes. +_SERVER_COMMON_KEYS = frozenset( + {"id", "server", "description", "environment", "roles", "customProperties"} +) + +# Characters outside the v3.1.0 ``StableId`` pattern (``^[A-Za-z0-9_-]+$``, +# which also satisfies the looser v3.2.0 pattern). +_STABLE_ID_INVALID = re.compile(r"[^A-Za-z0-9_-]") + +# ----- ODCS → FLUID -------------------------------------------------------- + + +def to_fluid(ctx: ImportCtx) -> None: + if "servers" not in ctx.odcs: + return + servers = ctx.odcs.get("servers") + if not isinstance(servers, list): + return + # Verbatim pass-through so re-export reproduces the exact same servers + # list — including an explicit empty one. + metadata_passthrough(ctx.fluid)["servers"] = [ + copy.deepcopy(dict(s)) for s in servers if isinstance(s, Mapping) + ] + expects = ctx.fluid.setdefault("expects", []) + for server in servers: + if not isinstance(server, Mapping): + continue + expect = _server_to_expect(server) + if expect: + expects.append(expect) + + +def _server_id(server: Mapping[str, Any]) -> Optional[str]: + # ODCS canonical id lives at ``server`` (v3.1.0); ``name`` is accepted + # for back-compat with legacy fixtures, then ``id``. + return server.get("server") or server.get("name") or server.get("id") + + +def _server_to_expect(server: Mapping[str, Any]) -> Optional[Dict[str, Any]]: + server_type = server.get("type") + if not server_type: + return None + expect: Dict[str, Any] = { + "id": _server_id(server) or "dependency", + "provider": server_type_to_provider(server_type), + } + location = _location_from_server(server) + if location: + expect["location"] = location + return expect + + +def _location_from_server(server: Mapping[str, Any]) -> Dict[str, Any]: + location: Dict[str, Any] = {} + for key in ( + "project", + "dataset", + "table", + "account", + "database", + "schema", + "bucket", + "path", + "region", + "host", + "port", + "format", + # ODCS v3.2.0 IcebergServer + "catalog", + "catalogUrl", + "namespace", + "warehouse", + ): + if key in server: + location[key] = copy.deepcopy(server[key]) + return location + + +def _select_server( + schema_object: Mapping[str, Any], servers: Sequence[Any] +) -> Optional[Tuple[int, Mapping[str, Any]]]: + """The server that unambiguously backs ``schema_object``, if any. + + Either the single typed server whose ``server`` id equals the object's + ``name``, or — when no server is named after it — the document's only + typed server. Anything else is ambiguous and yields ``None``. + """ + typed = [ + (i, s) for i, s in enumerate(servers or []) if isinstance(s, Mapping) and s.get("type") + ] + name = schema_object.get("name") + named = [(i, s) for i, s in typed if name and _server_id(s) == name] + if len(named) == 1: + return named[0] + if not named and len(typed) == 1: + return typed[0] + return None + + +def _platform_for_server_type(server_type: str) -> str: + """FLUID ``binding.platform`` for an ODCS ``Server.type``. + + Uses the usual provider name (``bigquery`` → ``gcp``) unless that mapping + is lossy (``redshift`` → ``aws`` → ``s3``), in which case the server type + itself is the platform. + """ + provider = server_type_to_provider(server_type) + if provider_to_server_type(provider, "v3.2.0") != str(server_type).lower(): + return str(server_type) + return provider + + +def _binding_from_server(server: Mapping[str, Any], schema_object: Mapping[str, Any]): + binding: Dict[str, Any] = {"platform": _platform_for_server_type(server["type"])} + location = _location_from_server(server) + fmt = location.pop("format", None) + if fmt is not None: + binding["format"] = fmt + if location: + binding["location"] = location + if schema_object.get("physicalName"): + binding["physical_name"] = schema_object["physicalName"] + return binding + + +def binding_for_object( + schema_object: Mapping[str, Any], servers: Sequence[Any] +) -> Optional[Dict[str, Any]]: + """FLUID ``binding`` for a SchemaObject, derived from the ODCS servers. + + ``SchemaObject.physicalType`` says *what* the object is (table, view, + topic), not *where* it lives, so it is never used to guess a platform. + Returns ``None`` when no server unambiguously backs the object. + """ + selected = _select_server(schema_object, servers) + if selected is None: + return None + return _binding_from_server(selected[1], schema_object) + + +# ----- FLUID → ODCS -------------------------------------------------------- + + +def to_odcs(ctx: ExportCtx) -> None: + """Emit ``servers`` from FLUID exposes/expects. + + ``servers`` is optional in every ODCS version we vendor. Precedence: + + 1. Imported doc that had a ``servers`` list — replayed verbatim (empty + list included) when FLUID's ``expects`` and derived expose bindings + are unchanged since import; otherwise a three-way merge rebuilds only + the edited servers, drops servers whose expect was removed (unless + an expose binding still selects them) and adds servers for new + expects. + 2. Imported doc that had *no* ``servers`` — emit none unless FLUID added + expects/bindings since, so the round-trip stays lossless and we never + synthesise a server that the source platform can't validate. + 3. Fresh FLUID — synthesise from bindings. + """ + api_version = target_api_version(ctx) + pt = get_metadata_passthrough(ctx.fluid) + stashed = pt.get("servers") + if isinstance(stashed, list): + ctx.odcs["servers"] = _merge_servers( + ctx.fluid, [s for s in stashed if isinstance(s, Mapping)], api_version + ) + return + if get_stash(pt) is not None: + servers = _merge_servers(ctx.fluid, [], api_version) + if servers: + ctx.odcs["servers"] = servers + return + + servers: List[Dict[str, Any]] = [] + for expose in ctx.fluid.get("exposes") or []: + s = _expose_to_server(expose, api_version) + if s: + servers.append(s) + for expect in ctx.fluid.get("expects") or []: + s = _expect_to_server(expect, api_version) + if s: + servers.append(s) + ctx.odcs["servers"] = servers + + +def _merge_servers( + fluid: Mapping[str, Any], stashed: List[Mapping[str, Any]], api_version: str +) -> List[Dict[str, Any]]: + """Three-way merge of the stashed servers with the current FLUID.""" + then_expects = [_server_to_expect(s) for s in stashed] + now_expects = [e for e in fluid.get("expects") or [] if isinstance(e, Mapping)] + + binding_edits = [] + # Stashed servers an expose binding still selects: they stay even when + # the expects projected from them are removed. + bound = set() + for expose in fluid.get("exposes") or []: + if not isinstance(expose, Mapping): + continue + obj_stash = get_stash(get_expose_passthrough(expose)) + selected = _select_server(obj_stash, stashed) if obj_stash is not None else None + then_b = _binding_from_server(selected[1], obj_stash) if selected else None + now_b = expose.get("binding") if isinstance(expose.get("binding"), Mapping) else None + if selected and now_b is not None: + bound.add(selected[0]) + if now_b != then_b: + binding_edits.append((selected[0] if selected else None, then_b, expose)) + + if now_expects == [e for e in then_expects if e is not None] and not binding_edits: + return [copy.deepcopy(dict(s)) for s in stashed] + + pending: Dict[Any, List[Mapping[str, Any]]] = {} + for expect in now_expects: + pending.setdefault(expect.get("id"), []).append(expect) + consumed = set() + out: List[Dict[str, Any]] = [] + out_pos: Dict[int, int] = {} + for i, (server, then_e) in enumerate(zip(stashed, then_expects)): + if then_e is None: + # Type-less server: FLUID has no projection of it to edit. + out_pos[i] = len(out) + out.append(copy.deepcopy(dict(server))) + continue + bucket = pending.get(then_e["id"]) + if not bucket: + if i in bound: + # Its expect was removed, but an expose binding still selects + # it: keep it (a binding edit is merged onto it below). + out_pos[i] = len(out) + out.append(copy.deepcopy(dict(server))) + continue # otherwise its expect was removed in FLUID + now_e = bucket.pop(0) + consumed.add(id(now_e)) + if now_e == then_e: + merged: Optional[Dict[str, Any]] = copy.deepcopy(dict(server)) + else: + same_provider = now_e.get("provider") == then_e.get("provider") + override = server.get("type") if same_provider else None + merged = _merge_server( + server, + _expect_to_server(then_e, api_version, override), + _expect_to_server(now_e, api_version, override), + ) + if merged is not None: + out_pos[i] = len(out) + out.append(merged) + + for expect in now_expects: + if id(expect) not in consumed: + s = _expect_to_server(expect, api_version) + if s: + out.append(s) + + for index, then_b, expose in binding_edits: + now_b = expose.get("binding") + if not isinstance(now_b, Mapping): + continue # binding removed; the server is still owned by its expect + if then_b is not None and index is not None and index in out_pos: + pos = out_pos[index] + same_platform = now_b.get("platform") == then_b.get("platform") + override = stashed[index].get("type") if same_platform else None + merged = _merge_server( + out[pos], + _expose_to_server({**expose, "binding": then_b}, api_version, override), + _expose_to_server(expose, api_version, override), + ) + if merged is not None: + out[pos] = merged + else: + s = _expose_to_server(expose, api_version) + if s: + out.append(s) + return out + + +def _merge_server( + stashed: Mapping[str, Any], + then_srv: Optional[Mapping[str, Any]], + now_srv: Optional[Mapping[str, Any]], +) -> Optional[Dict[str, Any]]: + """Overlay the FLUID edit (``now_srv`` − ``then_srv``) onto a stashed server.""" + if now_srv is None: + return None + if then_srv is None or then_srv.get("type") != now_srv.get("type"): + # Type changed: type-specific stashed keys no longer apply. + out = {k: copy.deepcopy(v) for k, v in stashed.items() if k in _SERVER_COMMON_KEYS} + out.update({k: copy.deepcopy(v) for k, v in now_srv.items() if k not in ("id", "server")}) + out.setdefault("server", now_srv.get("server")) + return out + keys = (set(then_srv) | set(now_srv)) - {"id", "server", "type"} + return overlay_native(stashed, then_srv, now_srv, keys=keys) + + +def _stable_id(value: Any) -> Optional[str]: + """Sanitise ``value`` to the ODCS ``StableId`` pattern (``None`` if empty).""" + if value is None: + return None + sid = _STABLE_ID_INVALID.sub("_", str(value)) + return sid or None + + +def _build_server( + entity: Mapping[str, Any], + entity_id: str, + api_version: Optional[str], + provider_override: Optional[str], +) -> Optional[Dict[str, Any]]: + binding = entity.get("binding") if isinstance(entity.get("binding"), Mapping) else None + provider = None + location: Optional[Mapping[str, Any]] = None + if binding: + provider = binding.get("platform") or binding.get("provider") + location = binding.get("location") if isinstance(binding.get("location"), Mapping) else None + if not provider: + provider = entity.get("provider") + if not provider: + return None + if provider_override: + provider = provider_override + + server: Dict[str, Any] = {} + sid = _stable_id(entity_id) + if sid: + server["id"] = sid + server["server"] = entity_id + server["type"] = provider_to_server_type(str(provider), api_version) + + if not isinstance(location, Mapping): + loc = entity.get("location") + location = loc if isinstance(loc, Mapping) else None + merged: Dict[str, Any] = dict(location) if isinstance(location, Mapping) else {} + if binding: + # Location values win; binding-level keys are the fallback. + for key in _BINDING_LEVEL_KEYS: + if key in binding and key not in merged: + merged[key] = binding[key] + if merged: + details_provider = "custom" if server["type"] == "custom" else str(provider) + server.update(copy.deepcopy(_server_details_from_location(merged, details_provider))) + return server + + +def _expose_to_server( + expose: Any, + api_version: Optional[str] = None, + provider_override: Optional[str] = None, +) -> Optional[Dict[str, Any]]: + if not isinstance(expose, Mapping): + return None + expose_id = expose.get("exposeId") or expose.get("id") or "default" + return _build_server(expose, expose_id, api_version, provider_override) + + +def _expect_to_server( + expect: Any, + api_version: Optional[str] = None, + provider_override: Optional[str] = None, +) -> Optional[Dict[str, Any]]: + if not isinstance(expect, Mapping): + return None + expect_id = expect.get("id") or "dependency" + return _build_server(expect, expect_id, api_version, provider_override) + + +def _server_details_from_location(location: Mapping[str, Any], provider: str) -> Dict[str, Any]: + """Map FLUID binding/location → ODCS server-type-specific allowed fields. + + ODCS servers use ``unevaluatedProperties: false`` — emitting any field + outside the allowed list for the server's type breaks validation. This + table mirrors the ``$defs/ServerSource/.properties`` keys + (v3.1.0 set plus the v3.2.0 engines); FLUID fields with no ODCS + counterpart are quietly dropped here and surface elsewhere via expose + pass-through if needed. + """ + p = provider.lower() + details: Dict[str, Any] = {} + + if p in ("gcp", "bigquery"): + for key in ("project", "dataset"): + if key in location: + details[key] = location[key] + elif p == "snowflake": + for key in ("host", "port", "account", "database", "schema", "warehouse"): + if key in location: + details[key] = location[key] + elif p in ("aws", "s3"): + # ODCS S3Server: location (URI), endpointUrl, format, delimiter + loc_val = location.get("location") or _build_s3_uri( + location.get("bucket"), location.get("path") or location.get("key") + ) + if loc_val: + details["location"] = loc_val + for key in ("endpointUrl", "format", "delimiter"): + if key in location: + details[key] = location[key] + elif p == "kafka": + host = location.get("host") or location.get("account") + if host: + details["host"] = host + if "format" in location: + details["format"] = location["format"] + elif p in ("postgres", "postgresql"): + for key in ("host", "port", "database", "schema"): + if key in location: + details[key] = location[key] + elif p == "mysql": + for key in ("host", "port", "database"): + if key in location: + details[key] = location[key] + elif p == "databricks": + for key in ("host", "catalog", "schema"): + if key in location: + details[key] = location[key] + elif p == "redshift": + for key in ("host", "database", "schema", "region", "account"): + if key in location: + details[key] = location[key] + elif p == "local": + for key in ("path", "format"): + if key in location: + details[key] = location[key] + elif p in ("athena",): + for key in ("stagingDir", "schema", "catalog", "regionName"): + if key in location: + details[key] = location[key] + # ---- ODCS v3.2.0 engines -------------------------------------------- + elif p == "iceberg": + for key in ("catalog", "catalogUrl", "namespace", "warehouse"): + if key in location: + details[key] = location[key] + elif p in ("hana", "sap-hana", "saphana"): + for key in ("host", "port", "database", "schema"): + if key in location: + details[key] = location[key] + elif p == "teradata": + for key in ("host", "port", "database"): + if key in location: + details[key] = location[key] + elif p == "exasol": + for key in ("host", "port", "schema"): + if key in location: + details[key] = location[key] + elif p in ( + "ingres", + "vectorwise", + "actian", + "versant", + "zen", + "btrieve", + "poet", + "fastobjects", + ): + # Actian family (Ingres / Vectorwise / Versant / Zen) and Poet / + # FastObjects all share the {host, port, database} shape. + for key in ("host", "port", "database"): + if key in location: + details[key] = location[key] + else: + # Custom / unknown provider → ODCS server.type=custom which accepts + # the union of all server-type fields (``CustomServer.properties``, + # identical in v3.1.0 and v3.2.0 apart from v3.2.0's ``encoding``). + custom_keys = { + "account", + "catalog", + "database", + "dataset", + "delimiter", + "endpointUrl", + "format", + "host", + "location", + "path", + "port", + "project", + "region", + "regionName", + "schema", + "serviceName", + "stagingDir", + "warehouse", + "stream", + } + for key, value in location.items(): + if key in custom_keys: + details[key] = value + return details + + +def _build_s3_uri(bucket: Optional[str], path: Optional[str]) -> Optional[str]: + if not bucket: + return None + if path: + return f"s3://{bucket.rstrip('/')}/{path.lstrip('/')}" + return f"s3://{bucket}" diff --git a/fluid_build/providers/odcs/mappers/sla.py b/fluid_build/providers/odcs/mappers/sla.py new file mode 100644 index 00000000..7a1c1fa1 --- /dev/null +++ b/fluid_build/providers/odcs/mappers/sla.py @@ -0,0 +1,298 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +"""SLA mapper: ODCS ``slaProperties[]`` ↔ FLUID expose ``qos``. + +The full ODCS list is preserved verbatim under +``metadata.odcs_passthrough.sla_properties`` so that an ODCS → FLUID → ODCS +round-trip is lossless. The first/only expose's ``qos`` block is also +populated (availability, freshnessSLO, labels) so the FLUID view is +meaningful in isolation, and flagged as the projection target. On export the +stash is replayed unless an expose's ``qos`` changed since import (the +flagged expose against the projection, any other against "no qos"); an edit +rewrites only the entries it touches. +""" + +from __future__ import annotations + +import copy +from collections.abc import Mapping, Sequence +from typing import Any, Dict, List, Optional + +from .base import ( + ExportCtx, + ImportCtx, + expose_passthrough, + get_expose_passthrough, + get_metadata_passthrough, + metadata_passthrough, +) + +# Expose pass-through flag: this expose's ``qos`` is the import's projection +# of the document-level ``slaProperties``. +SLA_PROJECTION_KEY = "sla_projection" + +# ----- ODCS → FLUID -------------------------------------------------------- + + +def to_fluid(ctx: ImportCtx) -> None: + sla_props = ctx.odcs.get("slaProperties") + if not isinstance(sla_props, list): + return + + # Verbatim pass-through (canonical source of truth on re-export) — an + # explicit empty list included. + metadata_passthrough(ctx.fluid)["sla_properties"] = [ + copy.deepcopy(dict(p)) for p in sla_props if isinstance(p, Mapping) + ] + if not sla_props: + return + + exposes = ctx.fluid.get("exposes") or [] + target = exposes[0] if exposes else None + if not isinstance(target, dict): + return + # Record which expose received the projection: a per-expose export of + # another SchemaObject (or the scoped view of this one) compares against + # what *that* expose was given at import, not against exposes[0]. + expose_passthrough(target)[SLA_PROJECTION_KEY] = True + + qos: Dict[str, Any] = target.get("qos") or {} + labels: Dict[str, Any] = qos.get("labels") or {} + + for prop in sla_props: + if not isinstance(prop, Mapping): + continue + _apply_sla_to_qos(prop, qos, labels) + + if labels: + qos["labels"] = labels + if qos: + target["qos"] = qos + + +def _project_qos(sla_props: Sequence[Any]) -> Optional[Dict[str, Any]]: + """The ``qos`` block an import projects from ``slaProperties`` (or ``None``).""" + qos: Dict[str, Any] = {} + labels: Dict[str, Any] = {} + for prop in sla_props: + if isinstance(prop, Mapping): + _apply_sla_to_qos(prop, qos, labels) + if labels: + qos["labels"] = labels + return qos or None + + +def _apply_sla_to_qos(prop: Mapping[str, Any], qos: Dict[str, Any], labels: Dict[str, Any]) -> None: + name = prop.get("property") + value = copy.deepcopy(prop.get("value")) + unit = prop.get("unit") + if not name: + return + if name == "availability": + try: + fraction = float(value) + qos["availability"] = f"{fraction * 100:g}%" if fraction <= 1 else f"{fraction:g}%" + except (TypeError, ValueError): + qos["availability"] = str(value) + elif name in _FRESHNESS_PROPERTIES: + qos.setdefault("freshnessSLO", str(value)) + elif name.startswith("label:"): + labels[name[len("label:") :]] = value + else: + labels[_label_key(name, unit)] = value + + +_FRESHNESS_PROPERTIES = ("interval", "frequency", "latency", "retention") + + +def _label_key(name: str, unit: Any) -> str: + if name.startswith("label:"): + return name[len("label:") :] + return f"{name}:{unit}" if unit else name + + +# ----- FLUID → ODCS -------------------------------------------------------- + + +def to_odcs(ctx: ExportCtx) -> None: + if not ctx.options.get("include_sla", True): + return + + # Imported contract: three-way merge with the verbatim pass-through. + raw = get_metadata_passthrough(ctx.fluid).get("sla_properties") + if isinstance(raw, list): + stashed = [p for p in raw if isinstance(p, Mapping)] + merged = _merge_sla(ctx.fluid, stashed) + if merged is not None: + ctx.odcs["slaProperties"] = merged + return + + properties = list(_build_sla_properties(ctx.fluid)) + if properties: + ctx.odcs["slaProperties"] = properties + + +def _merge_sla( + fluid: Mapping[str, Any], stashed: List[Mapping[str, Any]] +) -> Optional[List[Dict[str, Any]]]: + """Replay the stash unless some expose's ``qos`` changed; then apply the edit. + + Each expose is compared with what the import gave it: the expose that + received the projection (flagged with :data:`SLA_PROJECTION_KEY`) with + the projection of the stash, every other expose with "no qos". A + per-expose export of another SchemaObject therefore replays the + document-level SLA untouched, and a qos edit on any expose is honoured. + + Only the keys FLUID projects (``availability``, ``freshnessSLO``, + ``labels``) are compared; each edit rewrites, removes or appends just + the ``slaProperties`` entry it came from, so units, elements and other + unmodelled entries survive. + """ + out = [copy.deepcopy(dict(p)) for p in stashed] + projection = _project_qos(stashed) or {} + for expose in fluid.get("exposes") or []: + if not isinstance(expose, Mapping): + continue + projected = bool(get_expose_passthrough(expose).get(SLA_PROJECTION_KEY)) + then = projection if projected else {} + now = expose.get("qos") if isinstance(expose.get("qos"), Mapping) else {} + if _qos_view(now) != _qos_view(then): + _apply_qos_edit(out, then, now) + return out + + +def _apply_qos_edit( + out: List[Dict[str, Any]], then: Mapping[str, Any], now: Mapping[str, Any] +) -> None: + """Rewrite the ``slaProperties`` entries a ``then`` → ``now`` qos edit touches.""" + # availability + if now.get("availability") != then.get("availability"): + idx = _find(out, lambda p: p.get("property") == "availability") + parsed: Dict[str, Any] = {} + if now.get("availability") is not None: + _qos_to_sla({"availability": now["availability"]}, parsed) + _apply_edit(out, idx, "availability", parsed.get("availability")) + + # freshnessSLO — the first interval/frequency/latency/retention entry + now_f = now.get("freshnessSLO") or now.get("freshness_slo") + if now_f != then.get("freshnessSLO"): + idx = _find(out, lambda p: p.get("property") in _FRESHNESS_PROPERTIES) + prop_name = out[idx]["property"] if idx is not None else "interval" + _apply_edit(out, idx, prop_name, str(now_f) if now_f else None) + + # labels — keyed like the import projection; the last entry wins there + then_l = then.get("labels") or {} + now_l = now.get("labels") if isinstance(now.get("labels"), Mapping) else {} + for key in list(then_l) + [k for k in now_l if k not in then_l]: + if now_l.get(key) == then_l.get(key) and (key in now_l) == (key in then_l): + continue + idx = _find( + out, + lambda p, key=key: p.get("property") + and p.get("property") != "availability" + and p.get("property") not in _FRESHNESS_PROPERTIES + and _label_key(p["property"], p.get("unit")) == key, + last=True, + ) + prop_name = out[idx]["property"] if idx is not None else f"label:{key}" + _apply_edit(out, idx, prop_name, now_l.get(key) if key in now_l else None) + + +def _qos_view(qos: Mapping[str, Any]) -> Dict[str, Any]: + return { + "availability": qos.get("availability"), + "freshnessSLO": qos.get("freshnessSLO") or qos.get("freshness_slo"), + "labels": dict(qos.get("labels") or {}) or None, + } + + +def _find(items: List[Dict[str, Any]], pred, last: bool = False) -> Optional[int]: + hits = [i for i, p in enumerate(items) if pred(p)] + if not hits: + return None + return hits[-1] if last else hits[0] + + +def _apply_edit(items: List[Dict[str, Any]], idx: Optional[int], name: str, value: Any) -> None: + """Set (``idx``), remove (``value is None``) or append an SLA entry.""" + if value is None: + if idx is not None: + items.pop(idx) + return + if idx is not None: + items[idx]["value"] = _coerce_like(items[idx].get("value"), value) + else: + items.append({"property": name, "value": copy.deepcopy(value)}) + + +def _coerce_like(old: Any, new: Any) -> Any: + """Keep the stashed value's numeric type when the edit is a numeric string.""" + if isinstance(new, str) and isinstance(old, (int, float)) and not isinstance(old, bool): + try: + number = float(new) + except ValueError: + return new + return int(number) if isinstance(old, int) and number.is_integer() else number + return copy.deepcopy(new) + + +def _build_sla_properties(fluid: Mapping[str, Any]) -> Sequence[Dict[str, Any]]: + sla_values: Dict[str, Any] = {} + for expose in fluid.get("exposes", []): + if not isinstance(expose, Mapping): + continue + qos = expose.get("qos") or {} + if not isinstance(qos, Mapping): + continue + _qos_to_sla(qos, sla_values) + + metadata = fluid.get("metadata") or {} + if isinstance(metadata, Mapping): + if metadata.get("update_frequency") and "interval" not in sla_values: + sla_values["interval"] = metadata["update_frequency"] + if metadata.get("availability") and "availability" not in sla_values: + try: + a = float(metadata["availability"]) + sla_values["availability"] = a / 100 if a > 1 else a + except (TypeError, ValueError): + pass + if metadata.get("quality_threshold"): + try: + sla_values["completenessKpi"] = float(metadata["quality_threshold"]) + except (TypeError, ValueError): + pass + + out: List[Dict[str, Any]] = [] + for prop, value in sla_values.items(): + if value is None: + continue + out.append({"property": prop, "value": copy.deepcopy(value)}) + return out + + +def _qos_to_sla(qos: Mapping[str, Any], sla_values: Dict[str, Any]) -> None: + avail = qos.get("availability") + if avail is not None and "availability" not in sla_values: + try: + s = str(avail).strip() + if s.endswith("%"): + sla_values["availability"] = float(s.rstrip("%")) / 100 + else: + parsed = float(s) + sla_values["availability"] = parsed / 100 if parsed > 1 else parsed + except (TypeError, ValueError): + sla_values["availability"] = str(avail) + + freshness = qos.get("freshnessSLO") or qos.get("freshness_slo") + if freshness and "interval" not in sla_values: + sla_values["interval"] = str(freshness) + + labels = qos.get("labels") or {} + if isinstance(labels, Mapping): + for k, v in labels.items(): + sla_values.setdefault(f"label:{k}", v) diff --git a/fluid_build/providers/odcs/mappers/team.py b/fluid_build/providers/odcs/mappers/team.py new file mode 100644 index 00000000..dd41f65c --- /dev/null +++ b/fluid_build/providers/odcs/mappers/team.py @@ -0,0 +1,266 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +"""Team ↔ Owner mapping. + +ODCS v3.1.0 ``team`` is an **object**: +``{id, name, description, members[], tags, customProperties, authoritativeDefinitions}``. +Each ``members[]`` entry has a required ``username`` (often an email) plus +optional ``name``, ``role``, ``dateIn``, ``dateOut``, ``replacedByUsername``, +``description``, ``tags``, ``customProperties``, ``authoritativeDefinitions``. + +The v3.0.x array-of-members form (deprecated, still accepted by v3.1.0) is +read the same way; a legacy string form is accepted on import (best-effort). + +For lossless round-trip the verbatim team (object or array) is preserved +under ``metadata.odcs_passthrough.team`` and replayed unless the FLUID +``owner`` changed since import; an edit is merged onto it, keeping every +member key FLUID does not model. +""" + +from __future__ import annotations + +import copy +from collections.abc import Mapping +from typing import Any, Dict, List, Optional + +from .base import ( + ExportCtx, + ImportCtx, + get_metadata_passthrough, + metadata_passthrough, +) + +# ----- ODCS → FLUID -------------------------------------------------------- + + +def to_fluid(ctx: ImportCtx) -> None: + team = ctx.odcs.get("team") + if team is None: + return + owner = _import_owner(team) + if owner: + ctx.fluid["owner"] = owner + if isinstance(team, (Mapping, list)): + # Verbatim, whatever its shape — including the deprecated v3.0.x + # array-of-members form and an explicit empty ``{}`` / ``[]``. + metadata_passthrough(ctx.fluid)["team"] = copy.deepcopy(team) + + +def _import_owner(team: Any) -> Optional[Dict[str, Any]]: + """The FLUID ``owner`` an import projects from ``team`` (``None`` if empty).""" + if not team: + return None + return _team_to_owner(team) + + +def _team_to_owner(team: Any) -> Optional[Dict[str, Any]]: + if isinstance(team, str): + return {"team": team, "name": team} + if isinstance(team, list): + # ODCS v3.0.x ``team`` is an array of members (deprecated in v3.1.0). + team = {"members": team} + if not isinstance(team, Mapping): + return None + + owner: Dict[str, Any] = {} + team_name = team.get("name") + if team_name: + owner["team"] = team_name + + members = team.get("members") or [] + if not isinstance(members, list): + members = [] + + first = members[0] if members else None + rest = members[1:] if members else [] + + if isinstance(first, Mapping): + if first.get("name"): + owner["name"] = first["name"] + if first.get("username"): + owner["email"] = first["username"] + if first.get("role"): + owner["role"] = first["role"] + + contacts: List[Dict[str, Any]] = [] + for member in rest: + if not isinstance(member, Mapping): + continue + contact: Dict[str, Any] = {} + if member.get("name"): + contact["name"] = member["name"] + if member.get("username"): + contact["email"] = member["username"] + if member.get("role"): + contact["role"] = member["role"] + if contact: + contacts.append(contact) + if contacts: + owner["contacts"] = contacts + + if not owner: + owner["team"] = team_name or "unknown" + return owner + + +# ----- FLUID → ODCS -------------------------------------------------------- + + +def _current_owner(fluid: Mapping[str, Any]) -> Any: + return fluid.get("owner") or (fluid.get("metadata") or {}).get("owner") or None + + +def to_odcs(ctx: ExportCtx) -> None: + pt = get_metadata_passthrough(ctx.fluid) + pt_team = pt.get("team") + if isinstance(pt_team, (Mapping, list)): + # Imported contract: three-way merge, like metadata and schema. The + # stashed team is replayed verbatim unless the FLUID owner changed. + then = _import_owner(pt_team) + now = _current_owner(ctx.fluid) + if now == then: + ctx.odcs["team"] = copy.deepcopy(pt_team) + return + merged = _merge_team(pt_team, then or {}, ctx.fluid) + if merged is not None: + ctx.odcs["team"] = merged + return + + team = _owner_to_team(ctx.fluid) + if team: + ctx.odcs["team"] = team + + +# TeamMember keys FLUID models (owner/contacts name, email, role). +_MEMBER_NATIVE_KEYS = ("name", "username", "role") + + +def _merge_team(stashed: Any, then: Mapping[str, Any], fluid: Mapping[str, Any]) -> Any: + """Apply a FLUID owner edit to the stashed team, keeping unmodelled keys. + + Only a removed owner removes the team. A stashed team without a name + (the v3.0.x array form, or an object without ``name``) keeps its shape + and stays nameless unless the owner's ``team`` is set. + """ + now = _current_owner(fluid) + if not isinstance(now, Mapping) or not now: + return None # owner removed + + stashed_members = stashed.get("members") if isinstance(stashed, Mapping) else stashed + stashed_members = [m for m in stashed_members or [] if isinstance(m, Mapping)] + member_keys = ("name", "email", "role", "contacts") + if any(now.get(k) != then.get(k) for k in member_keys): + members = _merge_members(stashed_members, _owner_members(now)) + else: + members = copy.deepcopy(stashed_members) + + team_changed = now.get("team") != then.get("team") + if isinstance(stashed, list) and not team_changed: + return members # keep the array form when only members changed + + out = copy.deepcopy(dict(stashed)) if isinstance(stashed, Mapping) else {} + if team_changed: + name = now.get("team") or now.get("name") + if name: + out["name"] = name + else: + out.pop("name", None) + if members: + out["members"] = members + else: + out.pop("members", None) + return out + + +def _merge_members( + stashed: List[Mapping[str, Any]], fresh: List[Mapping[str, Any]] +) -> List[Dict[str, Any]]: + """Fresh members, each keeping the stashed extras (dateIn, tags, ...) of + the stashed member with the same ``username`` (or ``name``) — or, when + none matches, of the stashed member at the same position whose own key + no longer appears (its username was edited).""" + + def key_of(m: Mapping[str, Any]) -> Any: + return m.get("username") or m.get("name") + + fresh_keys = {key_of(m) for m in fresh} + out: List[Dict[str, Any]] = [] + used = set() + for pos, member in enumerate(fresh): + key = key_of(member) + match = None + for i, candidate in enumerate(stashed): + if i not in used and key and key_of(candidate) == key: + match = i + break + if ( + match is None + and pos < len(stashed) + and pos not in used + and key_of(stashed[pos]) not in fresh_keys + ): + match = pos + merged: Dict[str, Any] = {} + if match is not None: + used.add(match) + merged = { + k: copy.deepcopy(v) + for k, v in stashed[match].items() + if k not in _MEMBER_NATIVE_KEYS + } + merged.update(copy.deepcopy(dict(member))) + out.append(merged) + return out + + +def _owner_to_team(fluid: Mapping[str, Any]) -> Optional[Dict[str, Any]]: + owner = fluid.get("owner") or (fluid.get("metadata") or {}).get("owner") or {} + if not isinstance(owner, Mapping): + owner = {} + + team_name = owner.get("team") or owner.get("name") + if not team_name: + return None + + team_obj: Dict[str, Any] = {"name": team_name} + members = _owner_members(owner) + if members: + team_obj["members"] = members + return team_obj + + +def _owner_members(owner: Mapping[str, Any]) -> List[Dict[str, Any]]: + """ODCS ``TeamMember`` entries for a FLUID owner (primary, then contacts).""" + members: List[Dict[str, Any]] = [] + + # Primary contact from owner top-level + if owner.get("name") or owner.get("email"): + member: Dict[str, Any] = {} + if owner.get("name"): + member["name"] = owner["name"] + if owner.get("email"): + # ODCS TeamMember.username is the email-or-username slot + member["username"] = owner["email"] + if owner.get("role"): + member["role"] = owner["role"] + members.append(member) + + # Additional contacts + for contact in owner.get("contacts") or []: + if not isinstance(contact, Mapping): + continue + member = {} + if contact.get("name"): + member["name"] = contact["name"] + if contact.get("email"): + member["username"] = contact["email"] + if contact.get("role"): + member["role"] = contact["role"] + if member: + members.append(member) + return members diff --git a/fluid_build/providers/odcs/mappers/types.py b/fluid_build/providers/odcs/mappers/types.py new file mode 100644 index 00000000..005b9a9d --- /dev/null +++ b/fluid_build/providers/odcs/mappers/types.py @@ -0,0 +1,355 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +"""Pure type-mapping tables: FLUID ↔ ODCS logicalType and provider physical types.""" + +from __future__ import annotations + +from typing import Dict, FrozenSet, Mapping, Optional, Tuple + +from ..validation import DEFAULT_API_VERSION, api_version_tuple + +# First ODCS version with the ``map``/``vector`` logical types and the +# Iceberg / HANA / Teradata / Exasol / Actian / Poet server engines. +_V320 = (3, 2, 0) + + +def supports_v320(api_version: Optional[str]) -> bool: + """True when the target ODCS ``apiVersion`` is v3.2.0 or newer. + + ``None`` means "the emit default" (:data:`DEFAULT_API_VERSION`). + """ + return api_version_tuple(api_version or DEFAULT_API_VERSION) >= _V320 + + +# FLUID source type → ODCS logicalType. v3.1.0 enum: string,date,timestamp, +# time,number,integer,object,array,boolean; v3.2.0 adds map (RFC 0030) and +# vector (RFC 0042). +_FLUID_TO_LOGICAL: Dict[str, str] = { + "string": "string", + "text": "string", + "varchar": "string", + "char": "string", + "int": "integer", + "integer": "integer", + "bigint": "integer", + "long": "integer", + "int8": "integer", + "int16": "integer", + "int32": "integer", + "int64": "integer", + "smallint": "integer", + "tinyint": "integer", + "byteint": "integer", + "float": "number", + "double": "number", + "decimal": "number", + "numeric": "number", + "number": "number", + "float32": "number", + "float64": "number", + "real": "number", + "bignumeric": "number", + "bool": "boolean", + "boolean": "boolean", + "date": "date", + "datetime": "timestamp", + "timestamp": "timestamp", + "timestamp_ntz": "timestamp", + "timestamp_tz": "timestamp", + "timestamp_ltz": "timestamp", + "time": "time", + "json": "object", + "object": "object", + "array": "array", + "map": "map", + "vector": "vector", + # ODCS has no binary logical type; store as string and preserve physical + "binary": "string", + "bytes": "string", +} + +# ODCS logicalType → FLUID type (lossy; the verbatim stash preserves the +# original on round-trip) +_LOGICAL_TO_FLUID: Dict[str, str] = { + "string": "string", + "integer": "int", + "long": "bigint", + "float": "float", + "double": "double", + "decimal": "decimal", + "number": "double", + "boolean": "bool", + "date": "date", + "timestamp": "timestamp", + "time": "time", + "object": "object", + "array": "array", + "map": "map", + "vector": "vector", + "binary": "binary", +} + + +# v3.2.0-only logical types → the closest v3.1.0 logical type. +_PRE_V320_LOGICAL_FALLBACK: Dict[str, str] = {"map": "object", "vector": "array"} + + +def fluid_to_logical(fluid_type: str, api_version: Optional[str] = None) -> str: + """FLUID type → ODCS logicalType (best-effort, defaults to ``string``). + + ``map`` / ``vector`` exist only from ODCS v3.2.0; for an older target + (including the v3.1.0 emit default when ``api_version`` is ``None``) they + fall back to ``object`` / ``array`` so the output stays schema-valid. + """ + logical = _FLUID_TO_LOGICAL.get(str(fluid_type).lower(), "string") + if logical in _PRE_V320_LOGICAL_FALLBACK and not supports_v320(api_version): + return _PRE_V320_LOGICAL_FALLBACK[logical] + return logical + + +def logical_to_fluid(logical_type: str) -> str: + """ODCS logicalType → FLUID type (best-effort, defaults to ``string``).""" + return _LOGICAL_TO_FLUID.get(str(logical_type).lower(), "string") + + +_PHYSICAL_BY_PROVIDER: Dict[Tuple[str, ...], Dict[str, str]] = { + ("gcp", "bigquery"): { + "string": "STRING", + "text": "STRING", + "int": "INT64", + "integer": "INT64", + "bigint": "INT64", + "long": "INT64", + "int8": "INT64", + "int16": "INT64", + "int32": "INT64", + "int64": "INT64", + "smallint": "INT64", + "tinyint": "INT64", + "byteint": "INT64", + "float": "FLOAT64", + "double": "FLOAT64", + "number": "FLOAT64", + "float32": "FLOAT64", + "float64": "FLOAT64", + "real": "FLOAT64", + "decimal": "NUMERIC", + "numeric": "NUMERIC", + "bignumeric": "BIGNUMERIC", + "bool": "BOOL", + "boolean": "BOOL", + "date": "DATE", + "datetime": "DATETIME", + "timestamp": "TIMESTAMP", + "timestamp_ntz": "DATETIME", + "timestamp_tz": "TIMESTAMP", + "timestamp_ltz": "TIMESTAMP", + "time": "TIME", + "json": "JSON", + "object": "STRUCT", + "array": "ARRAY", + "binary": "BYTES", + "bytes": "BYTES", + }, + ("snowflake",): { + "string": "VARCHAR", + "text": "TEXT", + "int": "NUMBER", + "integer": "NUMBER", + "bigint": "NUMBER", + "long": "NUMBER", + "int8": "NUMBER", + "int16": "NUMBER", + "int32": "NUMBER", + "int64": "NUMBER", + "smallint": "NUMBER", + "tinyint": "NUMBER", + "byteint": "NUMBER", + "float": "FLOAT", + "double": "DOUBLE", + "number": "NUMBER", + "float32": "FLOAT", + "float64": "FLOAT", + "real": "FLOAT", + "decimal": "DECIMAL", + "numeric": "DECIMAL", + "bool": "BOOLEAN", + "boolean": "BOOLEAN", + "date": "DATE", + "datetime": "TIMESTAMP_NTZ", + "timestamp": "TIMESTAMP_NTZ", + "timestamp_ntz": "TIMESTAMP_NTZ", + "timestamp_tz": "TIMESTAMP_TZ", + "timestamp_ltz": "TIMESTAMP_LTZ", + "time": "TIME", + "json": "VARIANT", + "object": "OBJECT", + "array": "ARRAY", + "binary": "BINARY", + "bytes": "BINARY", + }, +} + + +def fluid_to_physical( + fluid_type: str, provider: Optional[str], api_version: Optional[str] = None +) -> Optional[str]: + """FLUID type → provider-specific physical type. Falls back to logicalType.""" + if not provider: + return fluid_to_logical(fluid_type, api_version) + prov = provider.lower() + for keys, table in _PHYSICAL_BY_PROVIDER.items(): + if prov in keys: + return table.get(str(fluid_type).lower()) + return fluid_to_logical(fluid_type, api_version) + + +# ODCS server.type ↔ FLUID provider +_PROVIDER_TO_SERVER_TYPE: Dict[str, str] = { + "gcp": "bigquery", + "bigquery": "bigquery", + "snowflake": "snowflake", + "aws": "s3", + "s3": "s3", + "redshift": "redshift", + "athena": "athena", + "azure": "azure", + "databricks": "databricks", + "postgres": "postgres", + "postgresql": "postgres", + "mysql": "mysql", + "kafka": "kafka", + "mongodb": "mongodb", + "elasticsearch": "elasticsearch", + "local": "local", + # ODCS v3.2.0 additions + "iceberg": "iceberg", + "hana": "hana", + "sap-hana": "hana", + "saphana": "hana", + "teradata": "teradata", + "exasol": "exasol", + "ingres": "ingres", + "vectorwise": "vectorwise", + "actian": "vectorwise", + "versant": "versant", + "zen": "zen", + "btrieve": "btrieve", + "poet": "poet", + "fastobjects": "fastobjects", +} + +_SERVER_TYPE_TO_PROVIDER: Dict[str, str] = { + "bigquery": "gcp", + "snowflake": "snowflake", + "s3": "aws", + "redshift": "aws", + "athena": "aws", + "azure": "azure", + "databricks": "databricks", + "postgres": "postgres", + "mysql": "mysql", + "kafka": "kafka", + "mongodb": "mongodb", + "elasticsearch": "elasticsearch", + "local": "local", + # ODCS v3.2.0 additions — identity so ``expects[].provider`` round-trips + # the exact engine name (servers themselves pass through verbatim). + "iceberg": "iceberg", + "hana": "hana", + "teradata": "teradata", + "exasol": "exasol", + "ingres": "ingres", + "vectorwise": "vectorwise", + "versant": "versant", + "zen": "zen", + "btrieve": "btrieve", + "poet": "poet", + "fastobjects": "fastobjects", +} + + +# ``Server.type`` values that only exist from ODCS v3.2.0 (``zen`` is +# already in v3.1.0). +V320_ONLY_SERVER_TYPES: FrozenSet[str] = frozenset( + { + "btrieve", + "exasol", + "fastobjects", + "hana", + "iceberg", + "ingres", + "poet", + "teradata", + "vectorwise", + "versant", + } +) + + +def provider_to_server_type(provider: str, api_version: Optional[str] = None) -> str: + """FLUID provider → ODCS ``Server.type``. + + Engines added in v3.2.0 map to ``custom`` for an older target (including + the v3.1.0 emit default when ``api_version`` is ``None``). + """ + server_type = _PROVIDER_TO_SERVER_TYPE.get(provider.lower(), "custom") + if server_type in V320_ONLY_SERVER_TYPES and not supports_v320(api_version): + return "custom" + return server_type + + +def server_type_to_provider(server_type: str) -> str: + return _SERVER_TYPE_TO_PROVIDER.get(server_type.lower(), "custom") + + +# ODCS SchemaObject.physicalType (table/view/topic/file) → FLUID binding.platform +_PHYSICAL_TYPE_TO_PLATFORM: Mapping[str, str] = { + "topic": "kafka", + "table": "bigquery", + "view": "bigquery", + "file": "s3", +} + + +def physical_type_to_platform(physical_type: str) -> str: + """Legacy guess, kept for API compatibility only. + + The ODCS importer no longer uses it: ``physicalType`` names the object + kind, not its platform, so ``binding.platform`` is derived from the + document's ``servers`` instead (see ``servers.binding_for_object``). + """ + return _PHYSICAL_TYPE_TO_PLATFORM.get(str(physical_type).lower(), "custom") + + +# Status mapping (shared by both directions) +_FLUID_TO_ODCS_STATUS: Dict[str, str] = { + "draft": "draft", + "active": "active", + "deprecated": "deprecated", + "retired": "retired", + "development": "draft", +} + +_ODCS_TO_FLUID_STATUS: Dict[str, str] = { + "draft": "draft", + "active": "active", + "deprecated": "deprecated", + "retired": "retired", +} + + +def fluid_to_odcs_status(status: str) -> str: + # Case-sensitive, exactly like 0.7.8: ``Development`` (not a key) maps to + # ``active``, not ``draft``. Changing that would silently change the + # status every capitalised-status contract publishes. + return _FLUID_TO_ODCS_STATUS.get(str(status), "active") + + +def odcs_to_fluid_status(status: str) -> str: + return _ODCS_TO_FLUID_STATUS.get(str(status).lower(), "active") diff --git a/fluid_build/providers/odcs/odcs-schema-v3.2.0.json b/fluid_build/providers/odcs/odcs-schema-v3.2.0.json new file mode 100644 index 00000000..f9340754 --- /dev/null +++ b/fluid_build/providers/odcs/odcs-schema-v3.2.0.json @@ -0,0 +1,3648 @@ +{ + "$schema": "https://json-schema.org/draft/2019-09/schema", + "title": "Open Data Contract Standard (ODCS)", + "description": "An open data contract specification to establish agreement between data producers and consumers.", + "type": "object", + "properties": { + "version": { + "type": "string", + "description": "Current version of the data contract." + }, + "kind": { + "type": "string", + "default": "DataContract", + "description": "The kind of file this is. Valid value is `DataContract`.", + "enum": ["DataContract"] + }, + "apiVersion": { + "type": "string", + "default": "v3.2.0", + "description": "Version of the standard used to build data contract. Default value is v3.2.0.", + "enum": ["v3.2.0", "v3.1.0", "v3.0.2", "v3.0.1", "v3.0.0", "v2.2.2", "v2.2.1", "v2.2.0"] + }, + "id": { + "type": "string", + "description": "A unique identifier used to reduce the risk of dataset name collisions, such as a UUID." + }, + "name": { + "type": "string", + "description": "Name of the data contract." + }, + "tenant": { + "type": "string", + "description": "Indicates the property the data is primarily associated with. Value is case insensitive." + }, + "tags": { + "$ref": "#/$defs/Tags" + }, + "status": { + "type": "string", + "description": "Current status of the dataset.", + "examples": [ + "proposed", "draft", "active", "deprecated", "retired" + ] + }, + "servers": { + "type": "array", + "description": "List of servers where the datasets reside.", + "items": { + "$ref": "#/$defs/Server" + } + }, + "dataProduct": { + "type": "string", + "description": "The name of the data product. DEPRECATED since v3.1.0.", + "deprecated": true + }, + "description": { + "type": "object", + "description": "High level description of the dataset.", + "properties": { + "usage": { + "type": "string", + "description": "Intended usage of the dataset." + }, + "purpose": { + "type": "string", + "description": "Purpose of the dataset." + }, + "limitations": { + "type": "string", + "description": "Limitations of the dataset." + }, + "authoritativeDefinitions": { + "$ref": "#/$defs/AuthoritativeDefinitions" + }, + "customProperties": { + "$ref": "#/$defs/CustomProperties" + } + } + }, + "domain": { + "type": "string", + "description": "Name of the logical data domain.", + "examples": ["imdb_ds_aggregate", "receiver_profile_out", "transaction_profile_out"] + }, + "context": { + "$ref": "#/$defs/Context" + }, + "schema": { + "type": "array", + "description": "A list of elements within the schema to be cataloged.", + "items": { + "$ref": "#/$defs/SchemaObject" + } + }, + "support": { + "$ref": "#/$defs/Support" + }, + "price": { + "$ref": "#/$defs/Pricing" + }, + "team": { + "oneOf": [ + { + "$ref": "#/$defs/Team", + "description": "Team information object with members array (v3.1.0+)." + }, + { + "type": "array", + "description": "DEPRECATED: Array of team members. Use the Team object structure instead. This array format is maintained for backward compatibility with v3.0.2 and earlier versions and will be removed in ODCS 4.0.", + "deprecated": true, + "items": { + "$ref": "#/$defs/TeamMember" + } + } + ] + }, + "roles": { + "type": "array", + "description": "A list of roles that will provide user access to the dataset.", + "items": { + "$ref": "#/$defs/Role" + } + }, + "slaDefaultElement": { + "type": "string", + "description": "DEPRECATED SINCE 3.1. WILL BE REMOVED IN ODCS 4.0. Element (using the element path notation) to do the checks on.", + "deprecated": true + }, + "slaProperties": { + "type": "array", + "description": "A list of key/value pairs for SLA specific properties. There is no limit on the type of properties (more details to come).", + "items": { + "$ref": "#/$defs/ServiceLevelAgreementProperty" + } + }, + "authoritativeDefinitions": { + "$ref": "#/$defs/AuthoritativeDefinitions" + }, + "customProperties": { + "$ref": "#/$defs/CustomProperties" + }, + "contractCreatedTs": { + "type": "string", + "format": "date-time", + "description": "Timestamp in UTC of when the data contract was created." + } + }, + "required": ["version", "apiVersion", "kind", "id"], + "additionalProperties": false, + "unevaluatedProperties": false, + "$defs": { + "Context": { + "description": "AI and semantic context block (RFC-0038). Structured guidance for AI agents, LLMs, BI tools, and semantic layer platforms. Optional and additive.", + "oneOf": [ + { + "type": "string", + "description": "Shorthand: equivalent to providing instructions only." + }, + { + "type": "object", + "properties": { + "instructions": { + "type": "string", + "description": "Natural language guidance for AI agents and tools on how to use this entity. Equivalent to a system prompt scoped to this level." + }, + "verifiedStatements": { + "type": "array", + "description": "Canonical questions (entries without `answer`) and verified Q&A pairs (entries with `answer`).", + "items": { + "type": "object", + "properties": { + "id": { "type": "string", "description": "Stable identifier for the entry." }, + "question": { "type": "string", "description": "The canonical question." }, + "answer": { "type": "string", "description": "The expected response or result description." }, + "authoritativeDefinitions": { "$ref": "#/$defs/AuthoritativeDefinitions" }, + "tags": { "type": "array", "items": { "type": "string" } }, + "customProperties": { "$ref": "#/$defs/CustomProperties" } + }, + "required": ["question"], + "additionalProperties": false + } + }, + "constraints": { + "type": "array", + "description": "Negative guidance: what AI agents must NOT do with this entity.", + "items": { + "type": "object", + "properties": { + "id": { "type": "string", "description": "Stable identifier for the constraint." }, + "constraint": { "type": "string", "description": "The constraint text (negative guidance for AI agents)." }, + "authoritativeDefinitions": { "$ref": "#/$defs/AuthoritativeDefinitions" }, + "tags": { "type": "array", "items": { "type": "string" } }, + "customProperties": { "$ref": "#/$defs/CustomProperties" } + }, + "required": ["constraint"], + "additionalProperties": false + } + } + }, + "additionalProperties": false + } + ] + }, + "ShorthandReference": { + "type": "string", + "description": "Shorthand notation using name fields (table_name.column_name)", + "pattern": "^[A-Za-z_][A-Za-z0-9_\\-]*(\\.[A-Za-z_][A-Za-z0-9_\\-]*)+$" + }, + "FullyQualifiedReference": { + "type": "string", + "description": "Fully qualified notation using id fields (section/id/properties/id), optionally prefixed with external file reference", + "pattern": "^(?:(?:https?:\\/\\/)?[A-Za-z0-9._\\-\\/]+\\.ya?ml#)?\\/?[A-Za-z_][A-Za-z0-9_]*\\/[^\\s.#/\\\\@!%&^]+(?:\\/[A-Za-z_][A-Za-z0-9_]*\\/[^\\s.#/\\\\@!%&^]+)*$" + }, + "StableId": { + "type": "string", + "description": "Stable technical identifier for references. Must be unique within its containing array. Per RFC-0026a an id cannot contain '.', '#', '/', '\\', '@', '!', '%', '&' or '^', nor whitespace.", + "pattern": "^[^\\s.#/\\\\@!%&^]+$" + }, + "Port": { + "type": ["integer", "string"], + "description": "A network port: an integer, or a string, e.g. to hold a variable reference such as ${DB_PORT} resolved at runtime (RFC 0050)." + }, + "Server": { + "type": "object", + "description": "Data source details of where data is physically stored.", + "properties": { + "id": { + "$ref": "#/$defs/StableId" + }, + "server": { + "type": "string", + "description": "Identifier of the server." + }, + "type": { + "type": "string", + "description": "Type of the server.", + "enum": [ + "api", "athena", "azure", "bigquery", "btrieve", "clickhouse", "databricks", "denodo", "dremio", + "duckdb", "exasol", "fastobjects", "glue", "hana", "cloudsql", "db2", "hive", "iceberg", "impala", "informix", "ingres", "kafka", "kinesis", "local", + "mysql", "oracle", "poet", "postgresql", "postgres", "presto", "pubsub", + "redshift", "s3", "sftp", "snowflake", "sqlserver", "synapse", "teradata", "trino", "vectorwise", "versant", "vertica", "zen", "custom" + ] + }, + "description": { + "type": "string", + "description": "Description of the server." + }, + "environment": { + "type": "string", + "description": "Environment of the server.", + "examples": ["prod", "preprod", "dev", "uat"] + }, + "roles": { + "type": "array", + "description": "List of roles that have access to the server.", + "items": { + "$ref": "#/$defs/Role" + } + }, + "customProperties": { + "$ref": "#/$defs/CustomProperties" + } + }, + "allOf": [ + { + "if": { + "properties": { + "type": { + "const": "api" + } + }, + "required": ["type"] + }, + "then": { + "$ref": "#/$defs/ServerSource/ApiServer" + } + }, + { + "if": { + "properties": { + "type": { + "const": "athena" + } + }, + "required": ["type"] + }, + "then": { + "$ref": "#/$defs/ServerSource/AthenaServer" + } + }, + { + "if": { + "properties": { + "type": { + "const": "azure" + } + }, + "required": ["type"] + }, + "then": { + "$ref": "#/$defs/ServerSource/AzureServer" + } + }, + { + "if": { + "properties": { + "type": { + "const": "bigquery" + } + }, + "required": ["type"] + }, + "then": { + "$ref": "#/$defs/ServerSource/BigQueryServer" + } + }, + { + "if": { + "properties": { + "type": { + "const": "btrieve" + } + }, + "required": ["type"] + }, + "then": { + "$ref": "#/$defs/ServerSource/ZenServer" + } + }, + { + "if": { + "properties": { + "type": { + "const": "clickhouse" + } + }, + "required": ["type"] + }, + "then": { + "$ref": "#/$defs/ServerSource/ClickHouseServer" + } + }, + { + "if": { + "properties": { + "type": { + "const": "databricks" + } + }, + "required": ["type"] + }, + "then": { + "$ref": "#/$defs/ServerSource/DatabricksServer" + } + }, + { + "if": { + "properties": { + "type": { + "const": "denodo" + } + }, + "required": ["type"] + }, + "then": { + "$ref": "#/$defs/ServerSource/DenodoServer" + } + }, + { + "if": { + "properties": { + "type": { + "const": "dremio" + } + }, + "required": ["type"] + }, + "then": { + "$ref": "#/$defs/ServerSource/DremioServer" + } + }, + { + "if": { + "properties": { + "type": { + "const": "duckdb" + } + }, + "required": ["type"] + }, + "then": { + "$ref": "#/$defs/ServerSource/DuckdbServer" + } + }, + { + "if": { + "properties": { + "type": { + "const": "exasol" + } + }, + "required": ["type"] + }, + "then": { + "$ref": "#/$defs/ServerSource/ExasolServer" + } + }, + { + "if": { + "properties": { + "type": { + "const": "fastobjects" + } + }, + "required": ["type"] + }, + "then": { + "$ref": "#/$defs/ServerSource/PoetServer" + } + }, + { + "if": { + "properties": { + "type": { + "const": "glue" + } + }, + "required": ["type"] + }, + "then": { + "$ref": "#/$defs/ServerSource/GlueServer" + } + }, + { + "if": { + "properties": { + "type": { + "const": "cloudsql" + } + }, + "required": ["type"] + }, + "then": { + "$ref": "#/$defs/ServerSource/GoogleCloudSqlServer" + } + }, + { + "if": { + "properties": { + "type": { + "const": "db2" + } + }, + "required": ["type"] + }, + "then": { + "$ref": "#/$defs/ServerSource/IBMDB2Server" + } + }, + { + "if": { + "properties": { + "type": { + "const": "hive" + } + }, + "required": ["type"] + }, + "then": { + "$ref": "#/$defs/ServerSource/HiveServer" + } + }, + { + "if": { + "properties": { + "type": { + "const": "impala" + } + }, + "required": ["type"] + }, + "then": { + "$ref": "#/$defs/ServerSource/ImpalaServer" + } + }, + { + "if": { + "properties": { + "type": { + "const": "informix" + } + }, + "required": ["type"] + }, + "then": { + "$ref": "#/$defs/ServerSource/InformixServer" + } + }, + { + "if": { + "properties": { + "type": { + "const": "ingres" + } + }, + "required": ["type"] + }, + "then": { + "$ref": "#/$defs/ServerSource/IngresServer" + } + }, + { + "if": { + "properties": { + "type": { + "const": "zen" + } + }, + "required": ["type"] + }, + "then": { + "$ref": "#/$defs/ServerSource/ZenServer" + } + }, + { + "if": { + "properties": { + "type": { + "const": "custom" + } + }, + "required": ["type"] + }, + "then": { + "$ref": "#/$defs/ServerSource/CustomServer" + } + }, + { + "if": { + "properties": { + "type": { + "const": "kafka" + } + }, + "required": ["type"] + }, + "then": { + "$ref": "#/$defs/ServerSource/KafkaServer" + } + }, + { + "if": { + "properties": { + "type": { + "const": "kinesis" + } + }, + "required": ["type"] + }, + "then": { + "$ref": "#/$defs/ServerSource/KinesisServer" + } + }, + { + "if": { + "properties": { + "type": { + "const": "local" + } + }, + "required": ["type"] + }, + "then": { + "$ref": "#/$defs/ServerSource/LocalServer" + } + }, + { + "if": { + "properties": { + "type": { + "const": "mysql" + } + }, + "required": ["type"] + }, + "then": { + "$ref": "#/$defs/ServerSource/MySqlServer" + } + }, + { + "if": { + "properties": { + "type": { + "const": "oracle" + } + }, + "required": ["type"] + }, + "then": { + "$ref": "#/$defs/ServerSource/OracleServer" + } + }, + { + "if": { + "properties": { + "type": { + "const": "poet" + } + }, + "required": ["type"] + }, + "then": { + "$ref": "#/$defs/ServerSource/PoetServer" + } + }, + { + "if": { + "properties": { + "type": { + "const": "hana" + } + }, + "required": ["type"] + }, + "then": { + "$ref": "#/$defs/ServerSource/HanaServer" + } + }, + { + "if": { + "properties": { + "type": { + "const": "iceberg" + } + }, + "required": ["type"] + }, + "then": { + "$ref": "#/$defs/ServerSource/IcebergServer" + } + }, + { + "if": { + "properties": { + "type": { + "const": "postgresql" + } + }, + "required": ["type"] + }, + "then": { + "$ref": "#/$defs/ServerSource/PostgresServer" + } + }, + { + "if": { + "properties": { + "type": { + "const": "postgres" + } + }, + "required": ["type"] + }, + "then": { + "$ref": "#/$defs/ServerSource/PostgresServer" + } + }, + { + "if": { + "properties": { + "type": { + "const": "presto" + } + }, + "required": ["type"] + }, + "then": { + "$ref": "#/$defs/ServerSource/PrestoServer" + } + }, + { + "if": { + "properties": { + "type": { + "const": "pubsub" + } + }, + "required": ["type"] + }, + "then": { + "$ref": "#/$defs/ServerSource/PubSubServer" + } + }, + { + "if": { + "properties": { + "type": { + "const": "redshift" + } + }, + "required": ["type"] + }, + "then": { + "$ref": "#/$defs/ServerSource/RedshiftServer" + } + }, + { + "if": { + "properties": { + "type": { + "const": "s3" + } + }, + "required": ["type"] + }, + "then": { + "$ref": "#/$defs/ServerSource/S3Server" + } + }, + { + "if": { + "properties": { + "type": { + "const": "sftp" + } + }, + "required": ["type"] + }, + "then": { + "$ref": "#/$defs/ServerSource/SftpServer" + } + }, + { + "if": { + "properties": { + "type": { + "const": "snowflake" + } + }, + "required": ["type"] + }, + "then": { + "$ref": "#/$defs/ServerSource/SnowflakeServer" + } + }, + { + "if": { + "properties": { + "type": { + "const": "sqlserver" + } + }, + "required": ["type"] + }, + "then": { + "$ref": "#/$defs/ServerSource/SqlserverServer" + } + }, + { + "if": { + "properties": { + "type": { + "const": "synapse" + } + }, + "required": ["type"] + }, + "then": { + "$ref": "#/$defs/ServerSource/SynapseServer" + } + }, + { + "if": { + "properties": { + "type": { + "const": "teradata" + } + }, + "required": ["type"] + }, + "then": { + "$ref": "#/$defs/ServerSource/TeradataServer" + } + }, + { + "if": { + "properties": { + "type": { + "const": "trino" + } + }, + "required": ["type"] + }, + "then": { + "$ref": "#/$defs/ServerSource/TrinoServer" + } + }, + { + "if": { + "properties": { + "type": { + "const": "vectorwise" + } + }, + "required": ["type"] + }, + "then": { + "$ref": "#/$defs/ServerSource/VectorwiseServer" + } + }, + { + "if": { + "properties": { + "type": { + "const": "versant" + } + }, + "required": ["type"] + }, + "then": { + "$ref": "#/$defs/ServerSource/VersantServer" + } + }, + { + "if": { + "properties": { + "type": { + "const": "vertica" + } + }, + "required": ["type"] + }, + "then": { + "$ref": "#/$defs/ServerSource/VerticaServer" + } + } + ], + "required": ["server", "type"], + "unevaluatedProperties": false + }, + "ServerSource": { + "ApiServer": { + "type": "object", + "title": "AthenaServer", + "properties": { + "location": { + "type": "string", + "format": "uri", + "description": "The url to the API.", + "examples": [ + "https://api.example.com/v1" + ] + } + }, + "required": [ + "location" + ] + }, + "AthenaServer": { + "type": "object", + "title": "AthenaServer", + "properties": { + "stagingDir": { + "type": "string", + "format": "uri", + "description": "Amazon Athena automatically stores query results and metadata information for each query that runs in a query result location that you can specify in Amazon S3.", + "examples": [ + "s3://my_storage_account_name/my_container/path" + ] + }, + "schema": { + "type": "string", + "description": "Identify the schema in the data source in which your tables exist." + }, + "catalog": { + "type": "string", + "description": "Identify the name of the Data Source, also referred to as a Catalog.", + "default": "awsdatacatalog" + }, + "regionName": { + "type": "string", + "description": "The region your AWS account uses.", + "examples": ["eu-west-1"] + }, + "workgroup": { + "type": "string", + "description": "The Athena workgroup to use. Workgroups can enforce query result location and other client-side settings via the 'Override client-side settings' option.", + "examples": ["primary"] + } + }, + "required": [ + "schema" + ] + }, + "AzureServer": { + "type": "object", + "title": "AzureServer", + "properties": { + "location": { + "type": "string", + "format": "uri", + "description": "Fully qualified path to Azure Blob Storage or Azure Data Lake Storage (ADLS), supports globs.", + "examples": [ + "az://my_storage_account_name.blob.core.windows.net/my_container/path/*.parquet", + "abfss://my_storage_account_name.dfs.core.windows.net/my_container_name/path/*.parquet" + ] + }, + "format": { + "type": "string", + "examples": [ + "parquet", + "delta", + "json", + "csv" + ], + "description": "File format." + }, + "encoding": { + "type": "string", + "description": "The expected character encoding for data payloads exposed through this server, for example UTF-8, ISO-8859-1, ASCII or UTF-16.", + "examples": [ + "UTF-8", + "ISO-8859-1", + "ASCII", + "UTF-16" + ] + }, + "delimiter": { + "type": "string", + "examples": [ + "new_line", + "array" + ], + "description": "Only for format = json. How multiple json documents are delimited within one file" + } + }, + "required": [ + "location", + "format" + ] + }, + "BigQueryServer": { + "type": "object", + "title": "BigQueryServer", + "properties": { + "project": { + "type": "string", + "description": "The GCP project name." + }, + "dataset": { + "type": "string", + "description": "The GCP dataset name." + } + }, + "required": [ + "project", + "dataset" + ] + }, + "ClickHouseServer": { + "type": "object", + "title": "ClickHouseServer", + "properties": { + "host": { + "type": "string", + "description": "The host of the ClickHouse server." + }, + "port": { + "$ref": "#/$defs/Port", + "description": "The port to the ClickHouse server." + }, + "database": { + "type": "string", + "description": "The name of the database." + } + }, + "required": [ + "host", + "port", + "database" + ] + }, + "DatabricksServer": { + "type": "object", + "title": "DatabricksServer", + "properties": { + "host": { + "type": "string", + "description": "The Databricks host", + "examples": [ + "dbc-abcdefgh-1234.cloud.databricks.com" + ] + }, + "catalog": { + "type": "string", + "description": "The name of the Hive or Unity catalog" + }, + "schema": { + "type": "string", + "description": "The schema name in the catalog" + } + }, + "required": [ + "catalog", + "schema" + ] + }, + "DenodoServer": { + "type": "object", + "title": "DenodoServer", + "properties": { + "host": { + "type": "string", + "description": "The host of the Denodo server." + }, + "port": { + "$ref": "#/$defs/Port", + "description": "The port of the Denodo server." + }, + "database": { + "type": "string", + "description": "The name of the database." + } + }, + "required": [ + "host", + "port" + ] + }, + "DremioServer": { + "type": "object", + "title": "DremioServer", + "properties": { + "host": { + "type": "string", + "description": "The host of the Dremio server." + }, + "port": { + "$ref": "#/$defs/Port", + "description": "The port of the Dremio server." + }, + "schema": { + "type": "string", + "description": "The name of the schema." + } + }, + "required": [ + "host", + "port" + ] + }, + "DuckdbServer": { + "type": "object", + "title": "DuckdbServer", + "properties": { + "database": { + "type": "string", + "description": "Path to duckdb database file." + }, + "schema": { + "type": "string", + "description": "The name of the schema." + } + }, + "required": [ + "database" + ] + }, + "ExasolServer": { + "type": "object", + "title": "ExasolServer", + "properties": { + "host": { + "type": "string", + "description": "Host of the Exasol server. May be a cluster connection range.", + "examples": [ + "exasol.acme.com", + "n11..14.acme.com" + ] + }, + "port": { + "$ref": "#/$defs/Port", + "description": "Port of the Exasol server.", + "default": 8563, + "examples": [ + 8563 + ] + }, + "schema": { + "type": "string", + "description": "Name of the schema." + } + }, + "required": [ + "host" + ] + }, + "GlueServer": { + "type": "object", + "title": "GlueServer", + "properties": { + "account": { + "type": "string", + "description": "The AWS Glue account", + "examples": [ + "1234-5678-9012" + ] + }, + "database": { + "type": "string", + "description": "The AWS Glue database name", + "examples": [ + "my_database" + ] + }, + "location": { + "type": "string", + "format": "uri", + "description": "The AWS S3 path. Must be in the form of a URL.", + "examples": [ + "s3://datacontract-example-orders-latest/data/{model}" + ] + }, + "format": { + "type": "string", + "description": "The format of the files", + "examples": [ + "parquet", + "csv", + "json", + "delta" + ] + }, + "encoding": { + "type": "string", + "description": "The expected character encoding for data payloads exposed through this server, for example UTF-8, ISO-8859-1, ASCII or UTF-16.", + "examples": [ + "UTF-8", + "ISO-8859-1", + "ASCII", + "UTF-16" + ] + } + }, + "required": [ + "account", + "database" + ] + }, + "GoogleCloudSqlServer": { + "type": "object", + "title": "GoogleCloudSqlServer", + "properties": { + "host": { + "type": "string", + "description": "The host of the Google Cloud Sql server." + }, + "port": { + "$ref": "#/$defs/Port", + "description": "The port of the Google Cloud Sql server." + }, + "database": { + "type": "string", + "description": "The name of the database." + }, + "schema": { + "type": "string", + "description": "The name of the schema." + } + }, + "required": [ + "host", + "port", + "database", + "schema" + ] + }, + "IBMDB2Server": { + "type": "object", + "title": "IBMDB2Server", + "properties": { + "host": { + "type": "string", + "description": "The host of the IBM DB2 server." + }, + "port": { + "$ref": "#/$defs/Port", + "description": "The port of the IBM DB2 server." + }, + "database": { + "type": "string", + "description": "The name of the database." + }, + "schema": { + "type": "string", + "description": "The name of the schema." + } + }, + "required": [ + "host", + "port", + "database" + ] + }, + "HiveServer": { + "type": "object", + "title": "HiveServer", + "properties": { + "host": { + "type": "string", + "description": "The host to the Hive server. " + }, + "port": { + "$ref": "#/$defs/Port", + "description": "The port to the Hive server. Defaults to 10000." + }, + "database": { + "type": "string", + "description": "The name of the Hive database." + } + }, + "required": [ + "host", + "database" + ] + }, + "ImpalaServer": { + "type": "object", + "title": "ImpalaServer", + "properties": { + "host": { + "type": "string", + "description": "The host to the Impala server." + }, + "port": { + "$ref": "#/$defs/Port", + "description": "The port to the Impala server. Defaults to 21050." + }, + "database": { + "type": "string", + "description": "The name of the Impala database." + } + }, + "required": [ + "host", + "database" + ] + }, + "InformixServer": { + "type": "object", + "title": "InformixServer", + "properties": { + "host": { + "type": "string", + "description": "The host to the Informix server. " + }, + "port": { + "$ref": "#/$defs/Port", + "description": "The port to the Informix server. Defaults to 9088." + }, + "database": { + "type": "string", + "description": "The name of the database." + } + }, + "required": [ + "host", + "database" + ] + }, + "IngresServer": { + "type": "object", + "title": "IngresServer", + "properties": { + "database": { + "type": "string", + "description": "Database name to connect to on the Ingres instance.", + "examples": [ + "sales" + ] + }, + "host": { + "type": "string", + "description": "Hostname or IP address of the Ingres server.", + "examples": [ + "ingres.acme.com" + ] + }, + "port": { + "$ref": "#/$defs/Port", + "description": "Connection port for the Ingres Data Access Server (DAS) / SQL connections.", + "default": 21064, + "examples": [ + 21064 + ] + } + }, + "required": [ + "database", + "host" + ] + }, + "ZenServer": { + "type": "object", + "title": "ZenServer", + "properties": { + "host": { + "type": "string", + "description": "Hostname or IP address of the Zen server." + }, + "port": { + "$ref": "#/$defs/Port", + "description": "Zen server SQL connections port. Defaults to 1583." + }, + "database": { + "type": "string", + "description": "Database name to connect to on the Zen server." + } + }, + "required": ["host", "database"] + }, + "CustomServer": { + "type": "object", + "title": "CustomServer", + "properties": { + "account": { + "type": "string", + "description": "Account used by the server." + }, + "catalog": { + "type": "string", + "description": "Name of the catalog." + }, + "database": { + "type": "string", + "description": "Name of the database." + }, + "dataset": { + "type": "string", + "description": "Name of the dataset." + }, + "delimiter": { + "type": "string", + "description": "Delimiter." + }, + "endpointUrl": { + "type": "string", + "description": "Server endpoint.", + "format": "uri" + }, + "format": { + "type": "string", + "description": "File format." + }, + "encoding": { + "type": "string", + "description": "The expected character encoding for data payloads exposed through this server, for example UTF-8, ISO-8859-1, ASCII or UTF-16.", + "examples": [ + "UTF-8", + "ISO-8859-1", + "ASCII", + "UTF-16" + ] + }, + "host": { + "type": "string", + "description": "Host name or IP address." + }, + "location": { + "type": "string", + "description": "A URL to a location.", + "format": "uri" + }, + "path": { + "type": "string", + "description": "Relative or absolute path to the data file(s)." + }, + "port": { + "$ref": "#/$defs/Port", + "description": "Port to the server. No default value is assumed for custom servers." + }, + "project": { + "type": "string", + "description": "Project name." + }, + "region": { + "type": "string", + "description": "Cloud region." + }, + "regionName": { + "type": "string", + "description": "Region name." + }, + "schema": { + "type": "string", + "description": "Name of the schema." + }, + "serviceName": { + "type": "string", + "description": "Name of the service." + }, + "stagingDir": { + "type": "string", + "description": "Staging directory." + }, + "warehouse": { + "type": "string", + "description": "Name of the cluster or warehouse." + }, + "stream": { + "type": "string", + "description": "Name of the data stream." + } + } + }, + "KafkaServer": { + "type": "object", + "title": "KafkaServer", + "description": "Kafka Server", + "properties": { + "host": { + "type": "string", + "description": "The bootstrap server of the kafka cluster." + }, + "format": { + "type": "string", + "description": "The format of the messages.", + "examples": ["json", "avro", "protobuf", "xml"], + "default": "json" + }, + "encoding": { + "type": "string", + "description": "The expected character encoding for data payloads exposed through this server, for example UTF-8, ISO-8859-1, ASCII or UTF-16.", + "examples": [ + "UTF-8", + "ISO-8859-1", + "ASCII", + "UTF-16" + ] + } + }, + "required": [ + "host" + ] + }, + "KinesisServer": { + "type": "object", + "title": "KinesisDataStreamsServer", + "description": "Kinesis Data Streams Server", + "properties": { + "region": { + "type": "string", + "description": "AWS region.", + "examples": [ + "eu-west-1" + ] + }, + "format": { + "type": "string", + "description": "The format of the record", + "examples": [ + "json", + "avro", + "protobuf" + ] + }, + "encoding": { + "type": "string", + "description": "The expected character encoding for data payloads exposed through this server, for example UTF-8, ISO-8859-1, ASCII or UTF-16.", + "examples": [ + "UTF-8", + "ISO-8859-1", + "ASCII", + "UTF-16" + ] + } + } + }, + "LocalServer": { + "type": "object", + "title": "LocalServer", + "properties": { + "path": { + "type": "string", + "description": "The relative or absolute path to the data file(s).", + "examples": [ + "./folder/data.parquet", + "./folder/*.parquet" + ] + }, + "format": { + "type": "string", + "description": "The format of the file(s)", + "examples": [ + "json", + "parquet", + "delta", + "csv" + ] + }, + "encoding": { + "type": "string", + "description": "The expected character encoding for data payloads exposed through this server, for example UTF-8, ISO-8859-1, ASCII or UTF-16.", + "examples": [ + "UTF-8", + "ISO-8859-1", + "ASCII", + "UTF-16" + ] + } + }, + "required": [ + "path", + "format" + ] + }, + "MySqlServer": { + "type": "object", + "title": "MySqlServer", + "properties": { + "host": { + "type": "string", + "description": "The host of the MySql server." + }, + "port": { + "$ref": "#/$defs/Port", + "description": "The port of the MySql server." + }, + "database": { + "type": "string", + "description": "The name of the database." + } + }, + "required": [ + "host", + "port", + "database" + ] + }, + "OracleServer": { + "type": "object", + "title": "OracleServer", + "properties": { + "host": { + "type": "string", + "description": "The host to the oracle server", + "examples": [ + "localhost" + ] + }, + "port": { + "$ref": "#/$defs/Port", + "description": "The port to the oracle server.", + "examples": [ + 1523 + ] + }, + "serviceName": { + "type": "string", + "description": "The name of the service.", + "examples": [ + "service" + ] + } + }, + "required": [ + "host", + "port", + "serviceName" + ] + }, + "PoetServer": { + "type": "object", + "title": "PoetServer", + "properties": { + "database": { + "type": "string", + "description": "Database name to connect to on the FastObjects instance.", + "examples": [ + "parts_catalog" + ] + }, + "host": { + "type": "string", + "description": "Hostname or IP address of the FastObjects server. The literal LOCAL selects the in-process embedded engine.", + "default": "LOCAL", + "examples": [ + "LOCAL", + "fastobjects02.acme.com" + ] + }, + "port": { + "$ref": "#/$defs/Port", + "description": "Connection port for FastObjects connections.", + "default": 6001, + "examples": [ + 6001 + ] + } + }, + "required": [ + "database" + ] + }, + "PostgresServer": { + "type": "object", + "title": "PostgresServer", + "properties": { + "host": { + "type": "string", + "description": "The host to the Postgres server" + }, + "port": { + "$ref": "#/$defs/Port", + "description": "The port to the Postgres server." + }, + "database": { + "type": "string", + "description": "The name of the database." + }, + "schema": { + "type": "string", + "description": "The name of the schema in the database." + } + }, + "required": [ + "host", + "port", + "database", + "schema" + ] + }, + "HanaServer": { + "type": "object", + "title": "HanaServer", + "properties": { + "host": { + "type": "string", + "description": "Host of the HANA server." + }, + "port": { + "$ref": "#/$defs/Port", + "description": "Port of the HANA server." + }, + "database": { + "type": "string", + "description": "Name of the database (tenant)." + }, + "schema": { + "type": "string", + "description": "Name of the schema." + } + }, + "required": [ + "host" + ] + }, + "IcebergServer": { + "type": "object", + "title": "IcebergServer", + "properties": { + "catalog": { + "type": "string", + "description": "Catalog name as registered in the query engine or catalog service.", + "examples": ["my_catalog"] + }, + "catalogUrl": { + "type": "string", + "format": "uri", + "description": "URL of the Iceberg compatible REST catalog service (Polaris, S3 Tables, Nessie, Unity Catalog, Glue, etc.)", + "examples": ["https://my-catalog-service.com"] + }, + "namespace": { + "type": "string", + "description": "Dot-separated namespace/schema path within the catalog (e.g. db.schema or just db)." + }, + "warehouse": { + "type": "string", + "description": "Logical catalog or base location for the catalog to use.", + "examples": ["s3://my-bucket/warehouse/sales/", "production", "development"] + } + }, + "required": [ + "catalog", + "catalogUrl" + ] + }, + "PrestoServer": { + "type": "object", + "title": "PrestoServer", + "properties": { + "host": { + "type": "string", + "description": "The host to the Presto server", + "examples": [ + "localhost:8080" + ] + }, + "catalog": { + "type": "string", + "description": "The name of the catalog.", + "examples": [ + "postgres" + ] + }, + "schema": { + "type": "string", + "description": "The name of the schema.", + "examples": [ + "public" + ] + } + }, + "required": [ + "host" + ] + }, + "PubSubServer": { + "type": "object", + "title": "PubSubServer", + "properties": { + "project": { + "type": "string", + "description": "The GCP project name." + } + }, + "required": [ + "project" + ] + }, + "RedshiftServer": { + "type": "object", + "title": "RedshiftServer", + "properties": { + "host": { + "type": "string", + "description": "An optional string describing the server." + }, + "database": { + "type": "string", + "description": "The name of the database." + }, + "schema": { + "type": "string", + "description": "The name of the schema." + }, + "region": { + "type": "string", + "description": "AWS region of Redshift server.", + "examples": ["us-east-1"] + }, + "account": { + "type": "string", + "description": "The account used by the server." + } + }, + "required": [ + "database", + "schema" + ] + }, + "S3Server": { + "type": "object", + "title": "S3Server", + "properties": { + "location": { + "type": "string", + "format": "uri", + "description": "S3 URL, starting with `s3://`", + "examples": [ + "s3://datacontract-example-orders-latest/data/{model}/*.json" + ] + }, + "endpointUrl": { + "type": "string", + "format": "uri", + "description": "The server endpoint for S3-compatible servers.", + "examples": ["https://minio.example.com"] + }, + "format": { + "type": "string", + "examples": [ + "parquet", + "delta", + "json", + "csv" + ], + "description": "File format." + }, + "encoding": { + "type": "string", + "description": "The expected character encoding for data payloads exposed through this server, for example UTF-8, ISO-8859-1, ASCII or UTF-16.", + "examples": [ + "UTF-8", + "ISO-8859-1", + "ASCII", + "UTF-16" + ] + }, + "delimiter": { + "type": "string", + "examples": [ + "new_line", + "array" + ], + "description": "Only for format = json. How multiple json documents are delimited within one file" + } + }, + "required": [ + "location" + ] + }, + "SftpServer": { + "type": "object", + "title": "SftpServer", + "properties": { + "location": { + "type": "string", + "format": "uri", + "pattern": "^sftp://.*", + "description": "SFTP URL, starting with `sftp://`", + "examples": [ + "sftp://123.123.12.123/{model}/*.json" + ] + }, + "format": { + "type": "string", + "examples": [ + "parquet", + "delta", + "json", + "csv" + ], + "description": "File format." + }, + "encoding": { + "type": "string", + "description": "The expected character encoding for data payloads exposed through this server, for example UTF-8, ISO-8859-1, ASCII or UTF-16.", + "examples": [ + "UTF-8", + "ISO-8859-1", + "ASCII", + "UTF-16" + ] + }, + "delimiter": { + "type": "string", + "examples": [ + "new_line", + "array" + ], + "description": "Only for format = json. How multiple json documents are delimited within one file" + } + }, + "required": [ + "location" + ] + }, + "SnowflakeServer": { + "type": "object", + "title": "SnowflakeServer", + "properties": { + "host": { + "type": "string", + "description": "The host to the Snowflake server" + }, + "port": { + "$ref": "#/$defs/Port", + "description": "The port to the Snowflake server." + }, + "account": { + "type": "string", + "description": "The Snowflake account used by the server." + }, + "database": { + "type": "string", + "description": "The name of the database." + }, + "schema": { + "type": "string", + "description": "The name of the schema." + }, + "warehouse": { + "type": "string", + "description": "The name of the cluster of resources that is a Snowflake virtual warehouse." + } + }, + "required": [ + "account", + "database", + "schema" + ] + }, + "SqlserverServer": { + "type": "object", + "title": "SqlserverServer", + "properties": { + "host": { + "type": "string", + "description": "The host to the database server", + "examples": [ + "localhost" + ] + }, + "port": { + "$ref": "#/$defs/Port", + "description": "The port to the database server.", + "default": 1433, + "examples": [ + 1433 + ] + }, + "database": { + "type": "string", + "description": "The name of the database.", + "examples": [ + "database" + ] + }, + "schema": { + "type": "string", + "description": "The name of the schema in the database.", + "examples": [ + "dbo" + ] + } + }, + "required": [ + "host", + "database", + "schema" + ] + }, + "SynapseServer": { + "type": "object", + "title": "SynapseServer", + "properties": { + "host": { + "type": "string", + "description": "The host of the Synapse server." + }, + "port": { + "$ref": "#/$defs/Port", + "description": "The port of the Synapse server." + }, + "database": { + "type": "string", + "description": "The name of the database." + } + }, + "required": [ + "host", + "port", + "database" + ] + }, + "TeradataServer": { + "type": "object", + "title": "TeradataServer", + "properties": { + "host": { + "type": "string", + "description": "Host of the Teradata server.", + "examples": [ + "teradata.acme.com" + ] + }, + "port": { + "$ref": "#/$defs/Port", + "description": "Port of the Teradata server.", + "default": 1025, + "examples": [ + 1025 + ] + }, + "database": { + "type": "string", + "description": "Name of the database." + } + }, + "required": [ + "host" + ] + }, + "TrinoServer": { + "type": "object", + "title": "TrinoServer", + "properties": { + "host": { + "type": "string", + "description": "The Trino host URL.", + "examples": [ + "localhost" + ] + }, + "port": { + "$ref": "#/$defs/Port", + "description": "The Trino port." + }, + "catalog": { + "type": "string", + "description": "The name of the catalog.", + "examples": [ + "hive" + ] + }, + "schema": { + "type": "string", + "description": "The name of the schema in the database.", + "examples": [ + "my_schema" + ] + } + }, + "required": [ + "host", + "port", + "catalog", + "schema" + ] + }, + "VectorwiseServer": { + "type": "object", + "title": "VectorwiseServer", + "properties": { + "database": { + "type": "string", + "description": "Database name to connect to on the Analytics Engine server.", + "examples": [ + "warehouse" + ] + }, + "host": { + "type": "string", + "description": "Hostname or IP address of the Analytics Engine server.", + "examples": [ + "analytics.acme.com" + ] + }, + "port": { + "$ref": "#/$defs/Port", + "description": "Connection port for the Data Access Server (DAS) / SQL connections.", + "default": 21064, + "examples": [ + 21064 + ] + } + }, + "required": [ + "database", + "host" + ] + }, + "VersantServer": { + "type": "object", + "title": "VersantServer", + "properties": { + "database": { + "type": "string", + "description": "Database name to connect to on the NoSQL Database instance.", + "examples": [ + "policies" + ] + }, + "host": { + "type": "string", + "description": "Hostname or IP address of the NoSQL Database server.", + "default": "localhost", + "examples": [ + "nosql.acme.com" + ] + }, + "port": { + "$ref": "#/$defs/Port", + "description": "Connection port for Actian NoSQL Database connections.", + "default": 5019, + "examples": [ + 5019 + ] + } + }, + "required": [ + "database" + ] + }, + "VerticaServer": { + "type": "object", + "title": "VerticaServer", + "properties": { + "host": { + "type": "string", + "description": "The host of the Vertica server." + }, + "port": { + "$ref": "#/$defs/Port", + "description": "The port of the Vertica server." + }, + "database": { + "type": "string", + "description": "The name of the database." + }, + "schema": { + "type": "string", + "description": "The name of the schema." + } + }, + "required": [ + "host", + "port", + "database", + "schema" + ] + } + }, + "SchemaElement": { + "type": "object", + "properties": { + "id": { + "$ref": "#/$defs/StableId" + }, + "name": { + "type": "string", + "description": "Name of the element." + }, + "physicalType": { + "type": "string", + "description": "The physical element data type in the data source.", + "examples": ["table", "view", "topic", "file"] + }, + "description": { + "type": "string", + "description": "Description of the element." + }, + "businessName": { + "type": "string", + "description": "The business name of the element." + }, + "deprecated": { + "type": "boolean", + "description": "Indicates this element is deprecated and should not be used in new implementations. Defaults to false.", + "default": false + }, + "synonyms": { + "$ref": "#/$defs/Synonyms" + }, + "authoritativeDefinitions": { + "$ref": "#/$defs/AuthoritativeDefinitions" + }, + "tags": { + "$ref": "#/$defs/Tags" + }, + "customProperties": { + "$ref": "#/$defs/CustomProperties" + } + } + }, + "SchemaObject": { + "type": "object", + "properties": { + "logicalType": { + "type": "string", + "description": "The logical element data type.", + "enum": ["object"] + }, + "physicalName": { + "type": "string", + "description": "Physical name.", + "examples": ["table_1_2_0"] + }, + "dataGranularityDescription": { + "type": "string", + "description": "Granular level of the data in the object.", + "examples": ["Aggregation by country"] + }, + "properties": { + "type": "array", + "description": "A list of properties for the object.", + "items": { + "$ref": "#/$defs/SchemaProperty" + } + }, + "relationships": { + "type": "array", + "description": "A list of relationships to other properties. Each relationship must have 'from', 'to' and optionally 'type' field.", + "items": { + "$ref": "#/$defs/RelationshipSchemaLevel" + } + }, + "quality": { + "$ref": "#/$defs/DataQualityChecks" + }, + "context": { + "$ref": "#/$defs/Context" + } + }, + "allOf": [ + { + "$ref": "#/$defs/SchemaElement" + } + ], + "required": ["name"], + "unevaluatedProperties": false + }, + "SchemaBaseProperty": { + "type": "object", + "properties": { + "primaryKey": { + "type": "boolean", + "description": "Boolean value specifying whether the element is primary or not. Default is false." + }, + "primaryKeyPosition": { + "type": "integer", + "default": -1, + "description": "If element is a primary key, the position of the primary key element. Starts from 1. Example of `account_id, name` being primary key columns, `account_id` has primaryKeyPosition 1 and `name` primaryKeyPosition 2. Default to -1." + }, + "logicalType": { + "type": "string", + "description": "The logical element data type.", + "enum": ["string", "date", "timestamp", "time", "number", "integer", "object", "array", "boolean", "map", "vector"] + }, + "logicalTypeOptions": { + "type": "object", + "description": "Additional optional metadata to describe the logical type." + }, + "physicalType": { + "type": "string", + "description": "The physical element data type in the data source. For example, VARCHAR(2), DOUBLE, INT." + }, + "physicalName": { + "type": "string", + "description": "Physical name.", + "examples": ["col_str_a"] + }, + "required": { + "type": "boolean", + "default": false, + "description": "Indicates if the element may contain Null values; possible values are true and false. Default is false." + }, + "unique": { + "type": "boolean", + "default": false, + "description": "Indicates if the element contains unique values; possible values are true and false. Default is false." + }, + "semanticType": { + "type": "string", + "enum": ["column", "measure", "dimension"], + "default": "column", + "description": "The semantic role the property plays in the data model. `column` (the default) is a physical column in the underlying data store; `measure` is an aggregated value (e.g., `SUM(revenue)`) whose aggregation expression is held in `transformLogic`; `dimension` is a categorical attribute used for grouping and filtering. See RFC 0034." + }, + "partitioned": { + "type": "boolean", + "default": false, + "description": "Indicates if the element is partitioned; possible values are true and false." + }, + "partitionKeyPosition": { + "type": "integer", + "default": -1, + "description": "If element is used for partitioning, the position of the partition element. Starts from 1. Example of `country, year` being partition columns, `country` has partitionKeyPosition 1 and `year` partitionKeyPosition 2. Default to -1." + }, + "classification": { + "type": "string", + "description": "Can be anything, like confidential, restricted, and public to more advanced categorization. Some companies like PayPal, use data classification indicating the class of data in the element; expected values are 1, 2, 3, 4, or 5.", + "examples": ["confidential", "restricted", "public"] + }, + "encryptedName": { + "type": "string", + "description": "The element name within the dataset that contains the encrypted element value. For example, unencrypted element `email_address` might have an encryptedName of `email_address_encrypt`." + }, + "transformSourceObjects": { + "type": "array", + "description": "List of objects in the data source used in the transformation.", + "items": { + "type": "string" + } + }, + "transformLogic": { + "type": "string", + "description": "Logic used in the element transformation." + }, + "transformDescription": { + "type": "string", + "description": "Describes the transform logic in very simple terms." + }, + "examples": { + "type": "array", + "description": "List of sample element values.", + "items": { + "$ref": "#/$defs/AnyType" + } + }, + "enum": { + "type": "array", + "description": "Enumeration of allowed values for this property. Each entry is an object with at least a `value`. See RFC 0033.", + "minItems": 1, + "uniqueItems": true, + "items": { + "$ref": "#/$defs/EnumValue" + } + }, + "criticalDataElement": { + "type": "boolean", + "default": false, + "description": "True or false indicator; If element is considered a critical data element (CDE) then true else false." + }, + "relationships": { + "type": "array", + "description": "A list of relationships to other properties. When defined at property level, the 'from' field is implicit and should not be specified.", + "items": { + "$ref": "#/$defs/RelationshipPropertyLevel" + } + }, + "quality": { + "$ref": "#/$defs/DataQualityChecks" + } + }, + "allOf": [ + { + "$ref": "#/$defs/SchemaElement" + }, + { + "if": { + "properties": { + "logicalType": { + "const": "string" + } + } + }, + "then": { + "properties": { + "logicalTypeOptions": { + "type": "object", + "additionalProperties": false, + "properties": { + "minLength": { + "type": "integer", + "minimum": 0, + "description": "Minimum length of the string." + }, + "maxLength": { + "type": "integer", + "minimum": 0, + "description": "Maximum length of the string." + }, + "pattern": { + "type": "string", + "description": "Regular expression pattern to define valid value. Follows regular expression syntax from ECMA-262 (https://262.ecma-international.org/5.1/#sec-15.10.1)." + }, + "format": { + "type": "string", + "examples": ["password", "byte", "binary", "email", "uuid", "uri", "hostname", "ipv4", "ipv6"], + "description": "Provides extra context about what format the string follows." + } + } + } + } + } + }, + { + "if": { + "properties": { + "logicalType": { + "const": "date" + } + } + }, + "then": { + "properties": { + "logicalTypeOptions": { + "type": "object", + "additionalProperties": false, + "properties": { + "format": { + "type": "string", + "examples": ["yyyy-MM-dd", "yyyy-MM-dd HH:mm:ss", "HH:mm:ss"], + "description": "Format of the date. Follows the format as prescribed by [JDK DateTimeFormatter](https://docs.oracle.com/javase/8/docs/api/java/time/format/DateTimeFormatter.html). For example, format 'yyyy-MM-dd'." + }, + "exclusiveMaximum": { + "type": "string", + "description": "All values must be strictly less than this value (values < exclusiveMaximum)." + }, + "maximum": { + "type": "string", + "description": "All date values are less than or equal to this value (values <= maximum)." + }, + "exclusiveMinimum": { + "type": "string", + "description": "All values must be strictly greater than this value (values > exclusiveMinimum)." + }, + "minimum": { + "type": "string", + "description": "All date values are greater than or equal to this value (values >= minimum)." + } + } + } + } + } + }, + { + "if": { + "anyOf": [ + { + "properties": { + "logicalType": { + "const": "timestamp" + } + } + }, + { + "properties": { + "logicalType": { + "const": "time" + } + } + } + ] + }, + "then": { + "properties": { + "logicalTypeOptions": { + "type": "object", + "additionalProperties": false, + "properties": { + "format": { + "type": "string", + "examples": ["yyyy-MM-dd", "yyyy-MM-dd HH:mm:ss", "HH:mm:ss"], + "description": "Format of the date. Follows the format as prescribed by [JDK DateTimeFormatter](https://docs.oracle.com/javase/8/docs/api/java/time/format/DateTimeFormatter.html). For example, format 'yyyy-MM-dd'." + }, + "exclusiveMaximum": { + "type": "string", + "description": "All values must be strictly less than this value (values < exclusiveMaximum)." + }, + "maximum": { + "type": "string", + "description": "All date values are less than or equal to this value (values <= maximum)." + }, + "exclusiveMinimum": { + "type": "string", + "description": "All values must be strictly greater than this value (values > exclusiveMinimum)." + }, + "minimum": { + "type": "string", + "description": "All date values are greater than or equal to this value (values >= minimum)." + }, + "timezone": { + "type": "boolean", + "description": "Whether the timestamp defines the timezone or not. If true, timezone information is included in the timestamp." + }, + "defaultTimezone": { + "type": "string", + "description": "The default timezone of the timestamp. If timezone is not defined, the default timezone UTC is used.", + "default": "Etc/UTC" + } + } + } + } + } + }, + { + "if": { + "anyOf": [ + { + "properties": { + "logicalType": { + "const": "integer" + } + } + } + ] + }, + "then": { + "properties": { + "logicalTypeOptions": { + "type": "object", + "additionalProperties": false, + "properties": { + "multipleOf": { + "type": "number", + "exclusiveMinimum": 0, + "description": "Values must be multiples of this number. For example, multiple of 5 has valid values 0, 5, 10, -5." + }, + "maximum": { + "type": "number", + "description": "All values are less than or equal to this value (values <= maximum)." + }, + "exclusiveMaximum": { + "type": "number", + "description": "All values must be strictly less than this value (values < exclusiveMaximum)." + }, + "minimum": { + "type": "number", + "description": "All values are greater than or equal to this value (values >= minimum)." + }, + "exclusiveMinimum": { + "type": "number", + "description": "All values must be strictly greater than this value (values > exclusiveMinimum)." + }, + "format": { + "type": "string", + "default": "i32", + "description": "Format of the value in terms of how many bits of space it can use and whether it is signed or unsigned (follows the Rust integer types).", + "enum": ["i8", "i16", "i32", "i64", "i128", "u8", "u16", "u32", "u64", "u128"] + } + } + } + } + } + }, + { + "if": { + "anyOf": [ + { + "properties": { + "logicalType": { + "const": "number" + } + } + } + ] + }, + "then": { + "properties": { + "logicalTypeOptions": { + "type": "object", + "additionalProperties": false, + "properties": { + "multipleOf": { + "type": "number", + "exclusiveMinimum": 0, + "description": "Values must be multiples of this number. For example, multiple of 5 has valid values 0, 5, 10, -5." + }, + "maximum": { + "type": "number", + "description": "All values are less than or equal to this value (values <= maximum)." + }, + "exclusiveMaximum": { + "type": "number", + "description": "All values must be strictly less than this value (values < exclusiveMaximum)." + }, + "minimum": { + "type": "number", + "description": "All values are greater than or equal to this value (values >= minimum)." + }, + "exclusiveMinimum": { + "type": "number", + "description": "All values must be strictly greater than this value (values > exclusiveMinimum)." + }, + "format": { + "type": "string", + "default": "i32", + "description": "Format of the value in terms of how many bits of space it can use (follows the Rust float types).", + "enum": ["f32", "f64"] + } + } + } + } + } + }, + { + "if": { + "properties": { + "logicalType": { + "const": "object" + } + } + }, + "then": { + "properties": { + "logicalTypeOptions": { + "type": "object", + "additionalProperties": false, + "properties": { + "maxProperties": { + "type": "integer", + "minimum": 0, + "description": "Maximum number of properties." + }, + "minProperties": { + "type": "integer", + "minimum": 0, + "default": 0, + "description": "Minimum number of properties." + }, + "required": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1, + "uniqueItems": true, + "description": "Property names that are required to exist in the object." + } + } + }, + "properties": { + "type": "array", + "description": "A list of properties for the object.", + "items": { + "$ref": "#/$defs/SchemaProperty" + } + } + } + } + }, + { + "if": { + "properties": { + "logicalType": { + "const": "array" + } + } + }, + "then": { + "properties": { + "logicalTypeOptions": { + "type": "object", + "additionalProperties": false, + "properties": { + "maxItems": { + "type": "integer", + "minimum": 0, + "description": "Maximum number of items." + }, + "minItems": { + "type": "integer", + "minimum": 0, + "default": 0, + "description": "Minimum number of items" + }, + "uniqueItems": { + "type": "boolean", + "default": false, + "description": "If set to true, all items in the array are unique." + } + } + }, + "items": { + "$ref": "#/$defs/SchemaItemProperty", + "description": "List of items in an array (only applicable when `logicalType: array`)." + } + } + } + }, + { + "if": { + "properties": { + "logicalType": { + "const": "map" + } + }, + "required": [ + "logicalType" + ] + }, + "then": { + "properties": { + "map": { + "type": "object", + "description": "Key/value definition for a map (only applicable when `logicalType: map`). See RFC 0030.", + "additionalProperties": false, + "properties": { + "key": { + "$ref": "#/$defs/SchemaItemProperty", + "description": "Definition of the map's key." + }, + "value": { + "$ref": "#/$defs/SchemaItemProperty", + "description": "Definition of the map's value." + } + }, + "required": ["key", "value"] + } + }, + "required": ["map"] + } + }, + { + "if": { + "properties": { + "logicalType": { + "const": "vector" + } + }, + "required": [ + "logicalType" + ] + }, + "then": { + "properties": { + "logicalTypeOptions": { + "type": "object", + "additionalProperties": false, + "properties": { + "dimensions": { + "type": "integer", + "minimum": 1, + "description": "The fixed length of the vector. Positive integer (only applicable when `logicalType: vector`). See RFC 0042.", + "examples": [384, 768, 1024, 1536, 3072] + }, + "elementType": { + "type": "string", + "enum": ["bfloat16", "binary", "float16", "float32", "float64", "int8", "uint8"], + "default": "float32", + "description": "Numeric type of each vector element. `binary` means each element is one bit (binary-quantized vectors)." + }, + "distanceMetric": { + "type": "string", + "enum": ["cosine", "dotProduct", "euclidean", "hamming", "manhattan"], + "description": "Intended similarity metric. Advisory \u2014 the physical index may differ." + }, + "embeddingModel": { + "type": "string", + "description": "Identifier of the model used to produce the vectors.", + "examples": ["openai/text-embedding-3-small", "cohere/embed-english-v3.0"] + }, + "embeddingModelVersion": { + "type": "string", + "description": "Version or revision of the embedding model, when the model identifier does not already carry one." + }, + "normalized": { + "type": "boolean", + "default": false, + "description": "True if vectors are L2-normalized before storage, which makes cosine and dotProduct equivalent." + } + }, + "required": ["dimensions"] + } + } + } + } + ], + "unevaluatedProperties": false + }, + "SchemaProperty": { + "type": "object", + "$ref": "#/$defs/SchemaBaseProperty", + "required": ["name"], + "unevaluatedProperties": false + }, + "SchemaItemProperty": { + "type": "object", + "$ref": "#/$defs/SchemaBaseProperty", + "properties": { + "properties": { + "type": "array", + "description": "A list of properties for the object.", + "items": { + "$ref": "#/$defs/SchemaProperty" + } + } + }, + "unevaluatedProperties": false + }, + "Tags": { + "type": "array", + "description": "A list of tags that may be assigned to the elements (object or property); the tags keyword may appear at any level. Tags may be used to better categorize an element. For example, `finance`, `sensitive`, `employee_record`.", + "examples": ["finance", "sensitive", "employee_record"], + "items": { + "type": "string" + } + }, + "DataQuality": { + "type": "object", + "properties": { + "id": { + "$ref": "#/$defs/StableId" + }, + "authoritativeDefinitions": { + "$ref": "#/$defs/AuthoritativeDefinitions" + }, + "businessImpact": { + "type": "string", + "description": "Consequences of the rule failure.", + "examples": ["operational", "regulatory"] + }, + "customProperties": { + "type": "array", + "description": "Additional properties required for rule execution.", + "items": { + "$ref": "#/$defs/CustomProperty" + } + }, + "description": { + "type": "string", + "description": "Describe the quality check to be completed." + }, + "dimension": { + "type": "string", + "description": "The key performance indicator (KPI) or dimension for data quality.", + "enum": ["accuracy", "completeness", "conformity", "consistency", "coverage", "timeliness", "uniqueness"] + }, + "method": { + "type": "string", + "examples": ["reconciliation"] + }, + "name": { + "type": "string", + "description": "Name of the data quality check." + }, + "schedule": { + "type": "string", + "description": "Rule execution schedule details.", + "examples": ["0 20 * * *"] + }, + "scheduler": { + "type": "string", + "description": "The name or type of scheduler used to start the data quality check.", + "examples": ["cron"] + }, + "severity": { + "type": "string", + "description": "The severance of the quality rule.", + "examples": ["info", "warning", "error"] + }, + "tags": { + "$ref": "#/$defs/Tags" + }, + "type": { + "type": "string", + "description": "The type of quality check. 'text' is human-readable text that describes the quality of the data. 'library' is a set of maintained predefined quality attributes such as row count or unique. 'sql' is an individual SQL query that returns a value that can be compared. 'custom' is quality attributes that are vendor-specific, such as Soda or Great Expectations.", + "enum": ["text", "library", "sql", "custom"], + "default": "library" + }, + "unit": { + "type": "string", + "description": "Unit the rule is using, popular values are `rows` or `percent`, but any value is allowed.", + "examples": ["rows", "percent"] + } + }, + "allOf": [ + { + "if": { + "anyOf": [ + { + "properties": { + "type": { + "const": "library" + } + }, + "required": ["type"] + }, + { + "properties": { + "metric": { + "type": "string" + } + }, + "required": ["metric"] + } + ] + }, + "then": { + "$ref": "#/$defs/DataQualityLibrary" + } + }, + { + "if": { + "properties": { + "type": { + "const": "sql" + } + }, + "required": ["type"] + }, + "then": { + "$ref": "#/$defs/DataQualitySql" + } + }, + { + "if": { + "properties": { + "type": { + "const": "custom" + } + }, + "required": ["type"] + }, + "then": { + "$ref": "#/$defs/DataQualityCustom" + } + } + ], + "unevaluatedProperties": false + }, + "DataQualityChecks": { + "type": "array", + "description": "Data quality rules with all the relevant information for rule setup and execution.", + "items": { + "$ref": "#/$defs/DataQuality" + } + }, + "DataQualityOperators": { + "type": "object", + "description": "Common comparison operators for data quality checks.", + "oneOf": [ + { + "properties": { + "mustBe": { + "description": "Must be equal to the value to be valid. When using numbers, it is equivalent to '='." + } + }, + "required": ["mustBe"] + }, + { + "properties": { + "mustNotBe": { + "description": "Must not be equal to the value to be valid. When using numbers, it is equivalent to '!='." + } + }, + "required": ["mustNotBe"] + }, + { + "properties": { + "mustBeGreaterThan": { + "type": "number", + "description": "Must be greater than the value to be valid. It is equivalent to '>'." + } + }, + "required": ["mustBeGreaterThan"] + }, + { + "properties": { + "mustBeGreaterOrEqualTo": { + "type": "number", + "description": "Must be greater than or equal to the value to be valid. It is equivalent to '>='." + } + }, + "required": ["mustBeGreaterOrEqualTo"] + }, + { + "properties": { + "mustBeLessThan": { + "type": "number", + "description": "Must be less than the value to be valid. It is equivalent to '<'." + } + }, + "required": ["mustBeLessThan"] + }, + { + "properties": { + "mustBeLessOrEqualTo": { + "type": "number", + "description": "Must be less than or equal to the value to be valid. It is equivalent to '<='." + } + }, + "required": ["mustBeLessOrEqualTo"] + }, + { + "properties": { + "mustBeBetween": { + "type": "array", + "description": "Must be between the two numbers to be valid. Smallest number first in the array.", + "minItems": 2, + "maxItems": 2, + "uniqueItems": true, + "items": { + "type": "number" + } + } + }, + "required": ["mustBeBetween"] + }, + { + "properties": { + "mustNotBeBetween": { + "type": "array", + "description": "Must not be between the two numbers to be valid. Smallest number first in the array.", + "minItems": 2, + "maxItems": 2, + "uniqueItems": true, + "items": { + "type": "number" + } + } + }, + "required": ["mustNotBeBetween"] + } + ] + }, + "DataQualityLibrary": { + "type": "object", + "allOf": [ + { + "$ref": "#/$defs/DataQualityOperators" + } + ], + "properties": { + "metric": { + "type": "string", + "description": "Define a data quality check based on the predefined metrics as per ODCS.", + "enum": ["nullValues", "missingValues", "invalidValues", "duplicateValues", "rowCount"] + }, + "rule": { + "type": "string", + "deprecated": true, + "description": "Use metric instead" + }, + "arguments": { + "type": "object", + "description": "Additional arguments for the metric, if needed." + } + }, + "required": ["metric"] + }, + "DataQualitySql": { + "type": "object", + "allOf": [ + { + "$ref": "#/$defs/DataQualityOperators" + } + ], + "properties": { + "query": { + "type": "string", + "description": "Query string that adheres to the dialect of the provided server.", + "examples": ["SELECT COUNT(*) FROM ${table} WHERE ${column} IS NOT NULL"] + } + }, + "required": ["query"] + }, + "DataQualityCustom": { + "type": "object", + "properties": { + "engine": { + "type": "string", + "description": "Name of the engine which executes the data quality checks.", + "examples": ["soda", "great-expectations", "monte-carlo", "dbt"] + }, + "implementation": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "object" + } + ] + } + }, + "required": ["engine", "implementation"] + }, + "AuthoritativeDefinitions": { + "type": "array", + "description": "List of links to sources that provide more details on the dataset; examples would be a link to an external definition, a training video, a git repo, data catalog, or another tool. Authoritative definitions follow the same structure in the standard.", + "items": { + "type": "object", + "properties": { + "id": { + "$ref": "#/$defs/StableId" + }, + "url": { + "type": "string", + "description": "URL to the authority." + }, + "type": { + "type": "string", + "description": "Type of definition for authority. See the recommended values in the documentation.", + "examples": ["businessDefinition", "canonicalUrl", "glossary", "implementation", "ontology", "taxonomy", "transformationImplementation", "tutorial", "videoTutorial"] + }, + "description": { + "type": "string", + "description": "Description of the authoritative definition for humans." + } + }, + "required": ["url", "type"], + "additionalProperties": false + } + }, + "Support": { + "type": "array", + "description": "Top level for support channels.", + "items": { + "$ref": "#/$defs/SupportItem" + } + }, + "SupportItem": { + "type": "object", + "properties": { + "id": { + "$ref": "#/$defs/StableId" + }, + "channel": { + "type": "string", + "description": "Channel name or identifier." + }, + "url": { + "type": "string", + "description": "Access URL using normal [URL scheme](https://en.wikipedia.org/wiki/URL#Syntax) (https, mailto, etc.)." + }, + "description": { + "type": "string", + "description": "Description of the channel, free text." + }, + "tool": { + "type": "string", + "description": "Name of the tool, value can be `email`, `slack`, `teams`, `discord`, `ticket`, `googlechat`, or `other`.", + "examples": ["email", "slack", "teams", "discord", "ticket", "googlechat", "other"] + }, + "scope": { + "type": "string", + "description": "Scope can be: `interactive`, `announcements`, `issues`, `notifications`.", + "examples": ["interactive", "announcements", "issues", "notifications"] + }, + "invitationUrl": { + "type": "string", + "description": "Some tools uses invitation URL for requesting or subscribing. Follows the [URL scheme](https://en.wikipedia.org/wiki/URL#Syntax)." + }, + "customProperties": { + "$ref": "#/$defs/CustomProperties" + } + }, + "required": ["channel"], + "additionalProperties": false + }, + "Pricing": { + "type": "object", + "properties": { + "id": { + "$ref": "#/$defs/StableId" + }, + "priceAmount": { + "type": "number", + "description": "Subscription price per unit of measure in `priceUnit`." + }, + "priceCurrency": { + "type": "string", + "description": "Currency of the subscription price in `price.priceAmount`." + }, + "priceUnit": { + "type": "string", + "description": "The unit of measure for calculating cost. Examples megabyte, gigabyte." + } + }, + "additionalProperties": false + }, + "TeamMember": { + "type": "object", + "description": "Team member information.", + "additionalProperties": false, + "properties": { + "id": { + "$ref": "#/$defs/StableId" + }, + "username": { + "type": "string", + "description": "The user's username or email." + }, + "name": { + "type": "string", + "description": "The user's name." + }, + "description": { + "type": "string", + "description": "The user's description." + }, + "role": { + "type": "string", + "description": "The user's job role; Examples might be owner, data steward. There is no limit on the role." + }, + "dateIn": { + "type": "string", + "format": "date", + "description": "The date when the user joined the team." + }, + "dateOut": { + "type": "string", + "format": "date", + "description": "The date when the user ceased to be part of the team." + }, + "replacedByUsername": { + "type": "string", + "description": "The username of the user who replaced the previous user." + }, + "tags": { + "$ref": "#/$defs/Tags" + }, + "customProperties": { + "type": "array", + "description": "Custom properties block.", + "items": { + "$ref": "#/$defs/CustomProperty" + } + }, + "authoritativeDefinitions": { + "$ref": "#/$defs/AuthoritativeDefinitions" + } + }, + "required": ["username"] + }, + "Team": { + "type": "object", + "description": "Team information.", + "additionalProperties": false, + "properties": { + "id": { + "$ref": "#/$defs/StableId" + }, + "name": { + "type": "string", + "description": "Team name." + }, + "description": { + "type": "string", + "description": "Team description." + }, + "members": { + "type": "array", + "description": "List of members.", + "items": { + "$ref": "#/$defs/TeamMember" + } + }, + "tags": { + "$ref": "#/$defs/Tags" + }, + "customProperties": { + "type": "array", + "description": "Custom properties block.", + "items": { + "$ref": "#/$defs/CustomProperty" + } + }, + "authoritativeDefinitions": { + "$ref": "#/$defs/AuthoritativeDefinitions" + } + } + }, + "Role": { + "type": "object", + "properties": { + "id": { + "$ref": "#/$defs/StableId" + }, + "role": { + "type": "string", + "description": "Name of the IAM role that provides access to the dataset." + }, + "description": { + "type": "string", + "description": "Description of the IAM role and its permissions." + }, + "access": { + "type": "string", + "description": "The type of access provided by the IAM role." + }, + "firstLevelApprovers": { + "type": "string", + "description": "The name(s) of the first-level approver(s) of the role." + }, + "secondLevelApprovers": { + "type": "string", + "description": "The name(s) of the second-level approver(s) of the role." + }, + "customProperties": { + "$ref": "#/$defs/CustomProperties" + } + }, + "required": ["role"], + "additionalProperties": false + }, + "ServiceLevelAgreementProperty": { + "type": "object", + "properties": { + "id": { + "$ref": "#/$defs/StableId" + }, + "property": { + "type": "string", + "description": "Specific property in SLA, check the periodic table. May requires units (more details to come)." + }, + "value": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "number" + }, + { + "type": "integer" + }, + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "description": "Agreement value. The label will change based on the property itself." + }, + "valueExt": { + "$ref": "#/$defs/AnyNonCollectionType", + "description": "Extended agreement value. The label will change based on the property itself." + }, + "unit": { + "type": "string", + "description": "**d**, day, days for days; **y**, yr, years for years, etc. Units use the ISO standard." + }, + "element": { + "type": "string", + "description": "Element(s) to check on. Multiple elements should be extremely rare and, if so, separated by commas." + }, + "driver": { + "type": "string", + "description": "Describes the importance of the SLA from the list of: `regulatory`, `analytics`, or `operational`.", + "examples": ["regulatory", "analytics", "operational"] + }, + "description": { + "type": "string", + "description": "Description of the SLA for humans.", + "examples": ["99.9% of the time, data is available by 6 AM UTC"] + }, + "scheduler": { + "type": "string", + "description": "Name of the scheduler, can be cron or any tool your organization support.", + "examples": ["cron"] + }, + "schedule": { + "type": "string", + "description": "Configuration information for the scheduling tool, for cron a possible value is 0 20 * * *.", + "examples": ["0 20 * * *"] + }, + "customProperties": { + "$ref": "#/$defs/CustomProperties" + }, + "authoritativeDefinitions": { + "$ref": "#/$defs/AuthoritativeDefinitions" + } + }, + "required": ["property", "value"], + "additionalProperties": false + }, + "CustomProperties": { + "type": "array", + "description": "A list of key/value pairs for custom properties.", + "items": { + "$ref": "#/$defs/CustomProperty" + } + }, + "EnumValue": { + "type": "object", + "description": "An allowed value within a property's enumeration. See RFC 0033.", + "properties": { + "value": { + "$ref": "#/$defs/AnyNonCollectionType", + "description": "The allowed value. Required." + }, + "label": { + "type": "string", + "description": "Human-readable label for the value, suitable for UI display." + }, + "id": { + "$ref": "#/$defs/StableId" + }, + "description": { + "type": "string", + "description": "Optional description of what this enum value represents." + }, + "customProperties": { + "type": "array", + "description": "Custom properties attached to this enum value (e.g., translations).", + "items": { + "$ref": "#/$defs/CustomProperty" + } + }, + "authoritativeDefinitions": { + "$ref": "#/$defs/AuthoritativeDefinitions" + }, + "tags": { + "$ref": "#/$defs/Tags" + } + }, + "required": ["value"], + "additionalProperties": false + }, + "Synonyms": { + "type": "array", + "description": "A list of alternative names for the object. See RFC 0041.", + "items": { + "$ref": "#/$defs/Synonym" + } + }, + "Synonym": { + "type": "object", + "description": "An alternative name for a named object, helping catalogs, AI/LLM tools, and natural language interfaces resolve business vocabulary to the underlying object. See RFC 0041.", + "properties": { + "id": { + "$ref": "#/$defs/StableId" + }, + "synonym": { + "type": "string", + "description": "The synonymous term." + }, + "description": { + "type": "string", + "description": "Short human-readable note about when or why this synonym is used." + }, + "locale": { + "type": "string", + "description": "BCP 47 language tag (e.g., `en-US`, `fr-FR`) when the synonym is language-specific." + }, + "source": { + "type": "string", + "description": "Origin of the synonym (e.g., `glossary`, `finance-team`, `legacy-system`)." + }, + "status": { + "type": "string", + "description": "Lifecycle status of the synonym (e.g., `active`, `deprecated`)." + }, + "customProperties": { + "$ref": "#/$defs/CustomProperties" + } + }, + "required": ["synonym"], + "additionalProperties": false + }, + "CustomProperty": { + "type": "object", + "properties": { + "id": { + "$ref": "#/$defs/StableId" + }, + "property": { + "type": "string", + "description": "The name of the key. Names should be in camel case–the same as if they were permanent properties in the contract." + }, + "value": { + "$ref": "#/$defs/AnyType", + "description": "The value of the key." + }, + "description": { + "type": "string", + "description": "Description of the custom property." + }, + "vendor": { + "type": "string", + "description": "Identifies the vendor, provider, or external system associated with this custom property. SHOULD be a stable, lowercase identifier matching ^[a-z0-9][a-z0-9-]*$ (e.g. confluent, zeenea, atlan, soda). Tools MUST preserve unknown vendor values.", + "examples": ["confluent", "zeenea", "atlan", "soda"] + } + }, + "required": ["property", "value"], + "additionalProperties": false + }, + "AnyType": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "number" + }, + { + "type": "integer" + }, + { + "type": "boolean" + }, + { + "type": "null" + }, + { + "type": "array" + }, + { + "type": "object" + } + ] + }, + "AnyNonCollectionType": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "number" + }, + { + "type": "integer" + }, + { + "type": "boolean" + }, + { + "type": "null" + } + ] + }, + "RelationshipBase": { + "type": "object", + "description": "Base definition for relationships between properties, typically for foreign key constraints.", + "properties": { + "id": { + "type": "string", + "description": "Optional stable identifier for this relationship, unique within its containing relationships array. SHOULD remain stable across contract versions. Cannot contain: . # / \\ @ ! % & ^" + }, + "type": { + "type": "string", + "description": "The type of relationship. Defaults to 'foreignKey'.", + "default": "foreignKey", + "enum": ["foreignKey"] + }, + "from": { + "oneOf": [ + { + "anyOf": [ + { + "$ref": "#/$defs/ShorthandReference" + }, + { + "$ref": "#/$defs/FullyQualifiedReference" + } + ], + "description": "Source property reference using fully qualified or shorthand notation." + }, + { + "type": "array", + "description": "Array of source properties for composite keys.", + "items": { + "anyOf": [ + { + "$ref": "#/$defs/ShorthandReference" + }, + { + "$ref": "#/$defs/FullyQualifiedReference" + } + ] + }, + "minItems": 1 + } + ], + "description": "Source property or properties." + }, + "to": { + "oneOf": [ + { + "anyOf": [ + { + "$ref": "#/$defs/ShorthandReference" + }, + { + "$ref": "#/$defs/FullyQualifiedReference" + } + ], + "description": "Target property reference using fully qualified or shorthand notation." + }, + { + "type": "array", + "description": "Array of target properties for composite keys.", + "items": { + "anyOf": [ + { + "$ref": "#/$defs/ShorthandReference" + }, + { + "$ref": "#/$defs/FullyQualifiedReference" + } + ] + }, + "minItems": 1 + } + ], + "description": "Target property or properties to reference." + }, + "customProperties": { + "$ref": "#/$defs/CustomProperties" + } + } + }, + "RelationshipSchemaLevel": { + "type": "object", + "description": "Relationship definition at schema level, requiring both 'from' and 'to' fields with matching types.", + "allOf": [ + { + "$ref": "#/$defs/RelationshipBase" + }, + { + "required": ["from", "to"] + }, + { + "oneOf": [ + { + "description": "Single-column relationship - both fields must be strings", + "properties": { + "from": { + "type": "string" + }, + "to": { + "type": "string" + } + } + }, + { + "description": "Composite key relationship - both fields must be arrays with matching lengths", + "properties": { + "from": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1 + }, + "to": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1 + } + } + } + ] + } + ], + "unevaluatedProperties": false + }, + "RelationshipPropertyLevel": { + "type": "object", + "description": "Relationship definition at property level, where 'from' is implicitly the current property.", + "allOf": [ + { + "$ref": "#/$defs/RelationshipBase" + }, + { + "type": "object", + "required": ["to"] + }, + { + "not": { + "required": ["from"] + }, + "description": "The 'from' field must not be specified at property level as it is implicitly derived from the property context" + } + ], + "unevaluatedProperties": false + }, + "Relationship": { + "description": "Compatibility wrapper for relationship definitions.", + "oneOf": [ + { + "$ref": "#/$defs/RelationshipSchemaLevel" + }, + { + "$ref": "#/$defs/RelationshipPropertyLevel" + } + ] + } + } +} diff --git a/fluid_build/providers/odcs/odcs.py b/fluid_build/providers/odcs/odcs.py index a7e90527..5163d6a7 100644 --- a/fluid_build/providers/odcs/odcs.py +++ b/fluid_build/providers/odcs/odcs.py @@ -5,1437 +5,188 @@ # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +"""Back-compat shim. -# fluid_build/providers/odcs/odcs.py -""" -ODCS (Open Data Contract Standard) Provider +The ODCS provider was split into :mod:`.provider`, :mod:`.mappers`, +:mod:`.validation`, and :mod:`.io` in the modular refactor. This module +re-exports :class:`OdcsProvider` plus a few thin delegating methods that +older tests still call as private members of the class (``_map_status_to_odcs``, +``_extract_team``, ``_extract_field_quality``, ``_fluid_field_to_odcs_property``, +``_map_type_to_logical``, ``_map_type_to_physical``). + +New code should import from :mod:`fluid_build.providers.odcs` directly: -Bidirectional conversion between FLUID and ODCS v3.1.0 (Bitol.io standard). -Handles data contract schema, quality, and SLA specifications. + from fluid_build.providers.odcs import OdcsProvider """ from __future__ import annotations -import json -import logging -import os -from collections.abc import Iterable, Mapping, Sequence -from pathlib import Path -from typing import Any, Dict, List, Optional, Union - -from fluid_build.providers.base import ApplyResult, BaseProvider, ProviderError - - -class OdcsProvider(BaseProvider): - """ - ODCS (Open Data Contract Standard) provider. - - Supports: - - Export: FLUID → ODCS v3.1.0 - - Import: ODCS v3.1.0 → FLUID - - Validation: Against JSON Schema - - Specification: https://github.com/bitol-io/open-data-contract-standard - """ - - def __init__(self): - super().__init__() - self.logger = logging.getLogger(f"{__name__}.{self.__class__.__name__}") - - # ODCS version - self.odcs_version = "v3.1.0" - self.odcs_spec_url = "https://github.com/bitol-io/open-data-contract-standard" - - # Load JSON Schema for validation - self.schema = self._load_schema() - - # Configuration - # Quality checks now enabled by default with ODCS v3.1.0 compliant format - # Disable with ODCS_INCLUDE_QUALITY=false if needed - self.include_quality_checks = os.getenv("ODCS_INCLUDE_QUALITY", "true").lower() == "true" - self.include_sla = os.getenv("ODCS_INCLUDE_SLA", "true").lower() == "true" - - @property - def name(self) -> str: - return "odcs" - - def capabilities(self) -> Mapping[str, bool]: - """Signal provider capabilities.""" - caps = super().capabilities() - caps = dict(caps) - caps.update( - { - "planning": False, - "apply": False, - "render": True, # Export capability - "validate": True, # Validation against ODCS schema - "supports_batch": False, - } - ) - return caps - - def _load_schema(self) -> Optional[Dict[str, Any]]: - """Load ODCS JSON Schema for validation.""" - schema_path = Path(__file__).parent / "odcs-schema-v3.1.0.json" - - if not schema_path.exists(): - self.logger.warning(f"ODCS schema not found: {schema_path}") - return None - - try: - with open(schema_path) as f: - return json.load(f) - except Exception as e: - self.logger.error(f"Failed to load ODCS schema: {e}") - return None - - def plan(self, contract: Mapping[str, Any]) -> List[Dict[str, Any]]: - """Plan not supported for conversion provider.""" - raise ProviderError( - "ODCS provider does not support plan(). Use render() for export or import() for conversion." - ) - - def apply(self, actions: Iterable[Mapping[str, Any]]) -> ApplyResult: - """Apply not supported - use render() or import.""" - raise ProviderError( - "ODCS provider does not support apply(). Use render() for export or import for conversion." - ) - - def render( - self, - src: Union[Mapping[str, Any], Sequence[Mapping[str, Any]]], - *, - out: Optional[Union[Path, str]] = None, - fmt: Optional[str] = "yaml", - expose_id: Optional[str] = None, - ) -> Dict[str, Any]: - """ - Export FLUID contract to ODCS format. - - Args: - src: FLUID contract dictionary - out: Output file path (optional) - fmt: Output format ('yaml' or 'json') - expose_id: When set, export only this output port (exposeId). - The resulting ODCS contract id is scoped to - ``{product_id}.{expose_id}`` and ``status`` is - derived from that expose's ``lifecycle.state``. - - Returns: - ODCS-compliant dictionary - """ - if isinstance(src, list): - raise ProviderError( - "ODCS export does not support batch processing. " - "Each contract should be exported separately." - ) - - self.logger.info("Converting FLUID contract to ODCS v3.1.0") - - # Scope to a single output port when requested - fluid = self._filter_to_expose(src, expose_id) if expose_id else src - - # Convert to ODCS - odcs_contract = self._fluid_to_odcs(fluid) - - # Validate if schema available (optional - can be disabled) - if self.schema and os.getenv("ODCS_VALIDATE", "false").lower() == "true": - self.validate_contract(odcs_contract) - - # Write output if path provided - if out: - self._write_output(odcs_contract, out, fmt) - self.logger.info(f"Exported ODCS contract: {out}") - - return odcs_contract - - def render_all_ports( - self, - fluid: Mapping[str, Any], - *, - out_dir: Optional[Union[Path, str]] = None, - fmt: Optional[str] = "yaml", - ) -> List[tuple]: - """ - Export one ODCS contract per output port. - - Args: - fluid: FLUID contract dictionary - out_dir: Directory to write files into. - Files are named ``product.odcs..``. - If *None*, files are not written. - fmt: Output format ('yaml' or 'json') - - Returns: - List of ``(expose_id, odcs_dict)`` tuples in expose order. - """ - import copy - - results = [] - for expose in fluid.get("exposes", []): - if not isinstance(expose, dict): - continue - eid = expose.get("exposeId") or expose.get("id") - if not eid: - self.logger.warning("Expose missing exposeId — skipping") - continue - out_path = None - if out_dir is not None: - out_path = Path(out_dir) / f"product.odcs.{eid}.{fmt}" - odcs = self.render(fluid, out=out_path, fmt=fmt, expose_id=eid) - results.append((eid, odcs)) - return results - - def _filter_to_expose(self, fluid: Mapping[str, Any], expose_id: str) -> Dict[str, Any]: - """ - Return a shallow copy of *fluid* filtered to a single output port. - - The returned dict carries two private sentinel keys used by - ``_fluid_to_odcs``: - - * ``_scoped_id`` – ``{product_id}.{expose_id}`` - * ``_scoped_status`` – expose-level ``lifecycle.state`` (or ``active``) - """ - import copy - - scoped = copy.deepcopy(dict(fluid)) - exposes = [ - e - for e in scoped.get("exposes", []) - if isinstance(e, dict) and (e.get("exposeId") == expose_id or e.get("id") == expose_id) - ] - if not exposes: - raise ProviderError( - f"Expose '{expose_id}' not found in contract. " - f"Available exposeIds: " - f"{[e.get('exposeId') or e.get('id') for e in fluid.get('exposes', [])]}" - ) - scoped["exposes"] = exposes - - # Derive scoped contract id - product_id = self._extract_contract_id(fluid) - scoped["_scoped_id"] = f"{product_id}.{expose_id}" - - # Derive status from expose lifecycle - lifecycle = exposes[0].get("lifecycle", {}) - scoped["_scoped_status"] = lifecycle.get("state", "active") - - return scoped - - def import_contract(self, odcs: Union[Mapping[str, Any], str, Path]) -> Dict[str, Any]: - """ - Import ODCS contract to FLUID format. - - Args: - odcs: ODCS contract (dict, JSON string, or file path) - - Returns: - FLUID contract dictionary - """ - # Parse input - if isinstance(odcs, (str, Path)): - odcs_data = self._read_input(odcs) - else: - odcs_data = dict(odcs) - - # Validate - if self.schema: - self.validate_contract(odcs_data) - - self.logger.info("Converting ODCS contract to FLUID") - - # Convert to FLUID - fluid_contract = self._odcs_to_fluid(odcs_data) - - return fluid_contract - - def _fluid_to_odcs(self, fluid: Mapping[str, Any]) -> Dict[str, Any]: - """ - Convert FLUID contract to ODCS. - - Args: - fluid: FLUID contract dictionary - - Returns: - ODCS-compliant contract dictionary - """ - metadata = fluid.get("metadata", {}) - fluid.get("contract", {}) - - # When filtered to a single expose, _scoped_id / _scoped_status are set - contract_id = fluid.get("_scoped_id") or self._extract_contract_id(fluid) - raw_status = fluid.get("_scoped_status") or metadata.get("status", "active") - - # Required fields - odcs_contract = { - "version": metadata.get("version", "1.0.0"), - "apiVersion": self.odcs_version, - "kind": "DataContract", - "id": contract_id, - "status": self._map_status_to_odcs(raw_status), - } - - # Optional but common fields - name = metadata.get("name") - if name: - odcs_contract["name"] = name - - description = metadata.get("description") - if description: - # ODCS description is an object, not a string - odcs_contract["description"] = {"purpose": description} - - # Team - team = self._extract_team(fluid) - if team: - odcs_contract["team"] = team - - # Tags - tags = metadata.get("tags", []) - if tags: - odcs_contract["tags"] = tags - - # Schema (from exposes) - always include - schema = self._extract_schema(fluid) - odcs_contract["schema"] = schema - - # Servers (from expects/exposes) - always include - servers = self._extract_servers(fluid) - odcs_contract["servers"] = servers - - # SLA Properties - if self.include_sla: - sla = self._extract_sla_properties(fluid) - if sla: - odcs_contract["slaProperties"] = sla - - # Quality (from schema fields) - if self.include_quality_checks: - quality = self._extract_quality(fluid) - if quality: - odcs_contract["quality"] = quality - - return odcs_contract - - def _odcs_to_fluid(self, odcs: Mapping[str, Any]) -> Dict[str, Any]: - """ - Convert ODCS contract to FLUID. - - Args: - odcs: ODCS contract dictionary - - Returns: - FLUID contract dictionary - """ - # Build FLUID structure - fluid_contract = { - "metadata": { - "version": odcs.get("version", "1.0.0"), - "name": odcs.get("name", odcs.get("id")), - "status": self._map_status_from_odcs(odcs.get("status", "active")), - }, - "contract": { - "id": odcs.get("id"), - }, - "exposes": [], - "expects": [], - } - - # Description - description = odcs.get("description") - if description: - fluid_contract["metadata"]["description"] = description - - # Tags - tags = odcs.get("tags", []) - if tags: - fluid_contract["metadata"]["tags"] = tags - - # Owner/Team - team = odcs.get("team") - if team: - fluid_contract["owner"] = self._odcs_team_to_fluid_owner(team) - - # Schema → Exposes - schema = odcs.get("schema", []) - if schema: - expose = self._odcs_schema_to_expose(odcs) - if expose: - fluid_contract["exposes"].append(expose) - - # Servers → Expects - servers = odcs.get("servers", []) - for server in servers: - expect = self._odcs_server_to_expect(server) - if expect: - fluid_contract["expects"].append(expect) - - return fluid_contract - - def _extract_contract_id(self, fluid: Mapping[str, Any]) -> str: - """Extract contract ID from FLUID.""" - # FLUID 0.7.1 uses top-level 'id' field - if "id" in fluid: - return fluid["id"] - - # Try contract.id (older format) - contract = fluid.get("contract") - if isinstance(contract, dict): - contract_id = contract.get("id") - if contract_id: - return contract_id - - # Fallback to metadata.id - metadata = fluid.get("metadata") - if isinstance(metadata, dict): - contract_id = metadata.get("id") - if contract_id: - return contract_id - - raise ProviderError( - "Contract missing required 'id' field. " - "Expected one of: fluid['id'], fluid['contract']['id'], or fluid['metadata']['id']" - ) - +from collections.abc import Mapping +from typing import Any, Dict, List, Optional + +from .mappers import quality as _quality_mapper +from .mappers import schema as _schema_mapper +from .mappers import servers as _servers_mapper +from .mappers import sla as _sla_mapper +from .mappers import team as _team_mapper +from .mappers.base import ExportCtx, get_field_passthrough +from .mappers.types import ( + fluid_to_logical, + fluid_to_odcs_status, + fluid_to_physical, + logical_to_fluid, + odcs_to_fluid_status, + provider_to_server_type, + server_type_to_provider, +) +from .provider import OdcsProvider as _OdcsProviderImpl +from .validation import validate as _validate + + +class OdcsProvider(_OdcsProviderImpl): + """OdcsProvider with legacy private-method shims for back-compat tests.""" + + # --- status mapping ------------------------------------------------- def _map_status_to_odcs(self, status: str) -> str: - """ - Map FLUID status to ODCS status. - - Mappings: - - draft → draft - - active → active - - deprecated → deprecated - - retired → retired - - development → draft - """ - mapping = { - "draft": "draft", - "active": "active", - "deprecated": "deprecated", - "retired": "retired", - "development": "draft", - } - - return mapping.get(status, "active") + return fluid_to_odcs_status(status) def _map_status_from_odcs(self, status: str) -> str: - """Map ODCS status to FLUID status.""" - # Reverse mapping - mapping = { - "draft": "draft", - "active": "active", - "deprecated": "deprecated", - "retired": "retired", - } - - return mapping.get(status, "active") - - def _extract_team(self, fluid: Mapping[str, Any]) -> Optional[str]: - """ - Extract team information from FLUID owner. - - ODCS v3.1.0 team can be: - - Team object with name and members array - - Array of team members (deprecated) - - String team name (not valid in v3.1.0, needs to be Team object) - - We'll create a proper Team object structure. - """ - # FLUID 0.7.1: owner may live under metadata.owner - owner = fluid.get("owner") or fluid.get("metadata", {}).get("owner", {}) - if not isinstance(owner, dict): - owner = {} - - team_name = owner.get("team") or owner.get("name") - if not team_name: - return None - - # Build Team object structure (v3.1.0+) - team_obj = {"name": team_name} - - # Add members if available - members = [] - - # Add owner as first member if has name or email - if owner.get("name") or owner.get("email"): - member = {} - if owner.get("name"): - member["name"] = owner["name"] - if owner.get("email"): - member["username"] = owner["email"] # Use username field for email - if owner.get("role"): - member["role"] = owner["role"] - members.append(member) - - # Add additional contacts as members - if "contacts" in owner: - for contact in owner["contacts"]: - if isinstance(contact, dict): - member = {} - if contact.get("name"): - member["name"] = contact["name"] - if contact.get("email"): - member["username"] = contact["email"] # Use username field for email - if contact.get("role"): - member["role"] = contact["role"] - members.append(member) - - if members: - team_obj["members"] = members - - return team_obj - - def _extract_schema(self, fluid: Mapping[str, Any]) -> List[Dict[str, Any]]: - """ - Extract ODCS schema from FLUID exposes. - - ODCS v3.1.0 requires schema to be an array of SchemaObjects (logicalType: "object") - with properties arrays containing the actual fields. - - Supports both FLUID 0.5.7 and 0.7.1: - - 0.5.7: exposes.schema.fields (object with fields array) - - 0.7.1: exposes.contract.schema (array of fields) - """ - odcs_schema = [] - - exposes = fluid.get("exposes", []) - if not isinstance(exposes, list): - self.logger.warning(f"exposes is not a list: {type(exposes)}") - return odcs_schema - - for expose in exposes: - if not isinstance(expose, dict): - self.logger.warning(f"Skipping non-dict expose: {type(expose)}") - continue - - contract_schema = [] - - # Try 0.7.1 format first: contract.schema (array) - contract = expose.get("contract") - if isinstance(contract, dict): - schema = contract.get("schema", []) - if isinstance(schema, list): - contract_schema = schema - - # Fall back to 0.5.7 format: schema.fields (object) - if not contract_schema: - schema_obj = expose.get("schema") - if isinstance(schema_obj, dict): - fields = schema_obj.get("fields", []) - if isinstance(fields, list): - contract_schema = fields - - # Process fields and create SchemaObject - if contract_schema: - # Get expose metadata for naming - expose_id = expose.get("exposeId") or expose.get("id", "dataset") - - # Determine physical type from binding - binding = expose.get("binding") - physical_type = "table" # default - if isinstance(binding, dict): - platform = binding.get("platform") - if platform == "kafka": - physical_type = "topic" - elif platform in ("bigquery", "snowflake"): - physical_type = "table" - - # Convert fields to properties - properties = [] - for field in contract_schema: - if not isinstance(field, dict): - self.logger.warning(f"Skipping non-dict field: {type(field)}") - continue - try: - odcs_property = self._fluid_field_to_odcs_property(field, expose) - properties.append(odcs_property) - except Exception as e: - self.logger.error( - f"Error converting field {field.get('name', 'unknown')}: {e}" - ) - - # Create SchemaObject wrapping the properties - if properties: - schema_object = { - "name": expose_id, - "logicalType": "object", - "physicalType": physical_type, - "properties": properties, - } - - # Add description if available - description = expose.get("description") - if description: - schema_object["description"] = description - - odcs_schema.append(schema_object) - - return odcs_schema - - def _fluid_field_to_odcs_property( - self, field: Mapping[str, Any], expose: Mapping[str, Any] - ) -> Dict[str, Any]: - """ - Convert FLUID field to ODCS schema property (inside SchemaObject.properties array). - - Args: - field: FLUID field dictionary - expose: Parent expose (for context) - - Returns: - ODCS schema property - """ - # Required fields - schema_entry = { - "name": field.get("name", "unknown"), - "logicalType": self._map_type_to_logical(field.get("type", "string")), - } - - # Physical type (provider-specific) - try binding.platform or direct provider - binding = expose.get("binding") - provider = None - - # Safely extract provider from binding (handle None case) - if isinstance(binding, dict): - provider = binding.get("platform") or binding.get("provider") - - # Fall back to direct provider field - if not provider: - provider = expose.get("provider") - - if provider: - physical_type = self._map_type_to_physical(field.get("type", "string"), provider) - if physical_type: - schema_entry["physicalType"] = physical_type - - # Description - description = field.get("description") - if description: - schema_entry["description"] = description - - # Required flag - required = field.get("required", False) - schema_entry["required"] = required - - # Classification - classification = field.get("classification") - if classification: - schema_entry["classification"] = classification - - # Tags - tags = field.get("tags", []) - if tags: - schema_entry["tags"] = tags - - # Quality checks - if self.include_quality_checks: - quality = self._extract_field_quality(field) - if quality: - schema_entry["quality"] = quality - - return schema_entry + return odcs_to_fluid_status(status) + # --- type mapping --------------------------------------------------- def _map_type_to_logical(self, fluid_type: str) -> str: - """ - Map FLUID type to ODCS logicalType. - - ODCS v3.1.0 valid logicalTypes: string, date, timestamp, time, number, integer, object, array, boolean - - Args: - fluid_type: FLUID field type - - Returns: - ODCS logical type - """ - mapping = { - "string": "string", - "text": "string", - "varchar": "string", - "char": "string", - "int": "integer", - "integer": "integer", - "bigint": "integer", - "long": "integer", - "float": "number", - "double": "number", - "decimal": "number", - "numeric": "number", - "bool": "boolean", - "boolean": "boolean", - "date": "date", - "datetime": "timestamp", - "timestamp": "timestamp", - "time": "time", - "json": "object", - "object": "object", - "array": "array", - "binary": "string", # ODCS doesn't have binary type, use string - "bytes": "string", # ODCS doesn't have binary type, use string - } - - return mapping.get(fluid_type.lower(), "string") + return fluid_to_logical(fluid_type, getattr(self, "odcs_version", None)) def _map_type_to_physical(self, fluid_type: str, provider: Optional[str]) -> Optional[str]: - """ - Map FLUID type to physical type for specific provider. - - Args: - fluid_type: FLUID field type - provider: Provider name (gcp, snowflake, etc.) or None + return fluid_to_physical(fluid_type, provider, getattr(self, "odcs_version", None)) - Returns: - Physical type string or None - """ - if not provider: - return self._map_type_to_logical(fluid_type) - - provider = provider.lower() - - # BigQuery types - if provider == "gcp" or provider == "bigquery": - mapping = { - "string": "STRING", - "text": "STRING", - "int": "INT64", - "integer": "INT64", - "bigint": "INT64", - "long": "INT64", - "float": "FLOAT64", - "double": "FLOAT64", - "decimal": "NUMERIC", - "numeric": "NUMERIC", - "bool": "BOOL", - "boolean": "BOOL", - "date": "DATE", - "datetime": "DATETIME", - "timestamp": "TIMESTAMP", - "time": "TIME", - "json": "JSON", - "object": "STRUCT", - "array": "ARRAY", - "binary": "BYTES", - "bytes": "BYTES", - } - return mapping.get(fluid_type.lower()) + # --- team / owner --------------------------------------------------- + def _extract_team(self, fluid: Mapping[str, Any]) -> Optional[Dict[str, Any]]: + ctx = ExportCtx(fluid=fluid, odcs={}, logger=self.logger) + _team_mapper.to_odcs(ctx) + return ctx.odcs.get("team") - # Snowflake types - elif provider == "snowflake": - mapping = { - "string": "VARCHAR", - "text": "TEXT", - "int": "NUMBER", - "integer": "NUMBER", - "bigint": "NUMBER", - "long": "NUMBER", - "float": "FLOAT", - "double": "DOUBLE", - "decimal": "DECIMAL", - "numeric": "DECIMAL", - "bool": "BOOLEAN", - "boolean": "BOOLEAN", - "date": "DATE", - "datetime": "TIMESTAMP_NTZ", - "timestamp": "TIMESTAMP_NTZ", - "time": "TIME", - "json": "VARIANT", - "object": "OBJECT", - "array": "ARRAY", - "binary": "BINARY", - "bytes": "BINARY", - } - return mapping.get(fluid_type.lower()) - - # Generic fallback: use logical type for unknown providers - return self._map_type_to_logical(fluid_type) + def _odcs_team_to_fluid_owner(self, team: Any) -> Dict[str, Any]: + return _team_mapper._team_to_owner(team) or {} + # --- field quality -------------------------------------------------- def _extract_field_quality(self, field: Mapping[str, Any]) -> Optional[List[Dict[str, Any]]]: - """ - Extract quality checks from FLUID field and convert to ODCS v3.1.0 format. - - ODCS v3.1.0 quality check structure: - - type: "library", "text", "sql", or "custom" - - For library type: metric (nullValues, missingValues, invalidValues, duplicateValues, rowCount) - + ONE operator property (mustBe, mustNotBe, mustBeGreaterThan, etc.) - - For text type: description (human-readable) - - Example: - [ - { - "type": "library", - "metric": "nullValues", - "mustBe": 0, - "dimension": "completeness", - "description": "Field must not contain null values" - } - ] - """ - quality_checks = [] - - # 1. Required field check (not null constraint) - if field.get("required"): - quality_checks.append( - { - "type": "library", - "metric": "nullValues", - "mustBe": 0, - "dimension": "completeness", - "description": f"Field '{field.get('name', 'unknown')}' must not contain null values", - } - ) - - # 2. Primary key check (uniqueness + not null) - tags = field.get("tags", []) - is_primary_key = "primary-key" in tags or "primaryKey" in tags - - if is_primary_key: - # Primary keys must be unique - quality_checks.append( - { - "type": "library", - "metric": "duplicateValues", - "mustBe": 0, - "dimension": "uniqueness", - "description": f"Primary key field '{field.get('name', 'unknown')}' must contain only unique values", - } - ) - # Also ensure not null if not already added - if not field.get("required"): - quality_checks.append( - { - "type": "library", - "metric": "nullValues", - "mustBe": 0, - "dimension": "completeness", - "description": f"Primary key field '{field.get('name', 'unknown')}' must not contain null values", - } - ) - - # 3. Check for explicit validations (can be list or dict) - validations = field.get("validations", []) - - # Handle both list and dict formats - if isinstance(validations, dict): - # Legacy dict format: {"pattern": "regex", "min_length": 5} - validation_list = [{"type": k, "value": v} for k, v in validations.items()] - elif isinstance(validations, list): - validation_list = validations - else: - validation_list = [] - - # Process validation list - for validation in validation_list: - if not isinstance(validation, dict): - continue - - val_type = validation.get("type", "") - val_value = validation.get("value") - val_values = validation.get("values") - field_name = field.get("name", "unknown") - - # Pattern/regex validation - if val_type in ("pattern", "regex") and val_value: - quality_checks.append( - { - "type": "text", - "description": f"Field '{field_name}' must match pattern: {val_value}", - } - ) - - # Min/max length constraints - elif val_type == "min_length" and val_value is not None: - quality_checks.append( - { - "type": "text", - "description": f"Field '{field_name}' must have minimum length of {val_value}", - } - ) - - elif val_type == "max_length" and val_value is not None: - quality_checks.append( - { - "type": "text", - "description": f"Field '{field_name}' must have maximum length of {val_value}", - } - ) - - # Min/max value constraints (for numeric fields) - elif val_type == "min_value" and val_value is not None: - quality_checks.append( - { - "type": "text", - "description": f"Field '{field_name}' must be greater than or equal to {val_value}", - } - ) - - elif val_type == "max_value" and val_value is not None: - quality_checks.append( - { - "type": "text", - "description": f"Field '{field_name}' must be less than or equal to {val_value}", - } - ) - - # Allowed values / enum constraints - elif val_type in ("allowed_values", "enum") and val_values: - values_str = ", ".join(str(v) for v in val_values[:5]) - if len(val_values) > 5: - values_str += f", ... ({len(val_values)} total)" - quality_checks.append( - { - "type": "text", - "description": f"Field '{field_name}' must be one of: {values_str}", - } - ) - - # Not null constraint - elif val_type == "not_null" and val_value: - if not field.get("required"): # Avoid duplicate if already added - quality_checks.append( - { - "type": "library", - "metric": "nullValues", - "mustBe": 0, - "dimension": "completeness", - "description": f"Field '{field_name}' must not contain null values", - } - ) - - # Unique constraint - elif val_type == "unique" and val_value: - if not is_primary_key: # Avoid duplicate if already added for PK - quality_checks.append( - { - "type": "library", - "metric": "duplicateValues", - "mustBe": 0, - "dimension": "uniqueness", - "description": f"Field '{field_name}' must contain only unique values", - } - ) - - # 4. Custom quality checks from field metadata - custom_quality = field.get("quality") - if isinstance(custom_quality, list): - for qc in custom_quality: - if isinstance(qc, dict): - # If it's already in ODCS format, keep it - if "type" in qc and "metric" in qc: - quality_checks.append(qc) - elif "type" in qc and qc.get("type") == "text": - quality_checks.append(qc) - # Otherwise convert legacy format - elif "description" in qc: - quality_checks.append({"type": "text", "description": qc["description"]}) - - return quality_checks if quality_checks else None - - def _extract_servers(self, fluid: Mapping[str, Any]) -> List[Dict[str, Any]]: - """ - Extract ODCS servers from FLUID expects/exposes. - - Servers define where data is stored/accessed. - """ - servers = [] - - # From exposes (data source locations) - exposes = fluid.get("exposes", []) - if isinstance(exposes, list): - for expose in exposes: - try: - server = self._expose_to_server(expose) - if server and isinstance(server, dict): - servers.append(server) - except Exception as e: - self.logger.error(f"Error extracting server from expose: {e}") - - # From expects (dependencies) - expects = fluid.get("expects", []) - if isinstance(expects, list): - for expect in expects: - try: - server = self._expect_to_server(expect) - if server and isinstance(server, dict): - servers.append(server) - except Exception as e: - self.logger.error(f"Error extracting server from expect: {e}") - - return servers - - def _expose_to_server(self, expose: Mapping[str, Any]) -> Optional[Dict[str, Any]]: - """Convert FLUID expose to ODCS server.""" - if not isinstance(expose, dict): - self.logger.warning(f"Skipping non-dict expose: {type(expose)}") + if not getattr(self, "include_quality_checks", True): return None + return _quality_mapper.to_odcs_property(field) - # Try 0.7.1 format: binding.platform - binding = expose.get("binding") + # --- field property (export) --------------------------------------- + def _fluid_field_to_odcs_property( + self, field: Mapping[str, Any], expose: Mapping[str, Any] + ) -> Dict[str, Any]: provider = None - location = None - - # Safely extract from binding (handle None case) - if isinstance(binding, dict): + binding = expose.get("binding") if isinstance(expose, Mapping) else None + if isinstance(binding, Mapping): provider = binding.get("platform") or binding.get("provider") - location = binding.get("location") - - # Fall back to 0.5.7 format: direct provider field - if not provider: + if not provider and isinstance(expose, Mapping): provider = expose.get("provider") + prop = _schema_mapper._field_to_property( + field, + provider, + include_quality=getattr(self, "include_quality_checks", True), + api_version=getattr(self, "odcs_version", None), + ) + prop.pop("odcs_passthrough", None) + if get_field_passthrough(field): + pass # _field_to_property already merged pass-through + return prop - if not provider: - return None - - # Use exposeId (0.7.1) or id (0.5.7) - expose_id = expose.get("exposeId") or expose.get("id", "default") - - server = { - "id": expose_id, - "server": expose_id, - "type": self._map_provider_to_server_type(provider), - } - - # Location/connection details - ensure it's a dict - if not isinstance(location, dict): - # Fall back to direct location (0.5.7) - location = expose.get("location") - - if isinstance(location, dict) and location: - try: - server.update(self._extract_server_details(location, provider)) - except Exception as e: - self.logger.error(f"Error extracting server details: {e}") - - return server - - def _expect_to_server(self, expect: Mapping[str, Any]) -> Optional[Dict[str, Any]]: - """Convert FLUID expect to ODCS server.""" - if not isinstance(expect, dict): - self.logger.warning(f"Skipping non-dict expect: {type(expect)}") - return None - - # Try binding.platform (0.7.1) or direct provider (0.5.7) - binding = expect.get("binding") - provider = None - location = None - - # Safely extract from binding (handle None case) - if isinstance(binding, dict): - provider = binding.get("platform") or binding.get("provider") - location = binding.get("location") - - if not provider: - provider = expect.get("provider") - - if not provider: - return None - - expect_id = expect.get("id", "dependency") - server = { - "id": expect_id, - "server": expect_id, - "type": self._map_provider_to_server_type(provider), - } - - # Location/connection details - ensure it's a dict - if not isinstance(location, dict): - location = expect.get("location") - - if isinstance(location, dict) and location: - try: - server.update(self._extract_server_details(location, provider)) - except Exception as e: - self.logger.error(f"Error extracting server details: {e}") - - return server - - def _map_provider_to_server_type(self, provider: str) -> str: - """ - Map FLUID provider to ODCS server type. - - ODCS supports 30+ server types. - """ - mapping = { - "gcp": "bigquery", - "bigquery": "bigquery", - "snowflake": "snowflake", - "aws": "s3", - "s3": "s3", - "redshift": "redshift", - "athena": "athena", - "azure": "azure", - "databricks": "databricks", - "postgres": "postgres", - "postgresql": "postgres", - "mysql": "mysql", - "kafka": "kafka", - "mongodb": "mongodb", - "elasticsearch": "elasticsearch", - "local": "local", - } - - return mapping.get(provider.lower(), "custom") - - def _extract_server_details(self, location: Mapping[str, Any], provider: str) -> Dict[str, Any]: - """ - Extract server connection details from location. - - Returns provider-specific fields for ODCS server. - """ - details = {} - - provider = provider.lower() - - # BigQuery - if provider in ("gcp", "bigquery"): - if "project" in location: - details["project"] = location["project"] - if "dataset" in location: - details["dataset"] = location["dataset"] - - # Snowflake - elif provider == "snowflake": - if "account" in location: - details["account"] = location["account"] - if "database" in location: - details["database"] = location["database"] - if "schema" in location: - details["schema"] = location["schema"] - if "table" in location: - details["table"] = location["table"] - - # S3 - elif provider in ("aws", "s3"): - if "bucket" in location: - details["bucket"] = location["bucket"] - if "path" in location or "key" in location: - details["path"] = location.get("path") or location.get("key") - if "region" in location: - details["region"] = location["region"] - - # Kafka - elif provider == "kafka": - # ODCS Kafka servers require 'host' property - # Extract host from topic name or location - if "host" in location: - details["host"] = location["host"] - elif "account" in location: - # Use account as host for streaming platform - details["host"] = location["account"] - - # Format is optional - if "format" in location: - details["format"] = location["format"] - - # Generic fields for other providers - else: - # Copy all location fields - details.update(location) - - return details - + # --- SLA ------------------------------------------------------------ def _extract_sla_properties(self, fluid: Mapping[str, Any]) -> Optional[List[Dict[str, Any]]]: - """ - Extract SLA properties from FLUID contract. - - ODCS SLA structure: - { - "interval": "daily", - "sla": "00:00", - "completenessKpi": 0.95, - ... - } - - Sources (in priority order): - 1. expose-level ``qos`` block (FLUID 0.7.1) - 2. ``metadata.update_frequency`` / ``metadata.availability`` (legacy) - """ - sla_values: Dict[str, Any] = {} - - # --- 1. Expose-level qos block (FLUID 0.7.1) ----------------------- - # When scoped to a single expose via render(expose_id=...) there will - # be exactly one expose; otherwise we merge qos from all exposes. - for expose in fluid.get("exposes", []): - if not isinstance(expose, dict): - continue - qos = expose.get("qos", {}) - if not isinstance(qos, dict): - continue - - # availability: "99.5%" → strip % and convert to float - avail = qos.get("availability") - if avail is not None and "availability" not in sla_values: - try: - avail_str = str(avail).strip() - if avail_str.endswith("%"): - sla_values["availability"] = float(avail_str.rstrip("%")) / 100 - else: - parsed = float(avail_str) - sla_values["availability"] = parsed / 100 if parsed > 1 else parsed - except (ValueError, TypeError): - sla_values["availability"] = str(avail) - - # freshnessSLO: ISO-8601 duration string (e.g. "PT5M") → interval - freshness = qos.get("freshnessSLO") or qos.get("freshness_slo") - if freshness and "interval" not in sla_values: - sla_values["interval"] = str(freshness) - - # SLA labels as custom properties - qos_labels = qos.get("labels", {}) - if isinstance(qos_labels, dict) and qos_labels: - for key, value in qos_labels.items(): - sla_values.setdefault(f"label:{key}", value) - - # --- 2. Metadata-level fallbacks (legacy) --------------------------- - metadata = fluid.get("metadata", {}) - - # Update frequency → interval (if not already set from qos) - update_frequency = metadata.get("update_frequency") - if update_frequency and "interval" not in sla_values: - sla_values["interval"] = update_frequency - - # Availability/uptime (if not already set from qos) - availability = metadata.get("availability") - if availability and "availability" not in sla_values: - try: - availability_f = float(availability) - sla_values["availability"] = ( - availability_f / 100 if availability_f > 1 else availability_f - ) - except (ValueError, TypeError): - pass - - # Quality thresholds - quality = metadata.get("quality_threshold") - if quality: - try: - sla_values["completenessKpi"] = float(quality) - except (ValueError, TypeError): - pass - - if not sla_values: + if not getattr(self, "include_sla", True): return None + ctx = ExportCtx( + fluid=fluid, + odcs={}, + logger=self.logger, + options={"include_sla": True}, + ) + _sla_mapper.to_odcs(ctx) + return ctx.odcs.get("slaProperties") - return [ - {"property": property_name, "value": value} - for property_name, value in sla_values.items() - if value is not None - ] - - def validate_contract(self, odcs: Mapping[str, Any]) -> None: - """Validate an ODCS contract payload against the configured schema.""" - self._validate_odcs(odcs) - + # --- contract-level quality ---------------------------------------- def _extract_quality(self, fluid: Mapping[str, Any]) -> Optional[Dict[str, Any]]: - """ - Extract quality properties from FLUID contract. - - ODCS quality structure: - { - "type": "SodaCL", - "specification": "..." - } - """ - # Check for quality definitions - quality_spec = fluid.get("quality", {}) - - if not quality_spec: - return None - - quality_type = quality_spec.get("type", "custom") - specification = quality_spec.get("specification", "") - - return {"type": quality_type, "specification": specification} - - def _odcs_team_to_fluid_owner(self, team: str) -> Dict[str, Any]: - """Convert ODCS team (string) to FLUID owner.""" - return { - "team": team, - "name": team, - } + ctx = ExportCtx( + fluid=fluid, + odcs={}, + logger=self.logger, + options={"include_quality_checks": True}, + ) + _quality_mapper.to_odcs(ctx) + return ctx.odcs.get("quality") + # --- schema (import) ----------------------------------------------- def _odcs_schema_to_expose(self, odcs: Mapping[str, Any]) -> Optional[Dict[str, Any]]: - """ - Convert ODCS schema to FLUID expose. - - Groups all schema fields into one expose. - """ - schema = odcs.get("schema", []) - if not schema: + """DEPRECATED. Legacy helper that flattens every SchemaObject in an + ODCS contract into a single FLUID expose. Kept only for back-compat + with one specific test in ``tests/test_odcs_mappings.py``; will be + removed in the next major release. + + New code must call :meth:`import_contract` instead, which correctly + emits **one FLUID expose per SchemaObject** — preserving the multi- + port shape that the new modular mapper pipeline produces. + """ + import warnings + + warnings.warn( + "OdcsProvider._odcs_schema_to_expose is deprecated; use " + "OdcsProvider.import_contract() instead. This helper collapses " + "multi-SchemaObject contracts into a single expose and will be " + "removed in the next major release.", + DeprecationWarning, + stacklevel=2, + ) + schema = odcs.get("schema") or [] + if not isinstance(schema, list) or not schema: return None - - expose = { + expose: Dict[str, Any] = { "id": odcs.get("id", "default"), "version": odcs.get("version", "1.0.0"), "description": odcs.get("description", ""), "schema": {"fields": []}, } - - # Convert each schema entry to field - for schema_entry in schema: - field = self._odcs_schema_to_field(schema_entry) - expose["schema"]["fields"].append(field) - + for schema_object in schema: + if not isinstance(schema_object, Mapping): + continue + for prop in schema_object.get("properties") or [schema_object]: + if not isinstance(prop, Mapping): + continue + expose["schema"]["fields"].append(self._odcs_schema_to_field(prop)) return expose def _odcs_schema_to_field(self, schema_entry: Mapping[str, Any]) -> Dict[str, Any]: - """Convert ODCS schema entry to FLUID field.""" - field = { - "name": schema_entry.get("name", "unknown"), - "type": self._map_logical_type_to_fluid(schema_entry.get("logicalType", "string")), - } - - # Description - description = schema_entry.get("description") - if description: - field["description"] = description - - # Required (inverse of isNullable) - is_nullable = schema_entry.get("isNullable", True) - field["required"] = not is_nullable - - # Classification - classification = schema_entry.get("classification") - if classification: - field["classification"] = classification - - # Tags - tags = schema_entry.get("tags", []) - if tags: - field["tags"] = tags - - return field + return _schema_mapper._property_to_field(schema_entry) def _map_logical_type_to_fluid(self, logical_type: str) -> str: - """Map ODCS logical type to FLUID type.""" - mapping = { - "string": "string", - "integer": "int", - "long": "bigint", - "float": "float", - "double": "double", - "decimal": "decimal", - "boolean": "bool", - "date": "date", - "timestamp": "timestamp", - "time": "time", - "object": "object", - "array": "array", - "binary": "binary", - } - - return mapping.get(logical_type.lower(), "string") + return logical_to_fluid(logical_type) + # --- servers (import / export) ------------------------------------- def _odcs_server_to_expect(self, server: Mapping[str, Any]) -> Optional[Dict[str, Any]]: - """Convert ODCS server to FLUID expect.""" - server_type = server.get("type") - if not server_type: - return None - - expect = { - "id": server.get("name", "dependency"), - "provider": self._map_server_type_to_provider(server_type), - } + return _servers_mapper._server_to_expect(server) - # Extract location details - location = self._extract_location_from_server(server) - if location: - expect["location"] = location - - return expect + def _map_provider_to_server_type(self, provider: str) -> str: + return provider_to_server_type(provider, getattr(self, "odcs_version", None)) def _map_server_type_to_provider(self, server_type: str) -> str: - """Map ODCS server type to FLUID provider.""" - mapping = { - "bigquery": "gcp", - "snowflake": "snowflake", - "s3": "aws", - "redshift": "aws", - "athena": "aws", - "azure": "azure", - "databricks": "databricks", - "postgres": "postgres", - "mysql": "mysql", - "kafka": "kafka", - "mongodb": "mongodb", - "elasticsearch": "elasticsearch", - "local": "local", - } - - return mapping.get(server_type.lower(), "custom") + return server_type_to_provider(server_type) def _extract_location_from_server(self, server: Mapping[str, Any]) -> Dict[str, Any]: - """Extract location details from ODCS server.""" - location = {} - - # Copy relevant fields - for key in [ - "project", - "dataset", - "table", - "account", - "database", - "schema", - "bucket", - "path", - "region", - "host", - "port", - ]: - if key in server: - location[key] = server[key] + return _servers_mapper._location_from_server(server) - return location + # --- expose scoping (legacy name) --------------------------------- + def _filter_to_expose(self, fluid: Mapping[str, Any], expose_id: str) -> Dict[str, Any]: + return self._scope_to_expose(fluid, expose_id) + # --- validation ----------------------------------------------------- def _validate_odcs(self, odcs: Mapping[str, Any]) -> None: - """ - Validate ODCS contract against JSON Schema. - - Args: - odcs: ODCS contract to validate - - Raises: - ProviderError: If validation fails - """ - if not self.schema: - self.logger.warning("ODCS schema not available, skipping validation") - return - - try: - import jsonschema - - jsonschema.validate(instance=odcs, schema=self.schema) - self.logger.info("ODCS contract validated successfully") - except ImportError: - self.logger.warning("jsonschema not installed, skipping validation") - except jsonschema.ValidationError as e: - raise ProviderError(f"ODCS validation failed: {e.message}") - - def _read_input(self, path: Union[str, Path]) -> Dict[str, Any]: - """Read ODCS contract from file.""" - input_path = Path(path) if not isinstance(path, Path) else path - - with open(input_path) as f: - if input_path.suffix in (".yaml", ".yml"): - import yaml - - return yaml.safe_load(f) - else: # JSON - return json.load(f) - - def _write_output(self, data: Dict[str, Any], path: Union[Path, str], fmt: str) -> None: - """Write ODCS contract to file.""" - output_path = Path(path) if not isinstance(path, Path) else path - output_path.parent.mkdir(parents=True, exist_ok=True) + # Version-selected from the document's own ``apiVersion`` (a v3.2.0 + # contract is judged by the v3.2.0 schema), like ``validate_contract``. + _validate(odcs) - with open(output_path, "w") as f: - if fmt == "yaml": - import yaml - yaml.dump(data, f, default_flow_style=False, sort_keys=False) - else: # json - json.dump(data, f, indent=2) +__all__ = ["OdcsProvider"] diff --git a/fluid_build/providers/odcs/provider.py b/fluid_build/providers/odcs/provider.py new file mode 100644 index 00000000..880b1021 --- /dev/null +++ b/fluid_build/providers/odcs/provider.py @@ -0,0 +1,479 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +"""OdcsProvider — thin orchestrator over the :mod:`mappers` pipeline.""" + +from __future__ import annotations + +import copy +import logging +import os +from collections.abc import Iterable, Mapping, Sequence +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple, Union + +from fluid_build.providers.base import ApplyResult, BaseProvider, ProviderError + +from .io import read_input, write_output +from .mappers import EXPORT_PIPELINE, IMPORT_PIPELINE +from .mappers.base import ExportCtx, ImportCtx, fluid_id, get_expose_passthrough, get_stash +from .validation import ( + DEFAULT_API_VERSION, + SUPPORTED_API_VERSIONS, + load_schema, + normalize_api_version, + roundtrip_check, + validate, + validate_via_vowl, +) + + +class OdcsProvider(BaseProvider): + """ODCS bidirectional provider (vendored schemas: v3.2.0, v3.1.0). + + Public surface: + - :meth:`render` — FLUID → ODCS (one expose or whole contract) + - :meth:`render_all_ports` — one ODCS per FLUID expose + - :meth:`import_contract` — ODCS → FLUID (lossless when paired with export) + - :meth:`validate_contract` — JSON Schema check, version-selected from ``apiVersion`` + - :meth:`roundtrip_check` — diff helper for round-trip tests + - :meth:`to_model` — optional typed view (``open-data-contract-standard``) + + ``api_version`` (or ``ODCS_API_VERSION``) sets the version *fresh* exports + declare. Re-exports of imported contracts preserve the document's own + ``apiVersion`` unless an explicit version is given. + + All real work lives in :mod:`fluid_build.providers.odcs.mappers`. + """ + + def __init__(self, api_version: Optional[str] = None) -> None: + super().__init__() + self.logger = logging.getLogger(f"{__name__}.{self.__class__.__name__}") + explicit = api_version or os.getenv("ODCS_API_VERSION") or None + self._explicit_api_version = normalize_api_version(explicit) if explicit else None + self.odcs_version = self._explicit_api_version or DEFAULT_API_VERSION + self.supported_versions = SUPPORTED_API_VERSIONS + self.odcs_spec_url = "https://github.com/bitol-io/open-data-contract-standard" + self.schema = load_schema(self.odcs_version) + self.include_quality_checks = os.getenv("ODCS_INCLUDE_QUALITY", "true").lower() == "true" + self.include_sla = os.getenv("ODCS_INCLUDE_SLA", "true").lower() == "true" + # Whether ``render()`` runs vowl's parser as a second-pass validator + # on the emitted ODCS. Off by default so the vowl dependency stays + # opt-in; flipped on by ``BitolOdpsProvider`` when ``strict_validation`` + # is set so the per-port ODCS contracts go through vowl too. + self._vowl_validate_on_export = os.getenv("ODCS_VOWL_VALIDATE", "false").lower() == "true" + + @property + def name(self) -> str: + return "odcs" + + def capabilities(self) -> Mapping[str, bool]: + caps = dict(super().capabilities()) + caps.update( + { + "planning": False, + "apply": False, + "render": True, + "validate": True, + "import": True, + "supports_batch": False, + } + ) + return caps + + # ---- unsupported lifecycle hooks ------------------------------------ + + def plan(self, contract: Mapping[str, Any]) -> List[Dict[str, Any]]: + raise ProviderError( + "ODCS provider does not support plan(). Use render() for export or " + "import_contract() for ODCS → FLUID." + ) + + def apply(self, actions: Iterable[Mapping[str, Any]]) -> ApplyResult: + raise ProviderError( + "ODCS provider does not support apply(). Use render() for export or " + "import_contract() for ODCS → FLUID." + ) + + # ---- export --------------------------------------------------------- + + def render( + self, + src: Union[Mapping[str, Any], Sequence[Mapping[str, Any]]], + *, + out: Optional[Union[Path, str]] = None, + fmt: Optional[str] = "yaml", + expose_id: Optional[str] = None, + contract_id: Optional[str] = None, + ) -> Dict[str, Any]: + """Render a FLUID contract to ODCS. + + When ``expose_id`` is provided the rendered ODCS scopes its ``id`` to + ``contract_id`` (default ``{product_id}.{expose_id}``) and reads + ``status`` from that expose's lifecycle — exactly what the Bitol ODPS + provider needs when emitting per-port contracts. ``contract_id`` lets + the caller keep an imported port's real ``contractId`` (a UUID, say) + as the emitted ODCS ``id`` instead of minting the convention id. + + An expose imported from an ODPS bundle carries its per-port ODCS + document state (``expose.odcs_passthrough._document``); the scoped + render then starts from that document so the re-emitted contract + reproduces the source (servers, team, SLA, apiVersion, top-level + fields) and only the expose's own edits are overlaid. + """ + if isinstance(src, list): + raise ProviderError( + "ODCS export does not support batch processing. " + "Each contract should be exported separately." + ) + + fluid = self._scope_to_expose(src, expose_id, contract_id=contract_id) if expose_id else src + odcs: Dict[str, Any] = {} + ctx = ExportCtx( + fluid=fluid, + odcs=odcs, + logger=self.logger, + options={ + "include_sla": self.include_sla, + "include_quality_checks": self.include_quality_checks, + "expose_id": expose_id, + "api_version": self._explicit_api_version, + }, + ) + for mapper in EXPORT_PIPELINE: + mapper.to_odcs(ctx) + + if os.getenv("ODCS_VALIDATE", "false").lower() == "true": + # Version-selected from the emitted apiVersion (may be a preserved + # older version for re-exports). + validate(odcs) + + # Second-pass validation through ``vowl`` (optional). Off by default + # so the dependency is opt-in; flip ``ODCS_VOWL_VALIDATE=true`` (or + # pass ``strict_validation=True`` callers that compose providers like + # ``BitolOdpsProvider``) to run vowl's parser on the way out. + if ( + self._vowl_validate_on_export + or os.getenv("ODCS_VOWL_VALIDATE", "false").lower() == "true" + ): + diag = validate_via_vowl(odcs) + if diag is not None: + self.logger.info( + "vowl: ODCS v%s parsed; %d checks across %d schema(s)", + diag["api_version"], + diag["total_checks"], + len(diag["schemas"]), + ) + + if out is not None and out != "-": + write_output(odcs, out, fmt or "yaml") + self.logger.info("Exported ODCS contract: %s", out) + return odcs + + def render_all_ports( + self, + fluid: Mapping[str, Any], + *, + out_dir: Optional[Union[Path, str]] = None, + fmt: Optional[str] = "yaml", + ) -> List[Tuple[str, Dict[str, Any]]]: + """One ODCS contract per FLUID expose. Files named ``product.odcs..``.""" + results: List[Tuple[str, Dict[str, Any]]] = [] + for expose in fluid.get("exposes", []) or []: + if not isinstance(expose, Mapping): + continue + eid = expose.get("exposeId") or expose.get("id") + if not eid: + self.logger.warning("Expose missing exposeId — skipping") + continue + out_path = None + if out_dir is not None: + out_path = Path(out_dir) / f"product.odcs.{eid}.{fmt}" + results.append((eid, self.render(fluid, out=out_path, fmt=fmt, expose_id=eid))) + return results + + # ---- import --------------------------------------------------------- + + def import_contract( + self, + odcs: Union[Mapping[str, Any], str, Path], + *, + lenient: bool = False, + ) -> Dict[str, Any]: + """ODCS (file or dict) → FLUID. + + The document is validated against the vendored JSON Schema matching + its ``apiVersion`` first; a schema-invalid contract raises + :class:`ProviderError` (the historical behaviour). ``lenient=True`` + downgrades that failure to a warning so partially-malformed + contracts can still be inspected. + """ + odcs_data = read_input(odcs) if isinstance(odcs, (str, Path)) else dict(odcs) + try: + self.validate_contract(odcs_data) + except ProviderError as exc: + if not lenient: + raise + self.logger.warning("ODCS import validation warning: %s", exc) + + fluid: Dict[str, Any] = {} + ctx = ImportCtx(odcs=odcs_data, fluid=fluid, logger=self.logger) + for mapper in IMPORT_PIPELINE: + mapper.to_fluid(ctx) + return fluid + + # ---- validation / round-trip --------------------------------------- + + def validate_contract(self, odcs: Mapping[str, Any]) -> None: + """Validate against the vendored JSON Schema matching ``odcs["apiVersion"]``. + + A contract declaring ``v3.1.0`` is judged by the v3.1.0 schema, one + declaring ``v3.2.0`` by v3.2.0; older-than-vendored and unknown + versions fall back to the newest vendored schema. + """ + validate(odcs) + + def to_model(self, odcs: Mapping[str, Any]) -> Any: + """Typed view via the optional ``open-data-contract-standard`` package. + + Returns an ``OpenDataContractStandard`` pydantic model — handy for + callers that want attribute access and readable field-level + validation errors. Requires the ``odcs-strict`` extra (Python 3.10+: + ``open-data-contract-standard`` does not support 3.9); raises + :class:`ProviderError` if the package is not installed. + """ + try: + from open_data_contract_standard.model import ( # type: ignore[import-not-found] + OpenDataContractStandard, + ) + except ImportError as exc: + raise ProviderError( + "open-data-contract-standard is not installed (it requires " + "Python >= 3.10); pip install 'fluid-forge[odcs-strict]'" + ) from exc + return OpenDataContractStandard.model_validate(dict(odcs)) + + def roundtrip_check(self, odcs: Mapping[str, Any]) -> Dict[str, Any]: + """Convert ODCS → FLUID → ODCS and return a structured diff. + + Returns ``{"equal": bool, "missing": [...], "extra": [...], + "changed": [...]}``. Pure; no I/O. + """ + fluid = self.import_contract(odcs, lenient=True) + reconstructed = self.render(fluid) + return roundtrip_check(odcs, reconstructed) + + # ---- internal helpers ---------------------------------------------- + + def _scope_to_expose( + self, + fluid: Mapping[str, Any], + expose_id: str, + contract_id: Optional[str] = None, + ) -> Dict[str, Any]: + """Filter a FLUID contract to a single output port for per-port export. + + ``contract_id`` (when given) becomes the scoped ODCS ``id``; otherwise + the ``{product_id}.{expose_id}`` convention is minted. + """ + exposes = [ + e + for e in fluid.get("exposes", []) + if isinstance(e, Mapping) + and (e.get("exposeId") == expose_id or e.get("id") == expose_id) + ] + if not exposes: + available = [ + e.get("exposeId") or e.get("id") + for e in fluid.get("exposes", []) + if isinstance(e, Mapping) + ] + raise ProviderError( + f"Expose '{expose_id}' not found in contract. " f"Available exposeIds: {available}" + ) + document = get_expose_passthrough(exposes[0]).get(DOCUMENT_KEY) + if isinstance(document, Mapping): + scoped = _port_document_view(document, exposes[0], fluid) + else: + scoped = copy.deepcopy(dict(fluid)) + scoped["exposes"] = [copy.deepcopy(dict(e)) for e in exposes] + lifecycle = exposes[0].get("lifecycle") or {} + scoped["_scoped_status"] = lifecycle.get("state", "active") + product_id = fluid_id(fluid) + if contract_id: + scoped["_scoped_id"] = contract_id + elif product_id: + scoped["_scoped_id"] = f"{product_id}.{expose_id}" + return scoped + + +# Key (inside ``expose.odcs_passthrough``) holding the per-port ODCS document +# state captured when an ODPS bundle is imported; see +# :meth:`BitolOdpsProvider._merge_odcs_into_expose`. +DOCUMENT_KEY = "_document" +# Keys (inside a ``_document`` snapshot) holding the importing product's +# FLUID ``owner`` / ``expects`` at import time; the per-port view merges the +# product's edits since then into the document's own. +PRODUCT_OWNER_KEY = "product_owner" +PRODUCT_EXPECTS_KEY = "product_expects" + +# FLUID ``expects`` keys the ODCS servers mapper reads. +EXPECT_SERVER_KEYS = ("provider", "location", "binding") + + +def _port_document_view( + document: Mapping[str, Any], + expose: Mapping[str, Any], + fluid: Optional[Mapping[str, Any]] = None, +) -> Dict[str, Any]: + """Rebuild the FLUID view a per-port ODCS document was imported from. + + ``document`` holds the sub-import's document-level FLUID (``metadata`` + with its ``odcs_passthrough`` stash of top-level fields, servers, team, + SLA and quality; ``owner``; ``contract``), any extra SchemaObjects, and + the expose's FLUID-native ``name``/``description`` at import time. The + current ``expose`` supplies the schema (and every edit made since). + Because the expose is shared with the ODPS port, its native name and + description are mapped back to the SchemaObject's own values unless the + user changed them after import. + + The product (``fluid``) owns ``owner`` and ``expects``: edits made to + them since import (against the snapshot's ``product_owner`` / + ``product_expects``) are merged into the document's own owner and + expects, so they reach the per-port ``team`` and ``servers``. An + unedited product leaves the document untouched. + """ + doc = copy.deepcopy(dict(document)) + extra = doc.pop("extra_exposes", None) or [] + imported = doc.pop("expose_native", None) or {} + has_product_state = PRODUCT_OWNER_KEY in doc or PRODUCT_EXPECTS_KEY in doc + owner_then = doc.pop(PRODUCT_OWNER_KEY, None) + expects_then = doc.pop(PRODUCT_EXPECTS_KEY, None) + if has_product_state and fluid is not None: + _merge_product_owner(doc, owner_then, product_owner(fluid)) + _merge_product_expects(doc, expects_then or [], fluid.get("expects") or []) + view_expose = copy.deepcopy(dict(expose)) + st = get_stash(get_expose_passthrough(view_expose)) + if st is not None: + name_now = view_expose.get("exposeId") or view_expose.get("id") + if name_now == imported.get("name") and st.get("name"): + view_expose["id"] = view_expose["exposeId"] = st["name"] + if (view_expose.get("description") or None) == (imported.get("description") or None): + if st.get("description"): + view_expose["description"] = st["description"] + else: + view_expose.pop("description", None) + if "binding" in imported and view_expose.get("binding") == imported.get("binding"): + # Unedited since import: hand the servers mapper the binding the ODCS + # importer derived, so it sees no binding edit and replays ``servers``. + odcs_binding = imported.get("odcs_binding") + if isinstance(odcs_binding, Mapping): + view_expose["binding"] = copy.deepcopy(dict(odcs_binding)) + else: + view_expose.pop("binding", None) + doc["exposes"] = [view_expose] + [copy.deepcopy(e) for e in extra if isinstance(e, Mapping)] + doc.setdefault("expects", []) + lifecycle = view_expose.get("lifecycle") + if isinstance(lifecycle, Mapping) and lifecycle.get("state"): + doc["_scoped_status"] = lifecycle["state"] + else: + doc["_scoped_status"] = (doc.get("metadata") or {}).get("status", "active") + return doc + + +def product_owner(fluid: Mapping[str, Any]) -> Any: + """A FLUID contract's ``owner`` (root, else ``metadata.owner``), as the + ODCS team mapper reads it.""" + metadata = fluid.get("metadata") + nested = metadata.get("owner") if isinstance(metadata, Mapping) else None + return fluid.get("owner") or nested or None + + +def _merge_product_owner(doc: Dict[str, Any], then: Any, now: Any) -> None: + """Overlay the product's owner edit (``now`` − ``then``) on ``doc["owner"]``. + + A per-port contract may have its own team, distinct from the product's. + A changed key is applied only where the port inherited the product's + import-time value for it; a port whose owner never matched the product + owner keeps its own identity. + """ + if now == then: + return + current = doc.get("owner") + if not isinstance(now, Mapping) or not isinstance(then, (Mapping, type(None))): + # Whole-owner replacement/removal: only for a port that carried the + # product's owner (or none at all). + if current is not None and current != then: + return + if now is None: + doc.pop("owner", None) + else: + doc["owner"] = copy.deepcopy(now) + return + then_map: Mapping[str, Any] = then or {} + owner: Dict[str, Any] = copy.deepcopy(dict(current)) if isinstance(current, Mapping) else {} + for key in set(then_map) | set(now): + if then_map.get(key) == now.get(key): + continue + if owner.get(key) != then_map.get(key): + continue # the port's own value, not inherited from the product + if now.get(key) is None: + owner.pop(key, None) + else: + owner[key] = copy.deepcopy(now[key]) + if owner: + doc["owner"] = owner + else: + doc.pop("owner", None) + + +def _merge_product_expects(doc: Dict[str, Any], then: List[Any], now: List[Any]) -> None: + """Merge the product's expects edits since import into ``doc["expects"]``. + + Expects pair up by ``id``. A product expect whose server-relevant keys + (provider / location / binding) changed updates the document's expect of + the same id; a removed one removes it; a new one is added. Untouched + expects leave the document (and so its verbatim ``servers``) as it was. + """ + + def by_id(items: List[Any]) -> Dict[str, Mapping[str, Any]]: + out: Dict[str, Mapping[str, Any]] = {} + for item in items: + if isinstance(item, Mapping): + out.setdefault(str(item.get("id")), item) + return out + + then_by, now_by = by_id(then), by_id(now) + expects = [dict(e) for e in doc.get("expects") or [] if isinstance(e, Mapping)] + changed = False + for eid, then_e in then_by.items(): + now_e = now_by.get(eid) + if now_e is None: + kept = [e for e in expects if str(e.get("id")) != eid] + changed = changed or len(kept) != len(expects) + expects = kept + continue + delta = [k for k in EXPECT_SERVER_KEYS if then_e.get(k) != now_e.get(k)] + if not delta: + continue + for expect in expects: + if str(expect.get("id")) != eid: + continue + changed = True + for key in delta: + if now_e.get(key) is None: + expect.pop(key, None) + else: + expect[key] = copy.deepcopy(now_e[key]) + for eid, now_e in now_by.items(): + if eid in then_by or any(str(e.get("id")) == eid for e in expects): + continue + added = {k: copy.deepcopy(now_e[k]) for k in ("id",) + EXPECT_SERVER_KEYS if k in now_e} + expects.append(added) + changed = True + if changed: + doc["expects"] = expects diff --git a/fluid_build/providers/odcs/validation.py b/fluid_build/providers/odcs/validation.py new file mode 100644 index 00000000..754fb429 --- /dev/null +++ b/fluid_build/providers/odcs/validation.py @@ -0,0 +1,291 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +"""ODCS JSON Schema validation (version-aware) + round-trip diff utility. + +Two ODCS schema versions are vendored side by side: + +- ``v3.2.0`` — current upstream. Additive over v3.1.0: optional ``context``, + ``synonyms``, ``enum``, ``semanticType``, ``deprecated``, per-element + ``id``; ``status`` no longer required; 10 new ``Server.type`` engines. +- ``v3.1.0`` — previous. Kept so a contract that *declares* ``v3.1.0`` is + validated against exactly the spec it claims, not a relaxed superset. + +:func:`validate` picks the schema from the document's own ``apiVersion`` +(see :func:`resolve_schema_version`). The emit default stays at +:data:`DEFAULT_API_VERSION` so existing downstream validators keep +accepting what we produce; opt into the newer version with the +``ODCS_API_VERSION`` environment variable or ``OdcsProvider(api_version=...)``. +""" + +from __future__ import annotations + +import datetime +import json +import logging +from collections.abc import Mapping +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple + +from fluid_build.providers.base import ProviderError + +LOG = logging.getLogger(__name__) + +# Newest first. The first entry is the superset fallback for any document +# whose apiVersion we don't vendor exactly (older-than-vendored versions are +# in every newer schema's ``apiVersion`` enum, and each release has only +# relaxed/added — never tightened — so the newest schema accepts them). +SUPPORTED_API_VERSIONS: Tuple[str, ...] = ("v3.2.0", "v3.1.0") + +# Conservative emit default. Bumping this changes what every export declares +# and would break downstream validators pinned to the older schema. +DEFAULT_API_VERSION = "v3.1.0" + +_SCHEMA_FILES: Dict[str, str] = {v: f"odcs-schema-{v}.json" for v in SUPPORTED_API_VERSIONS} +_SCHEMA_CACHE: Dict[str, Optional[Dict[str, Any]]] = {} + + +def normalize_api_version(version: Optional[str]) -> Optional[str]: + """Accept ``3.2.0`` / ``v3.2.0`` / ``V3.2.0`` and return ``v3.2.0``.""" + if version is None: + return None + v = str(version).strip() + if not v: + return None + if v[0] in ("v", "V"): + v = v[1:] + return f"v{v}" + + +def api_version_tuple(version: Optional[str]) -> Tuple[int, ...]: + """``v3.2.0`` → ``(3, 2, 0)`` for ordered comparison (``()`` if unparsable).""" + v = normalize_api_version(version) + if not v: + return () + try: + return tuple(int(p) for p in v[1:].split(".")) + except ValueError: + return () + + +def resolve_schema_version(doc: Mapping[str, Any]) -> str: + """Pick the vendored schema version to validate ``doc`` against. + + - Declared ``apiVersion`` that we vendor → that exact version. + - Anything else (missing, older-than-vendored, unknown-future) → newest + vendored, which is the acceptance superset. Unknown-future versions + still fail on the ``apiVersion`` enum, with the clearest error. + """ + declared = normalize_api_version(doc.get("apiVersion") if isinstance(doc, Mapping) else None) + if declared in _SCHEMA_FILES: + return declared + return SUPPORTED_API_VERSIONS[0] + + +def load_schema(api_version: Optional[str] = None) -> Optional[Dict[str, Any]]: + """Load a vendored ODCS JSON Schema. Returns None if missing. + + ``api_version`` defaults to :data:`DEFAULT_API_VERSION`. Unknown versions + raise :class:`ProviderError` so a typo in ``ODCS_API_VERSION`` fails loud. + """ + version = normalize_api_version(api_version) or DEFAULT_API_VERSION + if version not in _SCHEMA_FILES: + raise ProviderError( + f"Unsupported ODCS apiVersion {version!r}; " + f"vendored: {', '.join(SUPPORTED_API_VERSIONS)}" + ) + if version in _SCHEMA_CACHE: + return _SCHEMA_CACHE[version] + + schema_path = Path(__file__).parent / _SCHEMA_FILES[version] + schema: Optional[Dict[str, Any]] = None + if not schema_path.exists(): + LOG.warning("ODCS schema not found: %s", schema_path) + else: + try: + with open(schema_path) as f: + schema = json.load(f) + except Exception as exc: # pragma: no cover - defensive + LOG.error("Failed to load ODCS schema %s: %s", schema_path, exc) + _SCHEMA_CACHE[version] = schema + return schema + + +def json_instance(doc: Any) -> Any: + """``doc`` as JSON would carry it, for JSON-Schema validation. + + Bitol schemas are JSON Schemas, but contracts are usually YAML, where an + unquoted ``2024-01-01`` loads as a :class:`datetime.date`. JSON has no + date type, so such values are rendered as ISO-8601 strings (what the + author wrote) instead of failing every ``type: string`` check. Other + values are returned unchanged; the input is never mutated. + + Containers shared by reference in ``doc`` (YAML aliases load that way) + are converted once and stay shared in the result, so the conversion + never multiplies memory. + """ + return _json_instance(doc, {}) + + +def _json_instance(doc: Any, memo: Dict[int, Any]) -> Any: + if isinstance(doc, (datetime.datetime, datetime.date, datetime.time)): + return doc.isoformat() + if isinstance(doc, (Mapping, list, tuple)): + key = id(doc) + if key in memo: + return memo[key] + if isinstance(doc, Mapping): + out_map: Dict[Any, Any] = {} + memo[key] = out_map + for k, v in doc.items(): + out_map[k] = _json_instance(v, memo) + return out_map + out_list: List[Any] = [] + memo[key] = out_list + out_list.extend(_json_instance(v, memo) for v in doc) + return out_list + return doc + + +def validate(odcs: Mapping[str, Any], schema: Optional[Mapping[str, Any]] = None) -> None: + """Validate an ODCS contract against the JSON Schema. + + When ``schema`` is omitted the version is resolved from the document's + ``apiVersion`` (:func:`resolve_schema_version`) — so a v3.2.0 contract + is never judged by the v3.1.0 schema and vice versa. YAML date/time + values are validated as the ISO strings JSON would carry + (:func:`json_instance`). + + Raises :class:`ProviderError` on validation failure. Silently no-ops when + the optional :mod:`jsonschema` dependency is not installed. + """ + try: + import jsonschema + except ImportError: + LOG.warning("jsonschema not installed, skipping ODCS validation") + return + if schema is None: + schema = load_schema(resolve_schema_version(odcs)) + if schema is None: + LOG.warning("ODCS schema not available, skipping validation") + return + try: + jsonschema.validate(instance=json_instance(odcs), schema=schema) + except jsonschema.ValidationError as exc: + raise ProviderError(f"ODCS validation failed: {exc.message}") from exc + + +def validate_via_vowl(odcs: Mapping[str, Any]) -> Optional[Dict[str, Any]]: + """Second-pass ODCS validation via the ``vowl`` library (optional). + + Runs on the way out of every export to give us an independent, native + ODCS-tooling sanity check that the emitted contract is parseable by a + real ODCS consumer — not just spec-shape-valid against the vendored + JSON Schema. ``vowl`` is an official Bitol ODCS vendor (UK Government + Digital Service) and shipped as an MIT-licensed library, so importing + it here doesn't bring forge-cli into competition with another data- + contract toolchain — it's pure standards-body validation. + + Behaviour: + - Soft-imports ``vowl``. If absent, returns ``None`` so callers can + skip the second pass cleanly. The first-pass ``jsonschema`` + validation still ran. + - Builds ``vowl.Contract(odcs_dict)`` which runs vowl's own + jsonschema validation against the version-appropriate ODCS schema. + If the installed vowl predates the contract's ``apiVersion`` it + raises; we surface that as a :class:`ProviderError` with a + ``vowl:`` prefix rather than masking it. + - Enumerates the resolved check references so any lazy parse errors + surface here too. + - Returns a small diagnostic dict ``{api_version, schemas, total_checks}`` + the export path can log/expose. + """ + try: + from vowl import Contract # type: ignore[import-untyped] + except ImportError: + LOG.debug("vowl not installed, skipping second-pass ODCS validation") + return None + + try: + contract = Contract(dict(odcs)) + refs = contract.get_check_references_by_schema() + except ValueError as exc: + # vowl raises ValueError("Unsupported API version: vX.Y.Z ...") when + # the installed release predates the contract's apiVersion (vowl 0.0.5 + # tops out at v3.1.0; 0.0.6 adds v3.2.0). That is a tooling gap, not a + # contract defect — skip the second pass rather than fail the export. + if "Unsupported API version" in str(exc): + LOG.warning( + "vowl second-pass skipped — installed vowl does not support " "apiVersion %r (%s)", + odcs.get("apiVersion"), + exc, + ) + return None + raise ProviderError(f"vowl: {type(exc).__name__}: {exc}") from exc + except Exception as exc: + raise ProviderError(f"vowl: {type(exc).__name__}: {exc}") from exc + + return { + "api_version": contract.get_api_version(), + "schemas": list(contract.get_schema_names() or []), + "total_checks": sum(len(v) for v in (refs or {}).values()), + } + + +def roundtrip_check(odcs: Mapping[str, Any], reconstructed: Mapping[str, Any]) -> Dict[str, Any]: + """Compare an original ODCS dict to one rebuilt via ``import → export``. + + Returns a structured diff:: + + { + "equal": bool, + "missing": [".path.to.lost.field", ...], + "extra": [".path.added.by.export", ...], + "changed": [{"path": ".x", "old": ..., "new": ...}, ...] + } + + Used by tests and the forge ground-truth guard. Pure; no I/O. Backed by + :mod:`deepdiff` so list-of-objects ordering, nested-dict comparison, and + type coercion are handled consistently. + """ + from deepdiff import DeepDiff + + diff = DeepDiff( + dict(odcs), + dict(reconstructed), + ignore_order=False, + verbose_level=0, + view="tree", + ) + + def _path(node: Any) -> str: + # deepdiff path strings come in like ``root['schema'][0]['name']``. + return str(node.path(output_format="list")) + + missing: List[str] = [] + extra: List[str] = [] + changed: List[Dict[str, Any]] = [] + + for node in diff.get("dictionary_item_removed", []): + missing.append(_path(node)) + for node in diff.get("iterable_item_removed", []): + missing.append(_path(node)) + for node in diff.get("dictionary_item_added", []): + extra.append(_path(node)) + for node in diff.get("iterable_item_added", []): + extra.append(_path(node)) + for node in diff.get("values_changed", []): + changed.append({"path": _path(node), "old": node.t1, "new": node.t2}) + for node in diff.get("type_changes", []): + changed.append({"path": _path(node), "old": node.t1, "new": node.t2}) + + return { + "equal": not (missing or extra or changed), + "missing": missing, + "extra": extra, + "changed": changed, + } diff --git a/fluid_build/providers/odps/odps.py b/fluid_build/providers/odps/odps.py index ed932528..e3095856 100644 --- a/fluid_build/providers/odps/odps.py +++ b/fluid_build/providers/odps/odps.py @@ -14,10 +14,15 @@ # fluid_build/providers/odps/odps.py """ -ODPS (Open Data Product Specification) Provider +ODPI v4.1 (Open Data Product Initiative) Provider — Linux Foundation spec. -High-quality implementation of ODPS export functionality for FLUID contracts. -Provides comprehensive metadata extraction, validation, and standards compliance. +Export-only. Emits a single ODPS-v4.1 JSON document per FLUID contract. + +This is the **Open Data Product Initiative v4.1** spec — a different, +unrelated standard from Bitol's Open Data Product Standard v1.0.0. +For the Bitol provider see :mod:`fluid_build.providers.odps_standard` +(class :class:`BitolOdpsProvider`); ``fluid opds --spec odpi-4.1`` selects +*this* provider, and ``--spec bitol-1.0.0`` selects the Bitol one. Official Specification: https://github.com/Open-Data-Product-Initiative/v4.1 Standard: Linux Foundation / Open Data Product Initiative @@ -89,20 +94,14 @@ def _generate_uuid_from_id(data_product_id: str) -> str: class OdpsProvider(BaseProvider): - """ - OPDS (Open Data Product Specification) exporter. - - This provider converts FLUID contracts into OPDS-compliant JSON format, - enabling integration with data catalogs, governance platforms, and - ecosystem tools that support the OPDS standard. - - Features: - - Full OPDS v1.0 compliance - - Rich metadata extraction from FLUID contracts - - Support for batch processing of multiple contracts - - Comprehensive governance and lineage information - - SLA and quality metrics extraction - - Extensible metadata preservation under x-fluid namespace + """ODPI v4.1 (Open Data Product Initiative, Linux Foundation) exporter. + + Different from :class:`BitolOdpsProvider` (Bitol Open Data Product + Standard v1.0.0). This one emits a single ODPI-v4.1 JSON document; the + Bitol provider emits an ODPS doc plus per-port ODCS contracts. + + The user-facing ``fluid opds export --spec odpi-4.1`` flag selects this + provider; ``--spec bitol-1.0.0`` (the default) selects Bitol's. """ def __init__(self): diff --git a/fluid_build/providers/odps_standard/__init__.py b/fluid_build/providers/odps_standard/__init__.py index 33d49022..20539d0c 100644 --- a/fluid_build/providers/odps_standard/__init__.py +++ b/fluid_build/providers/odps_standard/__init__.py @@ -5,23 +5,23 @@ # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# fluid_build/providers/odps_standard/__init__.py -""" -ODPS (Open Data Product Standard) Provider - Bitol.io +"""Bitol Open Data Product Standard v1.0.0 provider. -This provider exports FLUID contracts to ODPS format for data marketplace -integration, particularly with Entropy Data. +Public surface: + - :class:`BitolOdpsProvider` — the new modular implementation. + - :class:`OdpsStandardProvider` — back-compat alias (same class). -ODPS Specification: https://github.com/bitol-io/open-data-product-standard +Specification: https://github.com/bitol-io/open-data-product-standard """ +from fluid_build.providers import register_provider + from .odps import OdpsStandardProvider +from .provider import BitolOdpsProvider + +# Register under both names — the canonical name is ``odps_bitol`` per the +# Phase 5 disambiguation plan; the ``odps-standard`` legacy name still works. +register_provider("odps_bitol", BitolOdpsProvider) +register_provider("odps-standard", OdpsStandardProvider) -__all__ = ["OdpsStandardProvider"] +__all__ = ["BitolOdpsProvider", "OdpsStandardProvider"] diff --git a/fluid_build/providers/odps_standard/io.py b/fluid_build/providers/odps_standard/io.py new file mode 100644 index 00000000..79022324 --- /dev/null +++ b/fluid_build/providers/odps_standard/io.py @@ -0,0 +1,12 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +"""Filesystem helpers — reuses the ODCS provider's read/write helpers.""" + +from fluid_build.providers.odcs.io import read_input, write_output + +__all__ = ["read_input", "write_output"] diff --git a/fluid_build/providers/odps_standard/mappers/__init__.py b/fluid_build/providers/odps_standard/mappers/__init__.py new file mode 100644 index 00000000..cd061668 --- /dev/null +++ b/fluid_build/providers/odps_standard/mappers/__init__.py @@ -0,0 +1,28 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +"""Ordered mapper pipeline for the Bitol ODPS provider. + +Export order: + - ``product`` establishes apiVersion/kind/id/status/name/version/description. + - ``team`` adds the team object. + - ``ports`` emits inputPorts/outputPorts; ``contractId`` follows + ``{productId}.{portName}`` so per-port ODCS contracts emitted by the + provider's :meth:`render` line up exactly. + - ``support`` carries support/management-port pass-throughs. + +Import order matches export. The provider drives full per-port ODCS +resolution in :meth:`import_contract` after this pipeline produces the +expose/expect stubs. +""" + +from . import ports, product, support, team # noqa: F401 + +EXPORT_PIPELINE = [product, team, ports, support] +IMPORT_PIPELINE = [product, team, ports, support] + +__all__ = ["EXPORT_PIPELINE", "IMPORT_PIPELINE"] diff --git a/fluid_build/providers/odps_standard/mappers/base.py b/fluid_build/providers/odps_standard/mappers/base.py new file mode 100644 index 00000000..3f730f45 --- /dev/null +++ b/fluid_build/providers/odps_standard/mappers/base.py @@ -0,0 +1,262 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +"""Section mapper protocol + pass-through namespace for the Bitol ODPS provider. + +Mirrors the design of :mod:`fluid_build.providers.odcs.mappers.base` so the +two providers share the same modular vocabulary — including the verbatim +stash + native overlay round-trip pattern (:func:`stash`, :func:`get_stash`, +:func:`overlay_native` are re-exported from there). Per-level pass-through +namespaces: + +- contract-level → ``metadata.odps_passthrough.*`` +- per-expose → ``expose.odps_passthrough.*`` +- per-expect → ``expect.odps_passthrough.*`` +""" + +from __future__ import annotations + +import logging +from collections.abc import Mapping, MutableMapping +from dataclasses import dataclass, field +from typing import Any, Dict, Iterable, List, Optional + +from fluid_build.providers.odcs.mappers.base import ( # noqa: F401 + STASH_KEY, + get_stash, + overlay_native, + stash, +) + +PASSTHROUGH_KEY = "odps_passthrough" + + +def metadata_passthrough(fluid: MutableMapping[str, Any]) -> Dict[str, Any]: + metadata = fluid.setdefault("metadata", {}) + if not isinstance(metadata, dict): + metadata = dict(metadata) + fluid["metadata"] = metadata + return metadata.setdefault(PASSTHROUGH_KEY, {}) + + +def get_metadata_passthrough(fluid: Mapping[str, Any]) -> Mapping[str, Any]: + metadata = fluid.get("metadata") or {} + if not isinstance(metadata, Mapping): + return {} + pt = metadata.get(PASSTHROUGH_KEY) + return pt if isinstance(pt, Mapping) else {} + + +def expose_passthrough(expose: MutableMapping[str, Any]) -> Dict[str, Any]: + return expose.setdefault(PASSTHROUGH_KEY, {}) + + +def get_expose_passthrough(expose: Mapping[str, Any]) -> Mapping[str, Any]: + pt = expose.get(PASSTHROUGH_KEY) + return pt if isinstance(pt, Mapping) else {} + + +def expect_passthrough(expect: MutableMapping[str, Any]) -> Dict[str, Any]: + return expect.setdefault(PASSTHROUGH_KEY, {}) + + +def get_expect_passthrough(expect: Mapping[str, Any]) -> Mapping[str, Any]: + pt = expect.get(PASSTHROUGH_KEY) + return pt if isinstance(pt, Mapping) else {} + + +@dataclass +class ImportCtx: + odps: Mapping[str, Any] + fluid: MutableMapping[str, Any] + logger: logging.Logger + options: Dict[str, Any] = field(default_factory=dict) + + +@dataclass +class ExportCtx: + fluid: Mapping[str, Any] + odps: MutableMapping[str, Any] + logger: logging.Logger + options: Dict[str, Any] = field(default_factory=dict) + + +def fluid_id(fluid: Mapping[str, Any]) -> Optional[str]: + if "id" in fluid: + return fluid["id"] + contract = fluid.get("contract") + if isinstance(contract, Mapping) and contract.get("id"): + return contract["id"] + metadata = fluid.get("metadata") + if isinstance(metadata, Mapping) and metadata.get("id"): + return metadata["id"] + return None + + +def downgrade_v110_keys( + doc: MutableMapping[str, Any], + keys: Iterable[str], + logger: logging.Logger, + *, + where: str, +) -> None: + """Make ``doc`` (a product or port) valid for an ODPS version below v1.1.0. + + Each v1.1.0-only key in ``keys`` is moved into the v1.0.0-compatible + ``customProperties`` list as ``{property: , value: }`` (the + legacy encoding of the product ``type``) unless an entry with that + property already exists. The v1.1.0-only ``id``/``vendor`` keys of + custom-property entries are dropped. Every change is logged as a warning. + """ + moved: List[str] = [] + custom = doc.get("customProperties") + custom_list: List[Any] = list(custom) if isinstance(custom, list) else [] + present = {c.get("property") for c in custom_list if isinstance(c, Mapping)} + for key in keys: + if key not in doc: + continue + value = doc.pop(key) + moved.append(key) + if key not in present and value is not None: + custom_list.append({"property": key, "value": value}) + present.add(key) + stripped = False + cleaned: List[Any] = [] + for entry in custom_list: + if isinstance(entry, Mapping) and ("id" in entry or "vendor" in entry): + entry = {k: v for k, v in entry.items() if k not in ("id", "vendor")} + stripped = True + cleaned.append(entry) + if (moved or stripped) and (cleaned or "customProperties" in doc): + doc["customProperties"] = cleaned + if moved: + logger.warning( + "ODPS < v1.1.0 has no %s on %s; moved to customProperties", + ", ".join(moved), + where, + ) + if stripped: + logger.warning( + "ODPS < v1.1.0 custom properties have no id/vendor; dropped them on %s", where + ) + + +def conform_to_schema( + doc: MutableMapping[str, Any], + schema: Mapping[str, Any], + logger: logging.Logger, +) -> None: + """Make ``doc`` fit ``schema`` (an ODPS JSON Schema) by removing what it + does not allow — used to render a product below v1.1.0. + + What is allowed is read from the schema itself, not from hand lists, so + every nested object is covered (ports, management ports, SBOM entries, + input contracts, support channels, team members, custom properties, …): + + - a key an object's schema does not list (``additionalProperties: + false``) is moved into that object's ``customProperties`` as + ``{property: , value: }`` when the object allows + ``customProperties`` (and has no entry of that name yet), else it is + dropped; + - an array entry still missing a key its schema requires (an SBOM entry + without ``url``, say) is dropped. + + Every change is logged as a warning. A document that already fits is + left untouched. + """ + _conform(doc, schema, schema, "", logger) + + +def _resolve_ref(node: Any, root: Mapping[str, Any]) -> Mapping[str, Any]: + seen = 0 + while isinstance(node, Mapping) and isinstance(node.get("$ref"), str) and seen < 32: + ref = node["$ref"] + if not ref.startswith("#/"): + return {} + target: Any = root + for part in ref[2:].split("/"): + target = target.get(part) if isinstance(target, Mapping) else None + node = target + seen += 1 + return node if isinstance(node, Mapping) else {} + + +def _conform(value: Any, node: Any, root: Mapping[str, Any], where: str, logger) -> None: + node = _resolve_ref(node, root) + if isinstance(value, MutableMapping): + props = node.get("properties") + if not isinstance(props, Mapping): + return + if node.get("additionalProperties") is False: + _strip_unknown_keys(value, props, where or "the product", logger) + for key, sub in list(value.items()): + if key in props: + _conform(sub, props[key], root, f"{where}.{key}" if where else key, logger) + elif isinstance(value, list): + items = _resolve_ref(node.get("items"), root) + if not items: + return + required = [r for r in items.get("required") or [] if isinstance(r, str)] + kept: List[Any] = [] + for index, item in enumerate(value): + item_where = f"{where}[{index}]" + _conform(item, items, root, item_where, logger) + missing = [r for r in required if isinstance(item, Mapping) and r not in item] + if missing: + logger.warning( + "ODPS < v1.1.0 requires %s on %s; dropped that entry", + ", ".join(missing), + item_where, + ) + continue + kept.append(item) + if len(kept) != len(value): + value[:] = kept + + +def _strip_unknown_keys( + obj: MutableMapping[str, Any], + props: Mapping[str, Any], + where: str, + logger: logging.Logger, +) -> None: + unknown = [k for k in obj if k not in props] + if not unknown: + return + moved: List[str] = [] + dropped: List[str] = [] + custom: Optional[List[Any]] = None + if "customProperties" in props: + existing = obj.get("customProperties") + custom = list(existing) if isinstance(existing, list) else [] + present = {c.get("property") for c in custom or [] if isinstance(c, Mapping)} + for key in unknown: + value = obj.pop(key) + if custom is not None and key not in present and value is not None: + custom.append({"property": key, "value": value}) + present.add(key) + moved.append(key) + else: + dropped.append(key) + if moved and custom is not None: + obj["customProperties"] = custom + if moved: + logger.warning( + "ODPS < v1.1.0 has no %s on %s; moved to customProperties", ", ".join(moved), where + ) + if dropped: + logger.warning("ODPS < v1.1.0 has no %s on %s; dropped", ", ".join(dropped), where) + + +def contract_id_for_port(product_id: str, port_name: str) -> str: + """Convention used by export + datamesh-manager: ``{productId}.{portName}``. + + The OdcsProvider's per-port render guarantees the emitted ODCS contract's + ``id`` matches this string exactly, which is the linking invariant for + Bitol ODPS fragments mode. + """ + return f"{product_id}.{port_name}" diff --git a/fluid_build/providers/odps_standard/mappers/ports.py b/fluid_build/providers/odps_standard/mappers/ports.py new file mode 100644 index 00000000..0e92914e --- /dev/null +++ b/fluid_build/providers/odps_standard/mappers/ports.py @@ -0,0 +1,379 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +"""Input / output port mapping. + +Each FLUID expose becomes one ``outputPorts[]`` entry; each FLUID expect +becomes one ``inputPorts[]`` entry. For *fresh* FLUID, ``contractId`` follows +the convention ``{productId}.{portName}`` (matches +:mod:`fluid_build.providers.datamesh_manager.datamesh_manager`) and equals the +``id`` of the per-port ODCS contract the provider emits as a sibling file. + +Lossless by construction: every port is stashed verbatim on import +(``.odps_passthrough._verbatim``) and used as the export base layer, so +``contractId``, ``type``, ``sbom``, ``inputContracts``, ``customProperties``, +``authoritativeDefinitions`` and the v1.1.0 ``id`` / ``deprecated`` / +``context`` / ``synonyms`` all round-trip untouched. Only ``name``, +``version``, ``description`` and ``tags`` are FLUID-native and overlaid when +edited. ``expose.odps_passthrough.contract_id`` is additionally exposed as a +convenience for callers. + +ODPS v1.1.0 relaxed ``OutputPort``/``InputPort`` ``required`` to just +``name`` — ``version`` and ``contractId`` may be absent on import and are +preserved absent on re-export. Fresh exports always emit both. + +Fresh output ports (no stash) keep the legacy ``OdpsStandardProvider`` +surface: ``expose.contract_id`` overrides the ``contractId`` convention +(set it to ``{exposeId}_contract`` to keep the id ``fluid odps-bitol export`` +minted in 0.7.8), +``ODPS_DEFAULT_PORT_VERSION`` (``ExportCtx.options["default_port_version"]``) +is the default ``version``, and ``customProperties`` carry ``server`` +(``expose.location``), ``environment``, ``containsPii``, ``autoApprove`` and +``status``. + +Ports may share a ``name`` and differ only in ``version`` (the upstream Bitol +examples do). Such ports get distinct FLUID ids on import (``name@version``), +recorded as ``odps_passthrough.fluid_id`` so re-export maps them back to the +original name. + +A product rendered at an ODPS version below v1.1.0 cannot carry the v1.1.0 +port keys (``id``/``deprecated``/``context``/``synonyms``); they are moved to +``customProperties`` (see :func:`.base.downgrade_v110_keys`). +""" + +from __future__ import annotations + +from collections import Counter +from collections.abc import Mapping +from typing import Any, Dict, Iterable, List, Optional, Set + +from ..validation import api_version_tuple +from .base import ( + ExportCtx, + ImportCtx, + contract_id_for_port, + downgrade_v110_keys, + expect_passthrough, + expose_passthrough, + fluid_id, + get_expect_passthrough, + get_expose_passthrough, + get_stash, + overlay_native, + stash, +) +from .types import provider_to_port_type + +# ----- FLUID → ODPS -------------------------------------------------------- + + +DEFAULT_PORT_VERSION = "1" + +# Port keys that only exist from ODPS v1.1.0. +_V110_PORT_KEYS = ("id", "deprecated", "context", "synonyms") + + +def to_odps(ctx: ExportCtx) -> None: + fluid = ctx.fluid + product_id = fluid_id(fluid) or "product" + default_version = str(ctx.options.get("default_port_version") or DEFAULT_PORT_VERSION) + + output_ports: List[Dict[str, Any]] = [] + for expose in fluid.get("exposes") or []: + if isinstance(expose, Mapping): + port = _expose_to_output_port( + expose, product_id, fluid=fluid, default_version=default_version + ) + if port: + output_ports.append(port) + + input_ports: List[Dict[str, Any]] = [] + for expect in fluid.get("expects") or []: + if isinstance(expect, Mapping): + port = _expect_to_input_port(expect, product_id) + if port: + input_ports.append(port) + + if not _declares_v110(ctx.odps.get("apiVersion")): + for port in output_ports: + _downgrade_port(port, ctx, default_version=default_version) + for port in input_ports: + _downgrade_port(port, ctx, default_version=default_version) + port.setdefault( + "contractId", contract_id_for_port(product_id, f"input.{port.get('name')}") + ) + + if output_ports: + ctx.odps["outputPorts"] = output_ports + if input_ports: + ctx.odps["inputPorts"] = input_ports + + +def _declares_v110(api_version: Any) -> bool: + if not api_version: + return True # product mapper not run — leave the ports untouched + return api_version_tuple(str(api_version)) >= (1, 1, 0) + + +def _downgrade_port(port: Dict[str, Any], ctx: ExportCtx, *, default_version: str) -> None: + downgrade_v110_keys(port, _V110_PORT_KEYS, ctx.logger, where=f"port {port.get('name')!r}") + # ``version`` is required on v1.0.0 ports. + port.setdefault("version", default_version) + + +def _native_output(expose: Mapping[str, Any], default_version: Any = 1) -> Dict[str, Any]: + """Total projection of an expose's FLUID-native fields to port keys.""" + tags = expose.get("tags") + return { + "name": expose.get("exposeId") or expose.get("id"), + "version": str(expose.get("version", default_version)), + "description": expose.get("description") or None, + "tags": list(tags) if tags else None, + } + + +def _expose_to_output_port( + expose: Mapping[str, Any], + product_id: str, + *, + fluid: Optional[Mapping[str, Any]] = None, + default_version: Any = DEFAULT_PORT_VERSION, +) -> Optional[Dict[str, Any]]: + name = expose.get("exposeId") or expose.get("id") + if not name: + return None + + pt = get_expose_passthrough(expose) + st = get_stash(pt) + if st is not None: + now = _native_output(expose) + then = _native_output(_project_output_port(st, fluid_name=pt.get("fluid_id"))) + return overlay_native(st, then, now) + + # ---- fresh FLUID: synthesise --------------------------------------- + now = _native_output(expose, default_version) + port: Dict[str, Any] = { + "name": name, + "version": now["version"], + # ``odps_passthrough.contract_id``: the original ODCS id of an expose + # imported from an ODCS-only directory. + "contractId": expose.get("contract_id") + or pt.get("contract_id") + or contract_id_for_port(product_id, name), + } + if now["description"]: + port["description"] = now["description"] + port_type = _port_type_from_expose(expose) + if port_type: + port["type"] = port_type + if now["tags"]: + port["tags"] = now["tags"] + port["customProperties"] = _port_custom_properties(expose, fluid or {}) + return port + + +def _port_custom_properties( + expose: Mapping[str, Any], fluid: Mapping[str, Any] +) -> List[Dict[str, Any]]: + """Legacy ``OdpsStandardProvider`` port custom properties.""" + metadata = fluid.get("metadata") if isinstance(fluid.get("metadata"), Mapping) else {} + props: List[Dict[str, Any]] = [] + location = expose.get("location") + if isinstance(location, Mapping) and location: + props.append({"property": "server", "value": dict(location)}) + environment = metadata.get("environment") + if environment: + props.append({"property": "environment", "value": environment}) + contains_pii = _contains_pii(expose) + props.append({"property": "containsPii", "value": contains_pii}) + props.append({"property": "autoApprove", "value": _auto_approve(fluid, contains_pii)}) + status = expose.get("status") or metadata.get("status") or "active" + props.append({"property": "status", "value": status}) + return props + + +def _mentions_pii(value: Any) -> bool: + text = str(value or "").lower() + return "pii" in text or "sensitive" in text + + +def _expose_fields(expose: Mapping[str, Any]) -> Iterable[Mapping[str, Any]]: + schema = expose.get("schema") + if isinstance(schema, Mapping) and isinstance(schema.get("fields"), list): + yield from (f for f in schema["fields"] if isinstance(f, Mapping)) + contract = expose.get("contract") + if isinstance(contract, Mapping) and isinstance(contract.get("schema"), list): + yield from (f for f in contract["schema"] if isinstance(f, Mapping)) + + +def _contains_pii(expose: Mapping[str, Any]) -> bool: + for fld in _expose_fields(expose): + if fld.get("pii") is True: + return True + if _mentions_pii(fld.get("classification")) or _mentions_pii(fld.get("sensitivity")): + return True + return "pii" in str(expose.get("classification") or "").lower() + + +def _auto_approve(fluid: Mapping[str, Any], contains_pii: bool) -> bool: + if contains_pii: + return False # never auto-approve access to PII + access_control = fluid.get("access_control") + if isinstance(access_control, Mapping) and access_control.get("auto_approve") is not None: + return bool(access_control["auto_approve"]) + return False # safe default + + +def _native_input(expect: Mapping[str, Any]) -> Dict[str, Any]: + tags = expect.get("tags") + return { + "name": expect.get("id"), + "version": str(expect.get("version", 1)), + "tags": list(tags) if tags else None, + } + + +def _expect_to_input_port(expect: Mapping[str, Any], product_id: str) -> Optional[Dict[str, Any]]: + name = expect.get("id") + if not name: + return None + + pt = get_expect_passthrough(expect) + st = get_stash(pt) + now = _native_input(expect) + if st is not None: + then = _native_input(_project_input_port(st, fluid_name=pt.get("fluid_id"))) + return overlay_native(st, then, now) + + # Fresh: prefer an explicit contractId, else mint one consistent with the + # product (Bitol input ports must reference *some* contract). + port: Dict[str, Any] = { + "name": name, + "version": now["version"], + "contractId": expect.get("contractId") or contract_id_for_port(product_id, f"input.{name}"), + } + if now["tags"]: + port["tags"] = now["tags"] + return port + + +def _port_type_from_expose(expose: Mapping[str, Any]) -> Optional[str]: + binding = expose.get("binding") + if isinstance(binding, Mapping): + platform = binding.get("platform") or binding.get("provider") + if platform: + return provider_to_port_type(platform) + provider = expose.get("provider") + if provider: + return provider_to_port_type(provider) + return None + + +# ----- ODPS → FLUID -------------------------------------------------------- + + +def to_fluid(ctx: ImportCtx) -> None: + """Skeleton expose/expect generation. + + The full ODCS-resolution step (which actually populates each expose's + schema/quality/qos) lives in the provider's ``import_contract`` method — + it needs the :class:`ContractResolver` instance. Here we just stub each + port with its identifying fields and stash it verbatim, so the resolver + step can fill in the rest. + """ + odps = ctx.odps + fluid = ctx.fluid + + outputs = [p for p in odps.get("outputPorts") or [] if isinstance(p, Mapping)] + exposes = fluid.setdefault("exposes", []) + for port, fluid_name in zip(outputs, _fluid_names(outputs, "output")): + exposes.append(_output_port_to_expose_stub(port, fluid_name)) + + inputs = [p for p in odps.get("inputPorts") or [] if isinstance(p, Mapping)] + expects = fluid.setdefault("expects", []) + for port, fluid_name in zip(inputs, _fluid_names(inputs, "input")): + expects.append(_input_port_to_expect_stub(port, fluid_name)) + + +def _fluid_names(ports: List[Mapping[str, Any]], default: str) -> List[Optional[str]]: + """Distinct FLUID ids for ports sharing a name (``name@version``). + + ``None`` means "use the port name as is". + """ + names = [str(p.get("name") or default) for p in ports] + counts = Counter(names) + used: Set[str] = {n for n in names if counts[n] == 1} + out: List[Optional[str]] = [] + for port, name in zip(ports, names): + if counts[name] == 1: + out.append(None) + continue + version = port.get("version") + candidate = f"{name}@{version}" if version not in (None, "") else f"{name}#1" + n = 1 + while candidate in used: + n += 1 + candidate = f"{name}#{n}" + used.add(candidate) + out.append(candidate) + return out + + +def _project_output_port( + port: Mapping[str, Any], fluid_name: Optional[str] = None +) -> Dict[str, Any]: + """Pure FLUID projection of an output port (no stash).""" + name = fluid_name or port.get("name") or "output" + stub: Dict[str, Any] = { + "id": name, + "exposeId": name, + "version": str(port.get("version", 1)), + } + if port.get("description"): + stub["description"] = port["description"] + if port.get("tags"): + stub["tags"] = list(port["tags"]) + return stub + + +def _output_port_to_expose_stub( + port: Mapping[str, Any], fluid_name: Optional[str] = None +) -> Dict[str, Any]: + stub = _project_output_port(port, fluid_name) + pt = expose_passthrough(stub) + if port.get("contractId"): + pt["contract_id"] = port["contractId"] + if fluid_name: + pt["fluid_id"] = fluid_name + stash(pt, port) + return stub + + +def _project_input_port( + port: Mapping[str, Any], fluid_name: Optional[str] = None +) -> Dict[str, Any]: + name = fluid_name or port.get("name") or "input" + stub: Dict[str, Any] = { + "id": name, + "version": str(port.get("version", 1)), + } + if port.get("contractId"): + stub["contractId"] = port["contractId"] + if port.get("tags"): + stub["tags"] = list(port["tags"]) + return stub + + +def _input_port_to_expect_stub( + port: Mapping[str, Any], fluid_name: Optional[str] = None +) -> Dict[str, Any]: + stub = _project_input_port(port, fluid_name) + pt = expect_passthrough(stub) + if fluid_name: + pt["fluid_id"] = fluid_name + stash(pt, port) + return stub diff --git a/fluid_build/providers/odps_standard/mappers/product.py b/fluid_build/providers/odps_standard/mappers/product.py new file mode 100644 index 00000000..90796fd1 --- /dev/null +++ b/fluid_build/providers/odps_standard/mappers/product.py @@ -0,0 +1,224 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +"""Product top-level fields: apiVersion, kind, id, name, version, status, +description, domain, tenant, tags, productCreatedTs, customProperties, +authoritativeDefinitions, and the v1.1.0 ``type``/``deprecated``/``context``/ +``synonyms``. + +The Bitol ODPS spec requires ``apiVersion``, ``kind``, ``id`` (``status`` was +also required before v1.1.0; we always emit it). + +Lossless by construction — the whole top-level document (minus the sections +owned by sibling mappers) is stashed verbatim on import and used as the +export base layer; see :mod:`fluid_build.providers.odcs.mappers.base`. + +``apiVersion`` on export: explicit (``ExportCtx.options["api_version"]``) > +imported document's own > :data:`DEFAULT_API_VERSION`. + +Product ``type`` (``sourceAligned`` / ``aggregate`` / ``consumerAligned`` …) +is a first-class field from v1.1.0. Fresh exports targeting ≥ v1.1.0 emit it +natively from ``metadata.product_type``; fresh v1.0.0 exports keep the legacy +``customProperties: [{property: type, value: …}]`` encoding so existing +consumers see no change. Any product rendered below v1.1.0 (including an +imported v1.1.0 product re-exported at an explicit v1.0.0) never carries the +v1.1.0-only ``type``/``deprecated``/``context``/``synonyms`` keys: they move to +``customProperties`` with a warning. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any, Dict, Optional + +from fluid_build.providers.base import ProviderError + +from ..validation import DEFAULT_API_VERSION, api_version_tuple, normalize_api_version +from .base import ( + ExportCtx, + ImportCtx, + downgrade_v110_keys, + fluid_id, + get_metadata_passthrough, + get_stash, + metadata_passthrough, + overlay_native, + stash, +) +from .types import bitol_to_fluid_status, fluid_to_bitol_status + +KIND = "DataProduct" + +# Top-level keys owned by sibling mappers — never stashed here. +SECTION_KEYS = frozenset({"outputPorts", "inputPorts", "team", "support", "managementPorts"}) + +_NATIVE_TYPE_MIN = (1, 1, 0) + +# Top-level keys that only exist from ODPS v1.1.0. A product rendered at an +# older version moves them to ``customProperties`` (the legacy encoding of +# ``type``) so the document stays valid against the version it declares. +_V110_PRODUCT_KEYS = ("type", "deprecated", "context", "synonyms") + + +# ----- ODPS → FLUID -------------------------------------------------------- + + +def _project_product(odps: Mapping[str, Any]) -> Dict[str, Any]: + """Pure projection of top-level ODPS scalars into FLUID ``metadata``.""" + metadata: Dict[str, Any] = { + "version": odps.get("version", "1.0.0"), + "name": odps.get("name", odps.get("id")), + "status": bitol_to_fluid_status(odps.get("status", "draft")), + } + description = odps.get("description") + if isinstance(description, Mapping): + if description.get("purpose"): + metadata["description"] = description["purpose"] + elif isinstance(description, str) and description: + metadata["description"] = description + for key in ("domain", "tenant"): + if odps.get(key): + metadata[key] = odps[key] + if odps.get("tags"): + metadata["tags"] = list(odps["tags"]) + if odps.get("type"): + metadata["product_type"] = odps["type"] + if "deprecated" in odps: + metadata["deprecated"] = bool(odps["deprecated"]) + return metadata + + +def to_fluid(ctx: ImportCtx) -> None: + odps = ctx.odps + fluid = ctx.fluid + + metadata = fluid.setdefault("metadata", {}) + metadata.update(_project_product(odps)) + + fluid.setdefault("contract", {})["id"] = odps.get("id") + fluid.setdefault("exposes", []) + fluid.setdefault("expects", []) + + stash( + metadata_passthrough(fluid), + {k: v for k, v in odps.items() if k not in SECTION_KEYS}, + ) + + +# ----- FLUID → ODPS -------------------------------------------------------- + + +def _product_type(fluid: Mapping[str, Any]) -> Optional[str]: + metadata = fluid.get("metadata") or {} + return ( + metadata.get("product_type") + or metadata.get("type") + or fluid.get("product_type") + or fluid.get("type") + or None + ) + + +def _native_product( + fluid: Mapping[str, Any], + metadata: Mapping[str, Any], + product_id: Optional[str], + *, + native_type: bool, +) -> Dict[str, Any]: + """Total projection of FLUID-native top-level fields to ODPS keys.""" + tags = metadata.get("tags") or fluid.get("tags") + return { + "id": product_id, + "name": metadata.get("name") or fluid.get("name") or None, + "version": str(metadata.get("version", "1.0.0")), + "status": fluid_to_bitol_status(metadata.get("status", "draft")), + "description": ( + {"purpose": metadata["description"]} if metadata.get("description") else None + ), + "domain": metadata.get("domain") or None, + "tenant": metadata.get("tenant") or None, + "tags": list(tags) if tags else None, + "type": _product_type(fluid) if native_type else None, + "deprecated": metadata.get("deprecated") if "deprecated" in metadata else None, + } + + +def _resolve_api_version(ctx: ExportCtx, st: Optional[Mapping[str, Any]]) -> str: + explicit = ctx.options.get("api_version") + if explicit: + return normalize_api_version(explicit) or DEFAULT_API_VERSION + if st and st.get("apiVersion"): + return str(st["apiVersion"]) + return DEFAULT_API_VERSION + + +def to_odps(ctx: ExportCtx) -> None: + fluid = ctx.fluid + odps = ctx.odps + metadata = fluid.get("metadata") or {} + + product_id = fluid_id(fluid) + if not product_id: + raise ProviderError("Contract missing required 'id' field for ODPS export") + if not (metadata.get("name") or fluid.get("name")): + raise ProviderError( + "Contract missing required 'name' field. " + "Set it at the contract root or under metadata.name." + ) + + st = get_stash(get_metadata_passthrough(fluid)) + api_version = _resolve_api_version(ctx, st) + native_type = api_version_tuple(api_version) >= _NATIVE_TYPE_MIN + + now = _native_product(fluid, metadata, product_id, native_type=native_type) + + if st is not None: + then_meta = _project_product(st) + then = _native_product( + {"metadata": then_meta}, then_meta, st.get("id"), native_type=native_type + ) + merged = overlay_native(st, then, now) + if ( + now["description"] != then["description"] + and now["description"] is not None + and isinstance(st.get("description"), Mapping) + ): + merged["description"] = {**st["description"], **now["description"]} + if not native_type: + downgrade_v110_keys(merged, _V110_PRODUCT_KEYS, ctx.logger, where="the product") + else: + merged = {k: v for k, v in now.items() if v is not None} + custom = [] + if not native_type: + # Legacy (< v1.1.0) encoding of the product type. + product_type = _product_type(fluid) + if product_type: + custom.append({"property": "type", "value": product_type}) + # Product-level customProperties origin/main's OdpsStandardProvider + # emitted; kept so existing consumers (e.g. DMM) see no change. + if metadata.get("domain"): + custom.append({"property": "domain", "value": metadata["domain"]}) + if metadata.get("version"): + custom.append({"property": "fluidVersion", "value": metadata["version"]}) + if custom: + merged["customProperties"] = custom + if not native_type: + downgrade_v110_keys(merged, _V110_PRODUCT_KEYS, ctx.logger, where="the product") + if ctx.options.get("include_custom_properties", True) is False: + merged.pop("customProperties", None) + + merged["apiVersion"] = api_version + merged["kind"] = KIND + + head = ("apiVersion", "kind", "id", "name", "version", "status") + for key in head: + if key in merged: + odps[key] = merged[key] + for key, value in merged.items(): + if key not in head: + odps[key] = value diff --git a/fluid_build/providers/odps_standard/mappers/support.py b/fluid_build/providers/odps_standard/mappers/support.py new file mode 100644 index 00000000..ac376e70 --- /dev/null +++ b/fluid_build/providers/odps_standard/mappers/support.py @@ -0,0 +1,44 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +"""Support channels and management ports (opaque pass-through). + +FLUID has no native model for support channels or management endpoints, so +both flow through ``metadata.odps_passthrough.{support, management_ports}`` +verbatim. +""" + +from __future__ import annotations + +import copy + +from .base import ( + ExportCtx, + ImportCtx, + get_metadata_passthrough, + metadata_passthrough, +) + +# Deep copies throughout: a stash sharing objects with the source document +# (also stashed) would make the YAML dump of the imported FLUID emit +# anchors/aliases, and a rendered document must never alias the FLUID. + + +def to_odps(ctx: ExportCtx) -> None: + pt = get_metadata_passthrough(ctx.fluid) + if "support" in pt: + ctx.odps["support"] = copy.deepcopy(list(pt["support"])) + if "management_ports" in pt: + ctx.odps["managementPorts"] = copy.deepcopy(list(pt["management_ports"])) + + +def to_fluid(ctx: ImportCtx) -> None: + pt = metadata_passthrough(ctx.fluid) + if ctx.odps.get("support"): + pt["support"] = copy.deepcopy(list(ctx.odps["support"])) + if ctx.odps.get("managementPorts"): + pt["management_ports"] = copy.deepcopy(list(ctx.odps["managementPorts"])) diff --git a/fluid_build/providers/odps_standard/mappers/team.py b/fluid_build/providers/odps_standard/mappers/team.py new file mode 100644 index 00000000..d1cad5d9 --- /dev/null +++ b/fluid_build/providers/odps_standard/mappers/team.py @@ -0,0 +1,59 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +"""Team object — Bitol ODPS schema is byte-identical to Bitol ODCS Team +(minus the optional ``id`` StableId). We delegate to the ODCS team mapper +to avoid duplicating the FLUID owner ↔ team object translation. + +Round-trip follows the ODCS team mapper: the imported team is stashed +verbatim under ``metadata.odps_passthrough.team`` and replayed unless the +FLUID ``owner`` changed since import; an edit is merged onto the stash, +keeping every key FLUID does not model. +""" + +from __future__ import annotations + +import copy +from collections.abc import Mapping + +from fluid_build.providers.odcs.mappers import team as _odcs_team + +from .base import ( + ExportCtx, + ImportCtx, + get_metadata_passthrough, + metadata_passthrough, +) + + +def to_odps(ctx: ExportCtx) -> None: + pt_team = get_metadata_passthrough(ctx.fluid).get("team") + if isinstance(pt_team, Mapping) and pt_team: + # Imported product: three-way merge (stash + FLUID owner edit). + then = _odcs_team._import_owner(pt_team) + now = _odcs_team._current_owner(ctx.fluid) + if now == then: + ctx.odps["team"] = copy.deepcopy(dict(pt_team)) + return + merged = _odcs_team._merge_team(pt_team, then or {}, ctx.fluid) + if isinstance(merged, Mapping): + ctx.odps["team"] = merged + return + team = _odcs_team._owner_to_team(ctx.fluid) + if team: + ctx.odps["team"] = team + + +def to_fluid(ctx: ImportCtx) -> None: + team = ctx.odps.get("team") + if not team: + return + owner = _odcs_team._team_to_owner(team) + if owner: + ctx.fluid["owner"] = owner + if isinstance(team, Mapping): + metadata_passthrough(ctx.fluid)["team"] = copy.deepcopy(dict(team)) diff --git a/fluid_build/providers/odps_standard/mappers/types.py b/fluid_build/providers/odps_standard/mappers/types.py new file mode 100644 index 00000000..d9b5ca97 --- /dev/null +++ b/fluid_build/providers/odps_standard/mappers/types.py @@ -0,0 +1,75 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +"""Tables shared by Bitol ODPS mappers (status mapping, provider → port type).""" + +from __future__ import annotations + +from typing import Dict + +_FLUID_TO_BITOL_STATUS: Dict[str, str] = { + "draft": "draft", + "active": "active", + "deprecated": "deprecated", + "retired": "retired", + "development": "draft", + "proposed": "proposed", +} + +_BITOL_TO_FLUID_STATUS: Dict[str, str] = { + "proposed": "draft", + "draft": "draft", + "active": "active", + "deprecated": "deprecated", + "retired": "retired", +} + + +def fluid_to_bitol_status(status: str) -> str: + return _FLUID_TO_BITOL_STATUS.get(str(status).lower(), "draft") + + +def bitol_to_fluid_status(status: str) -> str: + return _BITOL_TO_FLUID_STATUS.get(str(status).lower(), "draft") + + +# FLUID provider/platform → ODPS output port type. Bitol ODPS doesn't constrain +# this enum strictly, so we keep the same vocabulary the existing exporter used. +_PROVIDER_TO_PORT_TYPE: Dict[str, str] = { + "gcp": "bigquery", + "bigquery": "bigquery", + "snowflake": "snowflake", + "aws": "s3", + "s3": "s3", + "azure": "azure", + "databricks": "databricks", + "postgres": "postgres", + "postgresql": "postgres", + "mysql": "mysql", + "kafka": "kafka", + "local": "local", + # Engines introduced as ODCS v3.2.0 server types; ODPS port ``type`` is a + # free string so we mirror the ODCS vocabulary for consistency. + "iceberg": "iceberg", + "hana": "hana", + "sap-hana": "hana", + "saphana": "hana", + "teradata": "teradata", + "exasol": "exasol", + "ingres": "ingres", + "vectorwise": "vectorwise", + "actian": "vectorwise", + "versant": "versant", + "zen": "zen", + "btrieve": "btrieve", + "poet": "poet", + "fastobjects": "fastobjects", +} + + +def provider_to_port_type(provider: str) -> str: + return _PROVIDER_TO_PORT_TYPE.get(provider.lower(), "custom") diff --git a/fluid_build/providers/odps_standard/odps.py b/fluid_build/providers/odps_standard/odps.py index d7d8aa10..39093dff 100644 --- a/fluid_build/providers/odps_standard/odps.py +++ b/fluid_build/providers/odps_standard/odps.py @@ -5,476 +5,73 @@ # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# fluid_build/providers/odps_standard/odps.py -""" -ODPS-Bitol (Bitol.io Data Product Standard) Provider +"""Back-compat shim for the legacy ``OdpsStandardProvider``. -Converts FLUID contracts to ODPS-Bitol v1.0.0 format. -Used for data marketplace integration with Entropy Data. +The Bitol ODPS provider was split into :mod:`.provider`, :mod:`.mappers`, +:mod:`.validation`, and :mod:`.io` in the Phase 2 modular refactor. The +legacy ``OdpsStandardProvider`` class continues to work — it now subclasses +:class:`BitolOdpsProvider` and overrides the provider ``name`` so existing +plugin registrations keep functioning. -Note: This is Bitol.io's proprietary ODPS variant, distinct from the -official ODPS v4.1 (Linux Foundation / Open Data Product Initiative). +New code should import from :mod:`fluid_build.providers.odps_standard` +directly:: -Specification: https://github.com/bitol-io/open-data-product-standard -Marketplace: https://entropy-data.com + from fluid_build.providers.odps_standard import BitolOdpsProvider """ from __future__ import annotations -import json -import logging -import os -from collections.abc import Iterable, Mapping, Sequence +from collections.abc import Mapping, Sequence from pathlib import Path -from typing import Any, Dict, List, Optional, Union +from typing import Any, Dict, Optional, Union -from fluid_build.providers.base import ApplyResult, BaseProvider, ProviderError +from fluid_build.providers.base import ProviderError +from .provider import BitolOdpsProvider as _BitolOdpsProvider -class OdpsStandardProvider(BaseProvider): - """ - ODPS-Bitol (Bitol.io Data Product Standard) exporter. - Converts FLUID contracts into ODPS-Bitol v1.0.0 format for data marketplace - integration, particularly with Entropy Data. +class OdpsStandardProvider(_BitolOdpsProvider): + """Legacy class name — kept for back-compat with the original Phase-2 surface. - This is Bitol.io's variant. For the official ODPS v4.1 (Linux Foundation), - use the 'odps' provider (accessed via 'fluid opds' command). - - Specification: https://github.com/bitol-io/open-data-product-standard + Differs from :class:`BitolOdpsProvider` only in: + - ``name`` property returns ``"odps-standard"`` (legacy). + - ``odps_version`` / ``odps_spec_url`` attributes (legacy names). + - ``render`` returns the bare product dict (not the + ``{"product": ..., "contracts": ...}`` envelope) so existing CLI + callers and tests keep working unchanged. """ - def __init__(self): + def __init__(self) -> None: super().__init__() - self.logger = logging.getLogger(f"{__name__}.{self.__class__.__name__}") - - # ODPS version - self.odps_version = "v1.0.0" - self.odps_spec_url = "https://github.com/bitol-io/open-data-product-standard" - - # Configuration - self.include_custom_properties = os.getenv("ODPS_INCLUDE_CUSTOM", "true").lower() == "true" - self.default_port_version = os.getenv("ODPS_DEFAULT_PORT_VERSION", "1") + # Legacy attribute names — tests reference these directly. + self.odps_version = self.api_version + self.odps_spec_url = self.spec_url + # Legacy callers (e.g. DataMeshManagerProvider's publish path) feed + # partially-populated contracts to this back-compat alias and expect + # the render call to succeed even when the per-port ODCS misses + # spec-required server fields. New code should use BitolOdpsProvider + # directly to opt into strict validation (ODPS_STRICT=true). + self.strict_validation = False @property def name(self) -> str: return "odps-standard" - def capabilities(self) -> Mapping[str, bool]: - """Signal exporter capabilities.""" - caps = super().capabilities() - caps = dict(caps) - caps.update( - { - "planning": False, - "apply": False, - "render": True, # Primary capability - "validate": False, - "supports_batch": False, # ODPS is per-product - } - ) - return caps - - def plan(self, contract: Mapping[str, Any]) -> List[Dict[str, Any]]: - """Plan not supported for export-only provider.""" - raise ProviderError("ODPS provider does not support plan(). Use render() for export.") - - def apply(self, actions: Iterable[Mapping[str, Any]]) -> ApplyResult: - """Apply not supported - use render() instead.""" - raise ProviderError("ODPS provider does not support apply(). Use render() for export.") - - def render( + def render( # type: ignore[override] self, src: Union[Mapping[str, Any], Sequence[Mapping[str, Any]]], *, out: Optional[Union[Path, str]] = None, fmt: Optional[str] = "yaml", + out_dir: Optional[Union[Path, str]] = None, ) -> Dict[str, Any]: - """ - Export FLUID contract to ODPS format. - - Args: - src: FLUID contract dictionary - out: Output file path (optional) - fmt: Output format ('yaml' or 'json') - - Returns: - ODPS-compliant dictionary - """ - if isinstance(src, list): - raise ProviderError( - "ODPS export does not support batch processing. " - "Each data product should be exported separately." - ) - - self.logger.info("Converting FLUID contract to ODPS v1.0.0") - - # Convert to ODPS - odps_product = self._fluid_to_odps(src) - - # Write output if path provided - if out: - self._write_output(odps_product, out, fmt) - self.logger.info(f"Exported ODPS data product: {out}") - - return odps_product - - def _fluid_to_odps(self, fluid: Mapping[str, Any]) -> Dict[str, Any]: - """ - Convert FLUID contract to ODPS data product. - - Args: - fluid: FLUID contract dictionary - - Returns: - ODPS-compliant data product dictionary - """ - metadata = fluid.get("metadata", {}) - fluid.get("contract", {}) - - # Build ODPS structure - odps_product = { - "apiVersion": self.odps_version, - "kind": "DataProduct", - "id": self._extract_id(fluid), - "name": self._extract_name(fluid), - "status": self._extract_status(metadata), - } - - # Optional fields - description = self._extract_description(metadata) - if description: - odps_product["description"] = description - - team = self._extract_team(fluid) - if team: - odps_product["team"] = team - - tags = metadata.get("tags") or fluid.get("tags") or [] - if tags: - odps_product["tags"] = tags + """Render returning the bare product dict (legacy contract). - # Output ports (from exposes) - always include - output_ports = self._extract_output_ports(fluid) - odps_product["outputPorts"] = output_ports - - # Custom properties - if self.include_custom_properties: - custom_props = self._extract_custom_properties(fluid) - if custom_props: - odps_product["customProperties"] = custom_props - - return odps_product - - def _extract_id(self, fluid: Mapping[str, Any]) -> str: - """Extract data product ID.""" - # FLUID 0.7.1 uses top-level 'id' field - if "id" in fluid: - return fluid["id"] - - # Try contract.id (older format) - contract_id = fluid.get("contract", {}).get("id") - if contract_id: - return contract_id - - # Fall back to metadata.id - metadata_id = fluid.get("metadata", {}).get("id") - if metadata_id: - return metadata_id - - raise ProviderError("Contract missing required 'id' field") - - def _extract_name(self, fluid: Mapping[str, Any]) -> str: - """Extract data product name. - - Checks metadata.name first (explicit override), then falls back to - the top-level 'name' field which is the standard FLUID contract location. - """ - name = fluid.get("metadata", {}).get("name") or fluid.get("name") - if not name: - raise ProviderError( - "Contract missing required name field. " - "Set 'name:' at the contract root (or 'metadata.name' to override)." - ) - return name - - def _extract_status(self, metadata: Mapping[str, Any]) -> str: + Use :class:`BitolOdpsProvider.render` directly if you want the + product + per-port contracts bundle. """ - Extract and map status. - - FLUID -> ODPS mapping: - - draft -> draft - - active -> active - - deprecated -> deprecated - - retired -> retired - - development -> draft - """ - status = metadata.get("status", "draft") - - mapping = { - "draft": "draft", - "active": "active", - "deprecated": "deprecated", - "retired": "retired", - "development": "draft", - } - - return mapping.get(status.lower(), "draft") - - def _extract_description(self, metadata: Mapping[str, Any]) -> Optional[Dict[str, str]]: - """ - Extract description as ODPS structure. - - Returns: - {"purpose": "..."} or None - """ - description = metadata.get("description") - if not description: - return None - - return {"purpose": description} - - def _extract_team(self, fluid: Mapping[str, Any]) -> Optional[Dict[str, Any]]: - """ - Extract team information. - - ODPS team structure: - { - "name": "team-name", - "contacts": [ - {"name": "John Doe", "email": "john@company.com"} - ] - } - """ - owner = fluid.get("owner", {}) - if not owner: - return None - - team = {} - - # Team name - team_name = owner.get("team") or owner.get("name") - if team_name: - team["name"] = team_name - - # Contacts - contacts = [] - - # Primary contact from owner - if owner.get("name") or owner.get("email"): - contact = {} - if owner.get("name"): - contact["name"] = owner["name"] - if owner.get("email"): - contact["email"] = owner["email"] - contacts.append(contact) - - # Additional contacts - if "contacts" in owner: - for c in owner["contacts"]: - if isinstance(c, dict): - contacts.append(c) - - if contacts: - team["contacts"] = contacts - - return team if team else None - - def _extract_output_ports(self, fluid: Mapping[str, Any]) -> List[Dict[str, Any]]: - """ - Extract output ports from exposes section. - - Each expose becomes an output port. - """ - output_ports = [] - - for expose in fluid.get("exposes", []): - port = self._expose_to_output_port(expose, fluid) - output_ports.append(port) - - return output_ports - - def _expose_to_output_port( - self, expose: Mapping[str, Any], fluid: Mapping[str, Any] - ) -> Dict[str, Any]: - """ - Convert FLUID expose to ODPS output port. - - Args: - expose: FLUID expose dictionary - fluid: Full FLUID contract (for context) - - Returns: - ODPS output port dictionary - """ - port = { - "name": expose["id"], - "version": str(expose.get("version", self.default_port_version)), - "description": expose.get("description", ""), - } - - # Map provider to type - provider = expose.get("provider") - if not provider: - binding = expose.get("binding") - if isinstance(binding, dict): - provider = binding.get("platform") - if provider: - port["type"] = self._map_provider_to_type(provider) - - # Contract ID (reference to ODCS contract) - contract_id = expose.get("contract_id") or f"{expose['id']}_contract" - port["contractId"] = contract_id - - # Custom properties (server details, etc.) - custom_props = self._extract_port_custom_properties(expose, fluid) - if custom_props: - port["customProperties"] = custom_props - - return port - - def _map_provider_to_type(self, provider: str) -> str: - """ - Map FLUID provider to ODPS type. - - Args: - provider: FLUID provider name - - Returns: - ODPS type string - """ - mapping = { - "gcp": "bigquery", - "snowflake": "snowflake", - "aws": "s3", - "azure": "azure", - "databricks": "databricks", - "postgres": "postgres", - "postgresql": "postgres", - "mysql": "mysql", - "kafka": "kafka", - "local": "local", - } - - # Default to "custom" for unknown providers - return mapping.get(provider.lower(), "custom") - - def _extract_port_custom_properties( - self, expose: Mapping[str, Any], fluid: Mapping[str, Any] - ) -> List[Dict[str, Any]]: - """ - Extract custom properties for output port. - - Includes server details, PII flags, etc. - """ - custom_props = [] - - # Server/location details - location = expose.get("location") - if location and isinstance(location, dict): - custom_props.append({"property": "server", "value": location}) - - # Environment - environment = fluid.get("metadata", {}).get("environment") - if environment: - custom_props.append({"property": "environment", "value": environment}) - - # PII flag (from schema or metadata) - contains_pii = self._check_contains_pii(expose) - custom_props.append({"property": "containsPii", "value": contains_pii}) - - # Auto-approve flag (from access control) - auto_approve = self._check_auto_approve(expose, fluid) - custom_props.append({"property": "autoApprove", "value": auto_approve}) - - # Status - status = expose.get("status") or fluid.get("metadata", {}).get("status", "active") - custom_props.append({"property": "status", "value": status}) - - return custom_props - - def _check_contains_pii(self, expose: Mapping[str, Any]) -> bool: - """Check if expose contains PII data.""" - # Check schema fields for PII classification - schema = expose.get("schema", {}) - fields = schema.get("fields", []) - - for field in fields: - classification = field.get("classification", "").lower() - if "pii" in classification or "sensitive" in classification: - return True - - # Check expose-level classification - if "pii" in expose.get("classification", "").lower(): - return True - - return False - - def _check_auto_approve(self, expose: Mapping[str, Any], fluid: Mapping[str, Any]) -> bool: - """Check if access should be auto-approved.""" - # Check access control settings - access_control = fluid.get("access_control", {}) - - # If PII, don't auto-approve - if self._check_contains_pii(expose): - return False - - # Check explicit auto-approve setting - auto_approve = access_control.get("auto_approve") - if auto_approve is not None: - return bool(auto_approve) - - # Default: no auto-approve for safety - return False - - def _extract_custom_properties(self, fluid: Mapping[str, Any]) -> List[Dict[str, Any]]: - """ - Extract product-level custom properties. - - Preserves FLUID-specific metadata. - """ - custom_props = [] - - metadata = fluid.get("metadata", {}) - - # Product type (if specified) - product_type = ( - metadata.get("product_type") - or metadata.get("type") - or fluid.get("product_type") - or fluid.get("type") - ) - if product_type: - custom_props.append({"property": "type", "value": product_type}) - - # Domain - domain = metadata.get("domain") - if domain: - custom_props.append({"property": "domain", "value": domain}) - - # FLUID version - fluid_version = metadata.get("version") - if fluid_version: - custom_props.append({"property": "fluidVersion", "value": fluid_version}) - - return custom_props - - def _write_output(self, data: Dict[str, Any], path: Union[Path, str], fmt: str) -> None: - """Write ODPS data to file.""" - output_path = Path(path) if not isinstance(path, Path) else path - output_path.parent.mkdir(parents=True, exist_ok=True) + bundle = super().render(src, out=out, fmt=fmt, out_dir=out_dir) + return bundle["product"] - with open(output_path, "w") as f: - if fmt == "yaml": - import yaml - yaml.dump(data, f, default_flow_style=False, sort_keys=False) - else: # json - json.dump(data, f, indent=2) +__all__ = ["OdpsStandardProvider"] diff --git a/fluid_build/providers/odps_standard/provider.py b/fluid_build/providers/odps_standard/provider.py new file mode 100644 index 00000000..af34a446 --- /dev/null +++ b/fluid_build/providers/odps_standard/provider.py @@ -0,0 +1,763 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +"""BitolOdpsProvider — thin orchestrator over the :mod:`mappers` pipeline. + +Phase 2 adds the **export** side: render a FLUID contract to one ODPS product +document plus N sibling ODCS contracts (one per output port). The plan +guarantees ``port.contractId == odcs_contract.id`` for every port, so the +sibling files line up with the references inside the ODPS doc. + +Phase 3 will add :meth:`import_contract` and :meth:`import_directory`. +""" + +from __future__ import annotations + +import copy +import logging +import os +from collections.abc import Iterable, Mapping, Sequence +from pathlib import Path +from typing import Any, Dict, List, Optional, Union + +from fluid_build.providers.base import ApplyResult, BaseProvider, ProviderError +from fluid_build.providers.odcs.mappers.base import PASSTHROUGH_KEY as ODCS_PASSTHROUGH_KEY +from fluid_build.providers.odcs.provider import DOCUMENT_KEY as ODCS_DOCUMENT_KEY +from fluid_build.providers.odcs.provider import ( + EXPECT_SERVER_KEYS, + PRODUCT_EXPECTS_KEY, + PRODUCT_OWNER_KEY, + OdcsProvider, + product_owner, +) + +from .io import read_input, write_output +from .mappers import EXPORT_PIPELINE, IMPORT_PIPELINE +from .mappers.base import ExportCtx, ImportCtx, conform_to_schema +from .resolver import ContractResolver, ResolvedContract +from .validation import ( + DEFAULT_API_VERSION, + SUPPORTED_API_VERSIONS, + api_version_tuple, + load_schema, + normalize_api_version, + validate, +) + +# Sub-import keys never kept in a port's ODCS document snapshot: the +# SchemaObjects travel as the expose itself (+ ``extra_exposes``) and the +# server-derived expects are redundant with the verbatim ``servers`` stash. +_SUB_SKIP_KEYS = frozenset({"exposes", "expects"}) +# Key of the product file in :meth:`BitolOdpsProvider._bundle_paths` (cannot +# collide with a contractId, which is never empty). +_PRODUCT_PATH_KEY = "" + + +def _below_v110(api_version: Any) -> bool: + version = api_version_tuple(str(api_version)) if api_version else () + return bool(version) and version < (1, 1, 0) + + +class BitolOdpsProvider(BaseProvider): + """Bitol Open Data Product Standard — bidirectional provider + (vendored schemas: v1.1.0, v1.0.0). + + Surface: + - :meth:`render` — FLUID → ODPS product + per-port ODCS contracts. + - :meth:`import_contract` / :meth:`import_directory` — ODPS → FLUID. + - :meth:`validate_product` — JSON Schema validation, version-selected + from the document's ``apiVersion``. + + ``api_version`` (or ``ODPS_API_VERSION``) sets the version *fresh* exports + declare. Re-exports of imported products preserve the document's own + ``apiVersion`` unless an explicit version is given. The per-port ODCS + version is an independent axis (see :class:`OdcsProvider`). + + Per-port ODCS rendering is delegated to :class:`OdcsProvider` (calling + ``render(fluid, expose_id=..., contract_id=port.contractId)``), so the + emitted ODCS ``id`` equals the ODPS port's ``contractId`` by + construction — the ``{product_id}.{expose_id}`` convention for fresh + FLUID, the original id (often a UUID) for imported products. A port + without a ``contractId`` (allowed by v1.1.0) gets no sibling ODCS. + + Importing a bundle keeps each port's whole ODCS import state on its + expose (see :meth:`_merge_odcs_into_expose`), so import → export + reproduces every per-port ODCS document as well as the product. + """ + + def __init__(self, api_version: Optional[str] = None) -> None: + super().__init__() + self.logger = logging.getLogger(f"{__name__}.{self.__class__.__name__}") + explicit = api_version or os.getenv("ODPS_API_VERSION") or None + self._explicit_api_version = normalize_api_version(explicit) if explicit else None + self.api_version = self._explicit_api_version or DEFAULT_API_VERSION + self.supported_versions = SUPPORTED_API_VERSIONS + self.spec_url = "https://github.com/bitol-io/open-data-product-standard" + self.schema = load_schema(self.api_version) + self.include_custom_properties = os.getenv("ODPS_INCLUDE_CUSTOM", "true").lower() == "true" + # ``version`` of fresh output ports whose expose declares none. + self.default_port_version = os.getenv("ODPS_DEFAULT_PORT_VERSION", "1") + # Strict by default — a contract that fails the schema is a bug. + self.strict_validation = os.getenv("ODPS_STRICT", "true").lower() == "true" + self._odcs = OdcsProvider() + + @property + def name(self) -> str: + return "odps_bitol" + + def capabilities(self) -> Mapping[str, bool]: + caps = dict(super().capabilities()) + caps.update( + { + "planning": False, + "apply": False, + "render": True, + "validate": True, + "import": True, # implemented in Phase 3 + "supports_batch": False, + } + ) + return caps + + # ---- unsupported lifecycle hooks ------------------------------------ + + def plan(self, contract: Mapping[str, Any]) -> List[Dict[str, Any]]: + raise ProviderError("Bitol ODPS provider does not support plan(). Use render() for export.") + + def apply(self, actions: Iterable[Mapping[str, Any]]) -> ApplyResult: + raise ProviderError( + "Bitol ODPS provider does not support apply(). Use render() for export." + ) + + # ---- render --------------------------------------------------------- + + def render( + self, + src: Union[Mapping[str, Any], Sequence[Mapping[str, Any]]], + *, + out: Optional[Union[Path, str]] = None, + fmt: Optional[str] = "yaml", + out_dir: Optional[Union[Path, str]] = None, + ) -> Dict[str, Any]: + """Render a FLUID contract to one ODPS product + N per-port ODCS contracts. + + Returns ``{"product": dict, "contracts": {contractId: dict, ...}}``. + + When ``out`` is provided the product is written there; when ``out_dir`` + is provided the product **and** every contract are written there with + the layout ``.odps.`` + ``.odcs.``. + ``out`` and ``out_dir`` can be combined. + """ + if isinstance(src, list): + raise ProviderError( + "Bitol ODPS export does not support batch processing. " + "Each data product should be exported separately." + ) + + product: Dict[str, Any] = {} + ctx = ExportCtx( + fluid=src, + odps=product, + logger=self.logger, + options={ + "include_custom_properties": self.include_custom_properties, + "api_version": self._explicit_api_version, + "default_port_version": self.default_port_version, + }, + ) + for mapper in EXPORT_PIPELINE: + mapper.to_odps(ctx) + + # Propagate strict mode to the per-port OdcsProvider dynamically so + # late toggles (``provider.strict_validation = False`` in tests) take + # effect for the same render call. + self._odcs._vowl_validate_on_export = self.strict_validation + + # Emit per-port ODCS contracts, keyed by contractId. The ports mapper + # emits exactly one port per named expose, in order, so the two + # sequences pair up positionally. + named_exposes = [ + e + for e in (src.get("exposes") or []) + if isinstance(e, Mapping) and (e.get("exposeId") or e.get("id")) + ] + contracts: Dict[str, Dict[str, Any]] = {} + seen_expose_ids: set = set() + for expose, port in zip(named_exposes, product.get("outputPorts") or []): + expose_id = expose.get("exposeId") or expose.get("id") + if expose_id in seen_expose_ids: + raise ProviderError( + f"Duplicate output port name '{expose_id}' — each FLUID expose " + "must have a unique exposeId/id." + ) + seen_expose_ids.add(expose_id) + contract_id = port.get("contractId") + if not contract_id: + # ODPS v1.1.0 allows a port without a contract; it references + # no ODCS document, so there is no sibling to emit. + self.logger.debug("Output port %r has no contractId; no ODCS emitted", expose_id) + continue + odcs = self._odcs.render(src, expose_id=expose_id, contract_id=contract_id) + if odcs.get("id") != contract_id: + # Defensive — these are equal by construction + raise ProviderError( + f"Linking invariant violated: port.contractId={contract_id!r} " + f"≠ odcs.id={odcs.get('id')!r}" + ) + contracts[contract_id] = odcs + + # A product rendered below v1.1.0 (an imported v1.1.0 product exported + # with --spec bitol-1.0.0, say) must fit the schema it declares, down + # to nested objects (management ports, SBOM entries, ...). The mappers + # already moved the top-level/port keys with their legacy encodings; + # this pass derives the rest from the vendored v1.0.0 schema. + if _below_v110(product.get("apiVersion")): + v100_schema = load_schema("v1.0.0") + if v100_schema: + conform_to_schema(product, v100_schema, self.logger) + + # Strict-mode validation (fail-loud), version-selected from each + # emitted document's own apiVersion. + if self.strict_validation: + validate(product) + for odcs in contracts.values(): + self._odcs.validate_contract(odcs) + + # Write artefacts. Bundle paths are checked before anything is + # written so an unsafe id fails the whole export. + bundle_paths = None + if out_dir is not None: + bundle_paths = self._bundle_paths(product, contracts, Path(out_dir), fmt or "yaml") + if out is not None and out != "-": + write_output(product, out, fmt or "yaml") + self.logger.info("Exported ODPS product: %s", out) + if out_dir is not None: + self._write_bundle(product, contracts, Path(out_dir), fmt or "yaml", paths=bundle_paths) + + return {"product": product, "contracts": contracts} + + def _write_bundle( + self, + product: Mapping[str, Any], + contracts: Mapping[str, Mapping[str, Any]], + out_dir: Path, + fmt: str, + *, + paths: Optional[Dict[str, Path]] = None, + ) -> None: + """Emit the canonical layout: 1 ``.odps.`` + N ``.odcs.``.""" + if paths is None: + paths = self._bundle_paths(product, contracts, out_dir, fmt) + out_dir.mkdir(parents=True, exist_ok=True) + product_path = paths[_PRODUCT_PATH_KEY] + write_output(dict(product), product_path, fmt) + self.logger.info("Exported ODPS product: %s", product_path) + for contract_id, odcs in contracts.items(): + contract_path = paths[contract_id] + write_output(dict(odcs), contract_path, fmt) + self.logger.info("Exported ODCS contract: %s", contract_path) + + @staticmethod + def _bundle_paths( + product: Mapping[str, Any], + contracts: Mapping[str, Mapping[str, Any]], + out_dir: Path, + fmt: str, + ) -> Dict[str, Path]: + """Compute (and vet) every bundle file path before anything is written. + + File names come from ids an imported document controls, so an id + that is absolute or carries a path component would escape + ``out_dir``; such ids raise :class:`ProviderError`. + """ + product_name = product.get("id") or product.get("name") or "product" + paths: Dict[str, Path] = { + _PRODUCT_PATH_KEY: _safe_bundle_path(out_dir, str(product_name), f".odps.{fmt}") + } + for contract_id in contracts: + paths[contract_id] = _safe_bundle_path(out_dir, str(contract_id), f".odcs.{fmt}") + return paths + + # ---- import (Phase 3) ---------------------------------------------- + + def import_contract( + self, + odps_doc: Union[Mapping[str, Any], str, Path], + *, + base_path: Optional[Union[str, Path]] = None, + allow_remote: bool = True, + resolver: Optional[ContractResolver] = None, + lenient: bool = False, + ) -> Dict[str, Any]: + """ODPS product (file or dict) → one FLUID contract. + + Walks every port, resolves the referenced ODCS contract via the + :class:`ContractResolver`, and merges per-port schema/quality/qos + back into FLUID exposes/expects. The verbatim ODPS payload is + preserved via ``metadata.odps_passthrough.*`` so re-export reproduces + it exactly. + + ``lenient=True`` downgrades resolution failures to warnings (the + resulting expose/expect stub will lack a populated contract.schema). + """ + # Load doc and capture its base path for the resolver + odps_data, doc_base_path = self._load_odps_doc(odps_doc, base_path) + + if resolver is None: + resolver = ContractResolver( + base_path=doc_base_path, + allow_remote=allow_remote, + odcs_provider=self._odcs, + ) + + # Run the mapper pipeline to build the FLUID skeleton (metadata, team, + # port stubs, support pass-through) + fluid: Dict[str, Any] = {} + ctx = ImportCtx(odps=odps_data, fluid=fluid, logger=self.logger) + for mapper in IMPORT_PIPELINE: + mapper.to_fluid(ctx) + + # Resolve every port's contract and merge it into the stub + self._resolve_and_merge_ports(fluid, odps_data, resolver, lenient=lenient) + _record_product_state(fluid) + + # Preserve the original ODPS doc verbatim for round-trip (a deep copy: + # sharing objects with the stashes would make the YAML dump of the + # imported FLUID emit anchors/aliases). + from .mappers.base import metadata_passthrough + + metadata_passthrough(fluid)["source"] = copy.deepcopy(dict(odps_data)) + return fluid + + def import_directory( + self, + dir_path: Union[str, Path], + *, + allow_remote: bool = False, + lenient: bool = False, + ) -> Dict[str, Any]: + """Import a directory containing an ODPS doc + sibling ODCS files. + + Layout cases handled: + + - **Standard**: exactly one ``*.odps.yaml`` / ``*.odps.yml`` / + ``*.odps.json`` (or any file with ``kind: DataProduct``) plus N + sibling ODCS files. The resolver + gets the directory's file index, so per-port contracts always + hit local first. + - **ODCS-only**: no ODPS doc, just one or more ODCS files. We + return a FLUID contract with one expose per ODCS file and no + product wrapper. + + Raises if the directory contains zero or more than one ODPS doc. + """ + root = Path(dir_path) + if not root.is_dir(): + raise ProviderError(f"import_directory: {root} is not a directory") + + odps_files: List[Path] = [] + odcs_files: List[Path] = [] + for child in sorted(root.rglob("*")): + if not child.is_file() or not _is_within(child, root): + # Symlinks pointing outside the bundle are never read. + continue + suf = child.suffix.lower() + if suf not in (".yaml", ".yml", ".json"): + continue + kind = _sniff_kind(child) + if kind == "DataProduct": + odps_files.append(child) + elif kind == "DataContract": + odcs_files.append(child) + + if len(odps_files) > 1: + raise ProviderError( + f"import_directory: found {len(odps_files)} ODPS docs in {root}; " + "expected exactly one (or zero for ODCS-only mode). " + f"Found: {[str(p) for p in odps_files]}" + ) + + if not odps_files: + return self._import_odcs_only(root, odcs_files, lenient=lenient) + + # Standard path — build a resolver pre-indexed with the dir's ODCS files + resolver = ContractResolver( + base_path=root, + allow_remote=allow_remote, + odcs_provider=self._odcs, + additional_files=odcs_files, + ) + resolver.index_directory(root) + return self.import_contract( + odps_files[0], + base_path=root, + allow_remote=allow_remote, + resolver=resolver, + lenient=lenient, + ) + + # ---- import internals ---------------------------------------------- + + def _load_odps_doc( + self, + odps_doc: Union[Mapping[str, Any], str, Path], + base_path: Optional[Union[str, Path]], + ) -> tuple: + if isinstance(odps_doc, (str, Path)): + doc_path = Path(odps_doc) + data = read_input(doc_path) + if not isinstance(data, Mapping): + raise ProviderError(f"ODPS doc {doc_path} did not parse as a mapping") + return dict(data), Path(base_path) if base_path else doc_path.parent + if not isinstance(odps_doc, Mapping): + raise ProviderError("import_contract: odps_doc must be a path or a mapping") + return dict(odps_doc), Path(base_path) if base_path else None + + def _resolve_and_merge_ports( + self, + fluid: Dict[str, Any], + odps_data: Mapping[str, Any], + resolver: ContractResolver, + *, + lenient: bool, + ) -> None: + # The mapper pipeline already created an expose stub per output port + # and an expect stub per input port (with contractId attached via the + # pass-through bucket). We now resolve each contractId and merge the + # imported ODCS fields into the stub. + # + # Resolution policy: + # - **output ports** must resolve. They're the data this product + # produces; an unresolved output is a broken bundle. The ``lenient`` + # flag still applies here. + # - **input ports** reference upstream contracts that often live in + # other repos / catalogues. Failure to find them is normal — we + # warn and leave the stub with just its contractId reference. + # + # Stubs pair with ports by position (the ports mapper appends exactly + # one stub per mapping port, in order): two ports may share a name + # and differ only in version. + expose_stubs = [e for e in fluid.get("exposes", []) if isinstance(e, Mapping)] + output_ports = [p for p in odps_data.get("outputPorts") or [] if isinstance(p, Mapping)] + server_expects: Dict[str, Dict[str, Any]] = {} + for stub, port in zip(expose_stubs, output_ports): + cid = port.get("contractId") + if not cid: + continue + resolved = self._safe_resolve(resolver, cid, lenient=lenient) + if resolved is None: + continue + for expect in self._merge_odcs_into_expose(stub, resolved, port_name=port.get("name")): + server_expects.setdefault(str(expect.get("id")), expect) + + expect_stubs = [e for e in fluid.get("expects", []) if isinstance(e, Mapping)] + input_ports = [p for p in odps_data.get("inputPorts") or [] if isinstance(p, Mapping)] + for stub, port in zip(expect_stubs, input_ports): + cid = port.get("contractId") + if cid: + # Always lenient for input ports — upstream contracts may live elsewhere + resolved = self._safe_resolve(resolver, cid, lenient=True) + if resolved is not None: + self._merge_odcs_into_expect(stub, resolved) + # The output-port contracts describe the upstream sources they + # read as ODCS servers; give the matching expect its provider / + # location back when it has none of its own. + server = server_expects.get(str(port.get("name"))) + if server is not None: + for key in ("provider", "location"): + if key in server and key not in stub: + stub[key] = copy.deepcopy(server[key]) + + def _safe_resolve( + self, resolver: ContractResolver, contract_id: str, lenient: bool + ) -> Optional[ResolvedContract]: + try: + return resolver.resolve(contract_id) + except ProviderError as exc: + # ContractNotFound, RemoteFetchDisabled and ContractValidationError + # all mean "this port's contract is unavailable". + if lenient: + self.logger.warning("Skipping unresolved contract %s: %s", contract_id, exc) + return None + raise + + def _merge_odcs_into_expose( + self, + stub: Dict[str, Any], + resolved: ResolvedContract, + *, + port_name: Optional[str] = None, + ) -> List[Dict[str, Any]]: + """Run the ODCS importer to get FLUID fields, then merge into the stub. + + The OdcsProvider returns a FLUID skeleton with one expose per + SchemaObject. For per-port contracts we expect one, so we merge its + ``contract`` and ``qos`` into the port stub, keeping the port's + id/version/description/tags/pass-through. + + The port's complete ODCS import state is kept on the stub so a + re-export reproduces the per-port document: + + - ``expose.odcs_passthrough._verbatim`` — the SchemaObject stash; + - ``expose.odcs_passthrough._document`` — the sub-import's + document-level FLUID (``metadata`` with its ``odcs_passthrough`` + stash of top-level fields/apiVersion, servers, team, SLA and + quality; ``owner``; ``contract``), extra SchemaObjects, and the + stub's FLUID-native name/description at import time. + + The snapshot is kept even for an ODCS document without SchemaObjects + (``schema`` is optional): the port then has no schema to merge, but + its document-level fields still have to round-trip. + + Returns the expects the ODCS importer derived from ``servers`` so the + caller can backfill matching input ports. + """ + # The resolver already validated the document (strictly). + sub_fluid = self._odcs.import_contract(resolved.odcs, lenient=True) + sub_exposes = [e for e in sub_fluid.get("exposes") or [] if isinstance(e, Mapping)] + sub_expects = [e for e in sub_fluid.get("expects") or [] if isinstance(e, Mapping)] + primary: Mapping[str, Any] = sub_exposes[0] if sub_exposes else {} + if sub_exposes: + # Carry over the contract block (schema, quality, relationships, etc.) + if primary.get("contract"): + stub["contract"] = copy.deepcopy(primary["contract"]) + if "binding" not in stub: + binding = self._binding_from_servers(sub_expects, stub, primary, port_name) + if binding: + stub["binding"] = binding + if primary.get("qos") and "qos" not in stub: + stub["qos"] = copy.deepcopy(primary["qos"]) + # The SchemaObject description stays in the stash: copying it onto + # the stub would add a description to the ODPS port on re-export. + _attach_document(stub, sub_fluid, sub_exposes, sub_expects) + return sub_expects + + @staticmethod + def _binding_from_servers( + sub_expects: List[Dict[str, Any]], + stub: Mapping[str, Any], + primary: Mapping[str, Any], + port_name: Optional[str], + ) -> Optional[Dict[str, Any]]: + """FLUID-native binding for a port: the ODCS server named like the + port (or its SchemaObject), else the ODCS importer's own guess.""" + names = { + n + for n in ( + port_name, + stub.get("exposeId"), + stub.get("id"), + primary.get("exposeId"), + primary.get("id"), + ) + if n + } + for expect in sub_expects: + if expect.get("id") in names and expect.get("provider"): + binding: Dict[str, Any] = {"platform": expect["provider"]} + if isinstance(expect.get("location"), Mapping) and expect["location"]: + binding["location"] = copy.deepcopy(dict(expect["location"])) + return binding + if isinstance(primary.get("binding"), Mapping) and primary["binding"]: + return copy.deepcopy(dict(primary["binding"])) + return None + + def _merge_odcs_into_expect(self, stub: Dict[str, Any], resolved: ResolvedContract) -> None: + sub_fluid = self._odcs.import_contract(resolved.odcs, lenient=True) + sub_exposes = sub_fluid.get("exposes") or [] + if not sub_exposes: + return + primary = sub_exposes[0] + if primary.get("contract"): + stub["contract"] = copy.deepcopy(primary["contract"]) + if primary.get("binding") and "binding" not in stub: + stub["binding"] = copy.deepcopy(primary["binding"]) + + def _import_odcs_only( + self, root: Path, odcs_files: List[Path], *, lenient: bool = False + ) -> Dict[str, Any]: + """Build a wrapper-less FLUID contract from a directory of ODCS files. + + Each SchemaObject of each ODCS file becomes one expose under a + synthetic product. Useful when a team publishes raw ODCS contracts + without an ODPS wrapper. + + Like a bundle port, the first expose of each file carries that + file's ODCS document state (``expose.odcs_passthrough._document``) + and keeps the ODCS ``id`` as its contract id + (``expose.odps_passthrough.contract_id``), so a re-export reproduces + the document's id, name, description, domain, team, servers and SLA + instead of minting new ones. Further SchemaObjects of a file stay + exposes of their own (each carrying the document's top-level fields + under a minted id). + """ + if not odcs_files: + raise ProviderError(f"import_directory: {root} contains no ODPS doc and no ODCS files") + + # ``Path(".").name`` is "" — name the product after the real directory. + product_name = root.resolve().name or "product" + fluid: Dict[str, Any] = { + "metadata": { + "name": product_name, + "version": "1.0.0", + "status": "active", + }, + "contract": {"id": product_name}, + "exposes": [], + "expects": [], + } + from .mappers.base import expose_passthrough, fluid_id, metadata_passthrough + + seen_ids: set = set() + for odcs_file in odcs_files: + sub = self._odcs.import_contract(odcs_file, lenient=lenient) + sub_exposes = [e for e in sub.get("exposes") or [] if isinstance(e, Mapping)] + sub_expects = [e for e in sub.get("expects") or [] if isinstance(e, Mapping)] + odcs_id = fluid_id(sub) + if not isinstance(odcs_id, str) or not odcs_id or odcs_id in seen_ids: + odcs_id = None # two files sharing an id: the later ones get minted ids + else: + seen_ids.add(odcs_id) + for index, expose in enumerate(sub_exposes): + # Every SchemaObject is an expose of its own here, so none of + # them rides along as an extra object of another's document. + _attach_document(expose, sub, [expose], sub_expects) + if index == 0 and odcs_id: + expose_passthrough(expose)["contract_id"] = odcs_id + fluid["exposes"].append(expose) + _record_product_state(fluid) + + metadata_passthrough(fluid)["odcs_only_directory"] = True + metadata_passthrough(fluid)["odcs_only_warning"] = ( + f"Directory {root} contained no ODPS doc; " + f"FLUID built from {len(odcs_files)} ODCS file(s)." + ) + self.logger.warning( + "import_directory: no ODPS doc in %s — building FLUID from %d ODCS file(s)", + root, + len(odcs_files), + ) + return fluid + + # ---- validation ----------------------------------------------------- + + def validate_product(self, odps: Mapping[str, Any]) -> None: + """Validate against the vendored JSON Schema matching ``odps["apiVersion"]``.""" + validate(odps) + + +def _attach_document( + stub: Dict[str, Any], + sub_fluid: Mapping[str, Any], + sub_exposes: List[Mapping[str, Any]], + sub_expects: List[Mapping[str, Any]], +) -> None: + """Store an ODCS sub-import's document state on ``stub`` (the port's expose). + + ``sub_exposes[0]`` is the SchemaObject the stub stands for (absent for a + schema-less ODCS document); any further ones travel as + ``extra_exposes``. See :meth:`BitolOdpsProvider._merge_odcs_into_expose`. + """ + primary: Mapping[str, Any] = sub_exposes[0] if sub_exposes else {} + document = {k: copy.deepcopy(v) for k, v in sub_fluid.items() if k not in _SUB_SKIP_KEYS} + # The sub-import's expects are the FLUID projection of this document's + # ``servers``; the ODCS servers mapper three-way-merges against them, + # so they must travel with the snapshot or every server reads as + # "removed in FLUID". + document["expects"] = copy.deepcopy(list(sub_expects)) + if len(sub_exposes) > 1: + document["extra_exposes"] = copy.deepcopy(list(sub_exposes[1:])) + document["expose_native"] = { + "name": stub.get("exposeId") or stub.get("id"), + "description": stub.get("description") or None, + # The binding the port got here vs. the one the ODCS importer + # derived from ``servers``: an unedited port binding is mapped back + # to the latter on render so the per-port servers replay verbatim. + "binding": copy.deepcopy(stub.get("binding")), + "odcs_binding": copy.deepcopy(primary.get("binding")), + } + odcs_pt = stub.setdefault(ODCS_PASSTHROUGH_KEY, {}) + if primary is not stub: + odcs_pt.update(copy.deepcopy(dict(primary.get(ODCS_PASSTHROUGH_KEY) or {}))) + odcs_pt[ODCS_DOCUMENT_KEY] = document + + +def _record_product_state(fluid: Dict[str, Any]) -> None: + """Record the product-level ``owner`` / ``expects`` at import time in + every port's ODCS document snapshot. + + The per-port ODCS view three-way-merges them against the product's + current ones, so a FLUID owner or expects edit made after import reaches + the per-port ``team`` / ``servers`` while an unedited contract replays + verbatim (see :func:`fluid_build.providers.odcs.provider._port_document_view`). + """ + owner = copy.deepcopy(product_owner(fluid)) + # Only the keys the per-port merge compares (an input port's resolved + # contract would otherwise be copied into every port's snapshot). + expects = [ + {k: copy.deepcopy(e[k]) for k in ("id",) + EXPECT_SERVER_KEYS if k in e} + for e in fluid.get("expects") or [] + if isinstance(e, Mapping) + ] + for expose in fluid.get("exposes") or []: + if not isinstance(expose, Mapping): + continue + odcs_pt = expose.get(ODCS_PASSTHROUGH_KEY) + document = odcs_pt.get(ODCS_DOCUMENT_KEY) if isinstance(odcs_pt, Mapping) else None + if isinstance(document, dict): + document[PRODUCT_OWNER_KEY] = copy.deepcopy(owner) + document[PRODUCT_EXPECTS_KEY] = copy.deepcopy(expects) + + +def _sniff_kind(path: Path) -> Optional[str]: + """Return ``kind`` from a YAML/JSON file without raising. + + FLUID contracts also declare ``kind: DataProduct``; a document carrying + ``fluidVersion`` is FLUID, neither ODPS nor ODCS, and yields ``None``. + """ + try: + data = read_input(path) + except Exception: + return None + if not isinstance(data, Mapping): + return None + if "fluidVersion" in data: + return None + return data.get("kind") + + +def _is_within(path: Path, root: Path) -> bool: + """True when ``path`` (symlinks resolved) stays inside ``root``.""" + try: + return path.resolve().is_relative_to(root.resolve()) + except (OSError, RuntimeError, ValueError): + # ValueError: an embedded NUL byte. + return False + + +def _safe_bundle_path(out_dir: Path, stem: str, suffix: str) -> Path: + """``out_dir / (stem + suffix)``, refusing stems that would escape ``out_dir``.""" + if ( + not stem + or ".." in stem + or "/" in stem + or "\\" in stem + or os.sep in stem + or (os.altsep is not None and os.altsep in stem) + or os.path.isabs(stem) + or Path(stem).drive + ): + raise ProviderError( + f"Refusing to write bundle file for id {stem!r}: an id used as a file " + "name must not be absolute or contain path separators or '..'." + ) + candidate = out_dir / f"{stem}{suffix}" + if candidate.resolve().parent != out_dir.resolve(): + raise ProviderError( + f"Refusing to write bundle file {candidate}: it resolves outside {out_dir}." + ) + return candidate diff --git a/fluid_build/providers/odps_standard/resolver.py b/fluid_build/providers/odps_standard/resolver.py new file mode 100644 index 00000000..3ae6e53e --- /dev/null +++ b/fluid_build/providers/odps_standard/resolver.py @@ -0,0 +1,600 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +"""Resolve Bitol ODPS port ``contractId`` references to full ODCS documents. + +Resolution order (per the Phase 3 plan): + + 1. **local file** — probe candidate filenames in ``base_path`` and a few + conventional subdirs. + 2. **http(s) URL** — only when the ``contractId`` is itself a URL (or a + ``hint`` URL is given) and ``allow_remote=True``. + +Every resolved document is validated against the ODCS schema matching its +``apiVersion`` before it is returned. The resolver caches by ``contractId`` +so repeated lookups inside one import pass cost nothing. + +Safety rules (a ``contractId`` comes from a document the user may not trust): + +- A local probe only accepts a file whose ODCS ``id`` equals the requested + ``contractId`` (the last-segment fallback must not pick up an unrelated + contract). Explicit hints and URLs are exempt. +- A ``contractId`` that is absolute or contains a path separator or ``..`` is + never used as a local file stem, and every probed / indexed file must stay + inside ``base_path`` once symlinks are resolved. +- Remote fetches follow redirects only to http(s), read at most + :data:`DEFAULT_MAX_BYTES`, and give up after :data:`DEFAULT_TOTAL_TIMEOUT` + seconds in total: every socket operation (connect, request, status line, + headers, chunk-size lines, body) runs with a timeout of + ``min(timeout, time left)``, so a server dripping its headers cannot hold + a fetch past the deadline. +- YAML is loaded with the alias-bomb guard of + :func:`fluid_build.providers.odcs.io.load_yaml`. +- A malformed ``contractId`` (not a string, or one with a NUL byte) is + :class:`ContractNotFound`, never a crash, so lenient imports warn. + +Errors raised: + - :class:`ContractNotFound` — no candidate path matched. + - :class:`RemoteFetchDisabled` — URL needed but ``allow_remote=False``. + - :class:`ContractValidationError` — resolved document failed ODCS validation. +""" + +from __future__ import annotations + +import http.client +import io +import json +import logging +import os +import socket +import time +import urllib.request +from collections.abc import Mapping +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple, Union +from urllib.error import HTTPError, URLError +from urllib.parse import urljoin, urlparse + +from fluid_build.providers.base import ProviderError +from fluid_build.providers.odcs.io import load_yaml, read_input +from fluid_build.providers.odcs.provider import OdcsProvider + +LOG = logging.getLogger(__name__) + +DEFAULT_TIMEOUT = 10 # seconds per socket operation for http(s) fetch +DEFAULT_TOTAL_TIMEOUT = 30.0 # seconds for a whole http(s) fetch +DEFAULT_MAX_BYTES = 10 * 1024 * 1024 # largest remote contract body accepted +_READ_CHUNK = 64 * 1024 +# Probing matrix sized for the common case (Bitol fragments layout: ODPS doc +# + sibling ODCS files, optionally nested under contracts/). Previously we +# probed {"", contracts, odcs, odcs/contracts} × {full-id, last-segment} × 6 +# extensions = 48 candidates, which produced very noisy error messages on +# misses. Trimmed to the two most-used subdirs and the canonical +# ``.odcs.{yaml,json}`` extensions — 8 candidates total — which still covers +# every layout we've seen in the wild without burying the operator in +# tried-paths output. +DEFAULT_SUBDIRS = ("", "contracts") +DEFAULT_EXTENSIONS = (".odcs.yaml", ".odcs.yml", ".odcs.json", ".yaml") + + +class ContractNotFound(ProviderError): + """No candidate path / URL matched the requested contractId.""" + + _MAX_TRIED_DISPLAYED = 5 + + def __init__(self, contract_id: str, tried: List[str]): + self.contract_id = contract_id + self.tried = list(tried) + if tried: + head = tried[: self._MAX_TRIED_DISPLAYED] + tail = ( + f"\n …and {len(tried) - self._MAX_TRIED_DISPLAYED} more" + if len(tried) > self._MAX_TRIED_DISPLAYED + else "" + ) + msg = ( + f"Could not resolve contractId {contract_id!r}. Tried:\n " + + "\n ".join(head) + + tail + ) + else: + msg = f"Could not resolve contractId {contract_id!r} (no candidates probed)" + super().__init__(msg) + + +class RemoteFetchDisabled(ProviderError): + """A URL was given but ``allow_remote=False``.""" + + +class ContractValidationError(ProviderError): + """A resolved ODCS document failed schema validation.""" + + +@dataclass +class ResolvedContract: + """The result of a successful resolve() call.""" + + odcs: Dict[str, Any] + source: str # "local" | "remote" + origin: str # the path or URL the doc was loaded from + contract_id: str + + +@dataclass +class ContractResolver: + """Lookup ODCS contracts by ``contractId``. + + Parameters + ---------- + base_path: + Directory to probe for local candidates. May be a file (its parent + directory is used) or a directory. + allow_remote: + If ``False``, raise :class:`RemoteFetchDisabled` whenever a URL is the + only viable candidate. Default ``True``. + timeout: + Socket timeout for http(s) fetches (per socket operation). + odcs_provider: + Optional :class:`OdcsProvider` to use for validation. A shared + instance avoids reloading the JSON Schema for every resolve. + candidate_extensions / candidate_subdirs: + Override the default probing matrix. + additional_files: + Extra absolute paths to consider before the default probes — used by + :meth:`BitolOdpsProvider.import_directory` so files discovered by + directory scan get tried first. + max_bytes / total_timeout: + Remote fetch limits: the largest body accepted and the deadline for + a whole fetch (connect + redirects + headers + body). + """ + + base_path: Optional[Path] = None + allow_remote: bool = True + timeout: int = DEFAULT_TIMEOUT + odcs_provider: Optional[OdcsProvider] = None + candidate_extensions: tuple = DEFAULT_EXTENSIONS + candidate_subdirs: tuple = DEFAULT_SUBDIRS + additional_files: List[Path] = field(default_factory=list) + max_bytes: int = DEFAULT_MAX_BYTES + total_timeout: float = DEFAULT_TOTAL_TIMEOUT + + _cache: Dict[str, ResolvedContract] = field(default_factory=dict, init=False) + _index: Dict[str, Path] = field(default_factory=dict, init=False) + + def __post_init__(self) -> None: + if self.base_path is not None: + self.base_path = Path(self.base_path) + if self.base_path.is_file(): + self.base_path = self.base_path.parent + if self.odcs_provider is None: + self.odcs_provider = OdcsProvider() + # Build a quick id→path index for any pre-discovered files + for fpath in self.additional_files: + self._index_file(fpath) + + # ---- public API ----------------------------------------------------- + + def index_directory(self, dir_path: Union[str, Path]) -> None: + """Pre-scan a directory tree for ``*.odcs.yaml``/``*.odcs.yml``/``*.odcs.json`` + files and index them by their ODCS ``id`` for first-hit lookup.""" + root = Path(dir_path) + if not root.is_dir(): + return + for child in root.rglob("*"): + if ( + child.is_file() + and child.suffix.lower() in (".yaml", ".yml", ".json") + and _is_within(child, root) + ): + self._index_file(child) + + def resolve( + self, contract_id: str, *, hint: Optional[Union[str, Path]] = None + ) -> ResolvedContract: + if not isinstance(contract_id, str) or "\x00" in contract_id: + # Not usable as a file stem, a URL or an index key; a document + # can carry anything here, so this is "not found", not a crash. + raise ContractNotFound(str(contract_id), []) + if contract_id in self._cache: + return self._cache[contract_id] + + tried: List[str] = [] + + # 0. Explicit hint (file path or URL) wins + if hint: + resolved = self._try_hint(contract_id, hint, tried) + if resolved is not None: + return self._cache_and_return(contract_id, resolved) + + # 1. Pre-indexed file (from index_directory / additional_files) + indexed = self._index.get(contract_id) + if indexed is not None: + tried.append(str(indexed)) + resolved = self._load_local(contract_id, indexed) + if resolved is not None: + return self._cache_and_return(contract_id, resolved) + + # 2. Local file probes — only a document whose ``id`` is the + # requested contractId counts (the last-segment fallback would + # otherwise pick up an unrelated contract of the same port name). + if self.base_path is not None: + for candidate in self._local_candidates(contract_id): + tried.append(str(candidate)) + if candidate.exists(): + resolved = self._load_local(contract_id, candidate, require_id_match=True) + if resolved is not None: + return self._cache_and_return(contract_id, resolved) + + # 3. http(s) if the contractId itself is a URL + if _looks_like_url(contract_id): + if not self.allow_remote: + raise RemoteFetchDisabled( + f"Cannot resolve contractId {contract_id!r}: remote fetch disabled" + ) + tried.append(contract_id) + resolved = self._load_remote(contract_id, contract_id) + if resolved is not None: + return self._cache_and_return(contract_id, resolved) + + raise ContractNotFound(contract_id, tried) + + # ---- internals ------------------------------------------------------ + + def _local_candidates(self, contract_id: str) -> List[Path]: + if self.base_path is None: + return [] + if not _is_safe_stem(contract_id): + # An absolute path or one with separators / ".." would escape + # base_path (pathlib drops base_path for an absolute stem). + LOG.debug("contractId %r is not a plain file stem; no local probe", contract_id) + return [] + # The contractId may itself contain dots (e.g. "product.daily_orders"). + # We probe both the full id and the last segment (port name) to be lenient. + stems = [contract_id] + if "." in contract_id: + stems.append(contract_id.rsplit(".", 1)[-1]) + candidates: List[Path] = [] + for sub in self.candidate_subdirs: + sub_dir = self.base_path / sub if sub else self.base_path + for stem in stems: + for ext in self.candidate_extensions: + candidate = sub_dir / f"{stem}{ext}" + if _is_within(candidate, self.base_path): + candidates.append(candidate) + return candidates + + def _try_hint( + self, contract_id: str, hint: Union[str, Path], tried: List[str] + ) -> Optional[ResolvedContract]: + if isinstance(hint, Path) or not _looks_like_url(str(hint)): + path = Path(hint) + tried.append(str(path)) + if path.exists(): + return self._load_local(contract_id, path) + return None + # Hint is a URL + if not self.allow_remote: + raise RemoteFetchDisabled( + f"Cannot resolve contractId {contract_id!r} via hint URL: remote fetch disabled" + ) + tried.append(str(hint)) + return self._load_remote(contract_id, str(hint)) + + def _index_file(self, path: Path) -> None: + try: + data = read_input(path) + except Exception as exc: + LOG.debug("Skipping unreadable file %s: %s", path, exc) + return + if not isinstance(data, Mapping): + return + odcs_id = data.get("id") + if isinstance(odcs_id, str): + self._index.setdefault(odcs_id, path) + + def _load_local( + self, contract_id: str, path: Path, *, require_id_match: bool = False + ) -> Optional[ResolvedContract]: + try: + data = read_input(path) + except Exception as exc: + LOG.warning("Failed to read %s: %s", path, exc) + return None + if not isinstance(data, Mapping): + return None + if require_id_match and data.get("id") != contract_id: + LOG.debug("Skipping %s: id %r != contractId %r", path, data.get("id"), contract_id) + return None + self._validate(contract_id, data, origin=str(path)) + return ResolvedContract( + odcs=dict(data), + source="local", + origin=str(path), + contract_id=contract_id, + ) + + def _load_remote(self, contract_id: str, url: str) -> Optional[ResolvedContract]: + import yaml + + content_type, body = self._fetch(contract_id, url) + + # Refuse HTML-with-200 — common when the URL is wrong or it returned a portal page + if "text/html" in content_type.lower() or body.lstrip().startswith(b"<"): + raise ContractNotFound(contract_id, [url]) + + try: + text = body.decode("utf-8", errors="strict") + try: + data = json.loads(text) + except json.JSONDecodeError: + data = load_yaml(text, source=url) + except (UnicodeDecodeError, yaml.YAMLError, ProviderError) as exc: + LOG.warning("Could not parse %s: %s", url, exc) + raise ContractNotFound(contract_id, [url]) from exc + if not isinstance(data, Mapping): + raise ContractNotFound(contract_id, [url]) + self._validate(contract_id, data, origin=url) + return ResolvedContract( + odcs=dict(data), + source="remote", + origin=url, + contract_id=contract_id, + ) + + def _fetch(self, contract_id: str, url: str) -> Tuple[str, bytes]: + """GET ``url`` within the size and time limits; ``(content_type, body)``. + + Every socket operation is armed with ``min(timeout, time left)`` + (see :class:`_DeadlineSocket`), so the whole fetch (connect, + redirects, headers, chunk-size lines and body) ends by + ``total_timeout``. + """ + deadline = time.monotonic() + self.total_timeout + try: + with urlopen(url, timeout=self.timeout, deadline=deadline) as response: + if time.monotonic() > deadline: + raise ContractNotFound(contract_id, [url]) + final_url = response.geturl() if hasattr(response, "geturl") else url + if not _looks_like_url(str(final_url or url)): + raise ContractNotFound(contract_id, [url]) + content_type = response.headers.get("Content-Type", "") or "" + length = response.headers.get("Content-Length") + if length and length.isdigit() and int(length) > self.max_bytes: + raise ContractNotFound(contract_id, [url]) + chunks: List[bytes] = [] + total = 0 + while True: + if time.monotonic() > deadline: + LOG.warning("Fetching %s exceeded %gs; giving up", url, self.total_timeout) + raise ContractNotFound(contract_id, [url]) + want = min(_READ_CHUNK, self.max_bytes + 1 - total) + # ``read1`` returns after one socket read, so a slow + # server cannot hold a single call past the deadline. + chunk = response.read1(want) if hasattr(response, "read1") else None + if chunk is None: + chunk = response.read(want) + if not chunk: + break + chunks.append(chunk) + total += len(chunk) + if total > self.max_bytes: + LOG.warning("%s is larger than %d bytes; refusing it", url, self.max_bytes) + raise ContractNotFound(contract_id, [url]) + except (URLError, OSError, http.client.HTTPException, ValueError) as exc: + # URLError covers HTTP errors and refused redirects; OSError covers + # socket timeouts / resets; ValueError a malformed URL. + raise ContractNotFound(contract_id, [url]) from exc + return content_type, b"".join(chunks) + + def _validate(self, contract_id: str, data: Mapping[str, Any], *, origin: str) -> None: + if self.odcs_provider is None or not self.odcs_provider.schema: + return + try: + self.odcs_provider.validate_contract(data) + except ProviderError as exc: + raise ContractValidationError( + f"Resolved contract for {contract_id!r} from {origin} " + f"failed ODCS validation: {exc}" + ) from exc + + def _cache_and_return(self, contract_id: str, resolved: ResolvedContract) -> ResolvedContract: + self._cache[contract_id] = resolved + return resolved + + +def urlopen(url: str, *, timeout: Optional[float], deadline: Optional[float] = None) -> Any: + """``urllib.request.urlopen`` whose redirects may only lead to http(s). + + With ``deadline`` (a :func:`time.monotonic` value), every socket + operation of every connection the fetch opens (redirects included) + times out at ``min(timeout, deadline - now)``; a ``None`` per-operation + ``timeout`` leaves only the deadline. + """ + handlers: List[Any] = [_HttpOnlyRedirectHandler()] + if deadline is not None: + handlers.append(_DeadlineHTTPHandler(deadline)) + if _DeadlineHTTPSHandler is not None: + handlers.append(_DeadlineHTTPSHandler(deadline)) + remaining = max(deadline - time.monotonic(), 0.001) + timeout = remaining if timeout is None else min(timeout, remaining) + opener = urllib.request.build_opener(*handlers) + return opener.open(url, timeout=timeout) + + +def _time_left(deadline: float) -> float: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise socket.timeout("remote contract fetch exceeded its total deadline") + return remaining + + +class _DeadlineSocket: + """Socket proxy that arms each operation with ``min(timeout, time left)``. + + ``http.client`` reads the status line, headers and chunk-size lines + through ``sock.makefile("rb")``; the reader returned here goes through + :meth:`recv_into`, so those reads are bounded by the deadline as well. + """ + + def __init__(self, sock: Any, deadline: float, timeout: Optional[float]) -> None: + self._sock = sock + self._deadline = deadline + self._timeout = timeout + self._readers = 0 + self._closed = False + + def _arm(self) -> None: + remaining = _time_left(self._deadline) + per_op = remaining if self._timeout is None else min(self._timeout, remaining) + self._sock.settimeout(per_op) + + def recv_into(self, buffer: Any, nbytes: int = 0, flags: int = 0) -> int: + self._arm() + return self._sock.recv_into(buffer, nbytes, flags) + + def recv(self, bufsize: int, flags: int = 0) -> bytes: + self._arm() + return self._sock.recv(bufsize, flags) + + def send(self, data: Any, flags: int = 0) -> int: + self._arm() + return self._sock.send(data, flags) + + def sendall(self, data: Any, flags: int = 0) -> None: + self._arm() + self._sock.sendall(data, flags) + + def makefile(self, mode: str = "r", buffering: Optional[int] = None, **kwargs: Any) -> Any: + if mode not in ("rb", "br"): + raise ValueError(f"_DeadlineSocket.makefile supports only 'rb', not {mode!r}") + self._readers += 1 + size = buffering if buffering and buffering > 0 else io.DEFAULT_BUFFER_SIZE + return io.BufferedReader(_DeadlineReader(self), size) + + def _reader_closed(self) -> None: + self._readers -= 1 + if self._closed and self._readers <= 0: + self._sock.close() + + def close(self) -> None: + # Like ``socket.socket``: the descriptor stays open while a reader + # made by ``makefile`` is (urllib closes the connection's socket + # right after reading the headers and keeps reading the body). + self._closed = True + if self._readers <= 0: + self._sock.close() + + def __getattr__(self, name: str) -> Any: + return getattr(self._sock, name) + + +class _DeadlineReader(io.RawIOBase): + """Raw binary reader over a :class:`_DeadlineSocket`.""" + + def __init__(self, sock: _DeadlineSocket) -> None: + super().__init__() + self._dsock = sock + + def readable(self) -> bool: + return True + + def readinto(self, buffer: Any) -> int: + return self._dsock.recv_into(buffer) + + def close(self) -> None: + if not self.closed: + super().close() + self._dsock._reader_closed() + + +class _DeadlineHandlerMixin: + """Gives every connection an urllib handler opens a deadline-armed socket.""" + + _deadline: float + + def do_open(self, http_class: Any, req: Any, **http_conn_args: Any) -> Any: + deadline = self._deadline + + def connection(*args: Any, **kwargs: Any) -> Any: + conn = http_class(*args, **kwargs) + plain_connect = conn.connect + per_op = conn.timeout if isinstance(conn.timeout, (int, float)) else None + + def connect() -> None: + remaining = _time_left(deadline) + conn.timeout = remaining if per_op is None else min(per_op, remaining) + plain_connect() + conn.sock = _DeadlineSocket(conn.sock, deadline, per_op) + + conn.connect = connect + return conn + + return super().do_open(connection, req, **http_conn_args) # type: ignore[misc] + + +class _DeadlineHTTPHandler(_DeadlineHandlerMixin, urllib.request.HTTPHandler): + def __init__(self, deadline: float) -> None: + super().__init__() + self._deadline = deadline + + +_DeadlineHTTPSHandler: Optional[type] = None +if hasattr(urllib.request, "HTTPSHandler"): # Python built with ssl + + class _DeadlineHTTPSHandlerImpl(_DeadlineHandlerMixin, urllib.request.HTTPSHandler): + def __init__(self, deadline: float) -> None: + super().__init__() + self._deadline = deadline + + _DeadlineHTTPSHandler = _DeadlineHTTPSHandlerImpl + + +class _HttpOnlyRedirectHandler(urllib.request.HTTPRedirectHandler): + """Follow redirects only to http(s) (urllib's default also allows ftp).""" + + def redirect_request(self, req, fp, code, msg, headers, newurl): # type: ignore[override] + target = urljoin(req.full_url, newurl) + if urlparse(target).scheme.lower() not in ("http", "https"): + raise HTTPError( + req.full_url, + code, + f"refusing redirect to non-http(s) URL {target!r}", + headers, + fp, + ) + return super().redirect_request(req, fp, code, msg, headers, newurl) + + +def _is_safe_stem(contract_id: str) -> bool: + """A contractId usable as a local file stem: relative, no separators, no ``..``.""" + if not isinstance(contract_id, str) or not contract_id or ".." in contract_id: + return False + if "\x00" in contract_id: + return False + if "/" in contract_id or "\\" in contract_id or os.sep in contract_id: + return False + if os.altsep is not None and os.altsep in contract_id: + return False + return not (os.path.isabs(contract_id) or Path(contract_id).drive) + + +def _is_within(path: Path, root: Path) -> bool: + """True when ``path`` (symlinks resolved) stays inside ``root``.""" + try: + return path.resolve().is_relative_to(Path(root).resolve()) + except (OSError, RuntimeError, ValueError): + # ValueError: an embedded NUL byte. + return False + + +def _looks_like_url(value: str) -> bool: + try: + parsed = urlparse(value) + except ValueError: # e.g. "http://[" (invalid IPv6 literal) + return False + return parsed.scheme in ("http", "https") and bool(parsed.netloc) diff --git a/fluid_build/providers/odps_standard/schemas/odps-product-v1.0.0.json b/fluid_build/providers/odps_standard/schemas/odps-product-v1.0.0.json new file mode 100644 index 00000000..40b46c69 --- /dev/null +++ b/fluid_build/providers/odps_standard/schemas/odps-product-v1.0.0.json @@ -0,0 +1,517 @@ +{ + "$schema": "https://json-schema.org/draft/2019-09/schema", + "title": "Open Data Product Standard (ODPS)", + "description": "An open data product standard descriptor to enable defining data products.", + "type": "object", + "required": ["apiVersion", "kind", "id", "status"], + "additionalProperties": false, + "properties": { + "apiVersion": { + "type": "string", + "default": "v1.0.0", + "description": "Version of the standard used to build data product. Default value is v1.0.0.", + "enum": ["v0.9.0", "v1.0.0"] + }, + "kind": { + "type": "string", + "default": "DataProduct", + "description": "The kind of file this is. Valid value is `DataProduct`.", + "enum": ["DataProduct"] + }, + "id": { + "type": "string", + "description": "A unique identifier used to reduce the risk of dataset name collisions, such as a UUID." + }, + "name": { + "type": "string", + "description": "Name of the data product." + }, + "version": { + "type": "string", + "description": "Current version of the data product. Not required, but highly recommended." + }, + "status": { + "type": "string", + "description": "Current status of the data product.", + "examples": ["proposed", "draft", "active", "deprecated", "retired"] + }, + "domain": { + "type": "string", + "description": "Business domain" + }, + "tenant": { + "type": "string", + "description": "Organization identifier" + }, + "authoritativeDefinitions": { + "type": "array", + "description": "List of links to sources that provide more details on the data contract.", + "items": { + "$ref": "#/$defs/AuthoritativeDefinition" + } + }, + "description": { + "$ref": "#/$defs/Description" + }, + "customProperties": { + "type": "array", + "description": "A list of key/value pairs for custom properties.", + "items": { + "$ref": "#/$defs/CustomProperty" + } + }, + "tags": { + "$ref": "#/$defs/Tags" + }, + "inputPorts": { + "type": "array", + "description": "List of objects describing an input port. You need at least one as a data product needs to get data somewhere.", + "items": { + "$ref": "#/$defs/InputPort" + } + }, + "outputPorts": { + "type": "array", + "description": "List of objects describing an output port. You need at least one, as a data product without output is useless.", + "items": { + "$ref": "#/$defs/OutputPort" + } + }, + "managementPorts": { + "type": "array", + "description": "Management ports define access points for managing the data product.", + "items": { + "$ref": "#/$defs/ManagementPort" + } + }, + "support": { + "type": "array", + "description": "Support and communication channels.", + "items": { + "$ref": "#/$defs/Support" + } + }, + "team": { + "$ref": "#/$defs/Team" + }, + "productCreatedTs": { + "type": "string", + "format": "date-time", + "description": "Timestamp in UTC of when the data product was created, using ISO 8601." + } + }, + "$defs": { + "Tags": { + "type": "array", + "description": "A list of tags that may be assigned to the elements (object or property); the tags keyword may appear at any level. Tags may be used to better categorize an element. For example, `finance`, `sensitive`, `employee_record`.", + "examples": ["finance", "sensitive", "employee_record"], + "items": { + "type": "string" + } + }, + "Description": { + "type": "object", + "description": "Object containing the descriptions.", + "additionalProperties": false, + "properties": { + "purpose": { + "type": "string", + "description": "Intended purpose for the provided data." + }, + "limitations": { + "type": "string", + "description": "Technical, compliance, and legal limitations for data use." + }, + "usage": { + "type": "string", + "description": "Recommended usage of the data." + }, + "authoritativeDefinitions": { + "type": "array", + "description": "List of links to sources that provide more details on the data contract.", + "items": { + "$ref": "#/$defs/AuthoritativeDefinition" + } + }, + "customProperties": { + "type": "array", + "description": "A list of key/value pairs for custom properties.", + "items": { + "$ref": "#/$defs/CustomProperty" + } + } + } + }, + "CustomProperty": { + "type": "object", + "description": "A key/value pair for custom properties.", + "additionalProperties": false, + "properties": { + "property": { + "type": "string", + "description": "The name of the key. Names should be in camel case, the same as if they were permanent properties in the contract." + }, + "value": { + "description": "The value of the key." + }, + "description": { + "type": "string", + "description": "Optional description." + } + }, + "required": ["property", "value"] + }, + "AuthoritativeDefinition": { + "type": "object", + "description": "A type/link pair for authoritative definitions.", + "additionalProperties": false, + "properties": { + "type": { + "type": "string", + "description": "Type of definition for authority.", + "examples": ["businessDefinition", "transformationImplementation", "videoTutorial", "tutorial", "implementation"] + }, + "url": { + "type": "string", + "format": "uri", + "description": "URL to the authority." + }, + "description": { + "type": "string", + "description": "Optional description." + } + }, + "required": ["type", "url"] + }, + "InputPort": { + "type": "object", + "description": "An input port describing expectations.", + "additionalProperties": false, + "properties": { + "name": { + "type": "string", + "description": "Name of the input port." + }, + "version": { + "type": "string", + "description": "Version of the input port." + }, + "contractId": { + "type": "string", + "description": "Contract ID for the input port." + }, + "tags": { + "$ref": "#/$defs/Tags" + }, + "customProperties": { + "type": "array", + "description": "Custom properties block.", + "items": { + "$ref": "#/$defs/CustomProperty" + } + }, + "authoritativeDefinitions": { + "type": "array", + "description": "Authoritative definitions block.", + "items": { + "$ref": "#/$defs/AuthoritativeDefinition" + } + } + }, + "required": ["name", "version", "contractId"] + }, + "OutputPort": { + "type": "object", + "description": "An output port describing promises.", + "additionalProperties": false, + "properties": { + "name": { + "type": "string", + "description": "Name of the output port." + }, + "description": { + "type": "string", + "description": "Human readable short description of the output port." + }, + "type": { + "type": "string", + "description": "There can be different types of output ports, each automated and handled differently. Here you can indicate the type." + }, + "version": { + "type": "string", + "description": "For each version, a different instance of the output port is listed. The combination of the name and version is the key." + }, + "contractId": { + "type": "string", + "description": "Contract ID for the output port." + }, + "sbom": { + "type": "array", + "description": "The SBOM can/should be at the version level.", + "items": { + "$ref": "#/$defs/SBOM" + } + }, + "inputContracts": { + "type": "array", + "description": "Dependencies or input contracts.", + "items": { + "$ref": "#/$defs/InputContract" + } + }, + "tags": { + "$ref": "#/$defs/Tags" + }, + "customProperties": { + "type": "array", + "description": "Custom properties block.", + "items": { + "$ref": "#/$defs/CustomProperty" + } + }, + "authoritativeDefinitions": { + "type": "array", + "description": "Authoritative definitions block.", + "items": { + "$ref": "#/$defs/AuthoritativeDefinition" + } + } + }, + "required": ["name", "version"] + }, + "SBOM": { + "type": "object", + "description": "Software Bill of Materials.", + "additionalProperties": false, + "properties": { + "type": { + "type": "string", + "default": "external", + "description": "Type of SBOM." + }, + "url": { + "type": "string", + "format": "uri", + "description": "URL to the SBOM." + } + }, + "required": ["url"] + }, + "InputContract": { + "type": "object", + "description": "Input contract dependency.", + "additionalProperties": false, + "properties": { + "id": { + "type": "string", + "description": "Contract ID or contractId." + }, + "version": { + "type": "string", + "description": "Version of the input contract." + } + }, + "required": ["id", "version"] + }, + "ManagementPort": { + "type": "object", + "description": "Management port for managing the data product.", + "additionalProperties": false, + "properties": { + "name": { + "type": "string", + "description": "Endpoint identifier or unique name." + }, + "content": { + "type": "string", + "description": "Content type.", + "examples": ["discoverability", "observability", "control", "dictionary"] + }, + "type": { + "type": "string", + "default": "rest", + "description": "Type: can be `rest` or `topic`. Default is `rest`.", + "examples": ["rest", "topic"] + }, + "url": { + "type": "string", + "format": "uri", + "description": "URL to access the endpoint." + }, + "channel": { + "type": "string", + "description": "Channel to communicate with the data product." + }, + "description": { + "type": "string", + "description": "Purpose and usage." + }, + "tags": { + "$ref": "#/$defs/Tags" + }, + "customProperties": { + "type": "array", + "description": "Custom properties block.", + "items": { + "$ref": "#/$defs/CustomProperty" + } + }, + "authoritativeDefinitions": { + "type": "array", + "description": "Authoritative definitions block.", + "items": { + "$ref": "#/$defs/AuthoritativeDefinition" + } + } + }, + "required": ["name", "content"] + }, + "Support": { + "type": "object", + "description": "Support channel.", + "additionalProperties": false, + "properties": { + "channel": { + "type": "string", + "description": "Channel name or identifier." + }, + "url": { + "type": "string", + "format": "uri", + "description": "Access URL using normal URL scheme (https, mailto, etc.)." + }, + "description": { + "type": "string", + "description": "Description of the channel, free text." + }, + "tool": { + "type": "string", + "description": "Name of the tool.", + "examples": ["email", "slack", "teams", "discord", "ticket", "other"] + }, + "scope": { + "type": "string", + "description": "Scope can be: `interactive`, `announcements`, `issues`.", + "examples": ["interactive", "announcements", "issues"] + }, + "invitationUrl": { + "type": "string", + "format": "uri", + "description": "Some tools uses invitation URL for requesting or subscribing. Follows the URL scheme." + }, + "tags": { + "$ref": "#/$defs/Tags" + }, + "customProperties": { + "type": "array", + "description": "Custom properties block.", + "items": { + "$ref": "#/$defs/CustomProperty" + } + }, + "authoritativeDefinitions": { + "type": "array", + "description": "Authoritative definitions block.", + "items": { + "$ref": "#/$defs/AuthoritativeDefinition" + } + } + }, + "required": ["channel", "url"] + }, + "TeamMember": { + "type": "object", + "description": "Team member information.", + "additionalProperties": false, + "properties": { + "username": { + "type": "string", + "description": "The user's username or email." + }, + "name": { + "type": "string", + "description": "The user's name." + }, + "description": { + "type": "string", + "description": "The user's description." + }, + "role": { + "type": "string", + "description": "The user's job role; Examples might be owner, data steward. There is no limit on the role." + }, + "dateIn": { + "type": "string", + "format": "date", + "description": "The date when the user joined the team." + }, + "dateOut": { + "type": "string", + "format": "date", + "description": "The date when the user ceased to be part of the team." + }, + "replacedByUsername": { + "type": "string", + "description": "The username of the user who replaced the previous user." + }, + "tags": { + "$ref": "#/$defs/Tags" + }, + "customProperties": { + "type": "array", + "description": "Custom properties block.", + "items": { + "$ref": "#/$defs/CustomProperty" + } + }, + "authoritativeDefinitions": { + "type": "array", + "description": "Authoritative definitions block.", + "items": { + "$ref": "#/$defs/AuthoritativeDefinition" + } + } + }, + "required": ["username"] + }, + "Team": { + "type": "object", + "description": "Team information.", + "additionalProperties": false, + "properties": { + "name": { + "type": "string", + "description": "Team name." + }, + "description": { + "type": "string", + "description": "Team description." + }, + "members": { + "type": "array", + "description": "List of members.", + "items": { + "$ref": "#/$defs/TeamMember" + } + }, + "tags": { + "$ref": "#/$defs/Tags" + }, + "customProperties": { + "type": "array", + "description": "Custom properties block.", + "items": { + "$ref": "#/$defs/CustomProperty" + } + }, + "authoritativeDefinitions": { + "type": "array", + "description": "Authoritative definitions block.", + "items": { + "$ref": "#/$defs/AuthoritativeDefinition" + } + } + } + } + } +} \ No newline at end of file diff --git a/fluid_build/providers/odps_standard/schemas/odps-product-v1.1.0.json b/fluid_build/providers/odps_standard/schemas/odps-product-v1.1.0.json new file mode 100644 index 00000000..3384d244 --- /dev/null +++ b/fluid_build/providers/odps_standard/schemas/odps-product-v1.1.0.json @@ -0,0 +1,717 @@ +{ + "$schema": "https://json-schema.org/draft/2019-09/schema", + "title": "Open Data Product Standard (ODPS)", + "description": "An open data product standard descriptor to enable defining data products.", + "type": "object", + "required": ["apiVersion", "kind", "id"], + "additionalProperties": false, + "properties": { + "apiVersion": { + "type": "string", + "default": "v1.1.0", + "description": "Version of the standard used to build data product. Default value is v1.1.0.", + "enum": ["v1.1.0", "v1.0.0", "v0.9.0"] + }, + "kind": { + "type": "string", + "default": "DataProduct", + "description": "The kind of file this is. Valid value is `DataProduct`.", + "enum": ["DataProduct"] + }, + "id": { + "type": "string", + "description": "A unique identifier used to reduce the risk of dataset name collisions, such as a UUID." + }, + "name": { + "type": "string", + "description": "Name of the data product." + }, + "deprecated": { + "type": "boolean", + "description": "Indicates this data product is deprecated and should not be used in new implementations. Defaults to false.", + "default": false + }, + "synonyms": { + "$ref": "#/$defs/Synonyms" + }, + "version": { + "type": "string", + "description": "Current version of the data product. Not required, but highly recommended." + }, + "status": { + "type": "string", + "description": "Current status of the data product.", + "examples": ["proposed", "draft", "active", "deprecated", "retired"] + }, + "domain": { + "type": "string", + "description": "Business domain" + }, + "type": { + "type": "string", + "description": "Architectural type of the data product. Common values: `sourceAligned`, `aggregate`, `consumerAligned`. Organizations may define custom types." + }, + "context": { + "$ref": "#/$defs/Context" + }, + "tenant": { + "type": "string", + "description": "Organization identifier" + }, + "authoritativeDefinitions": { + "type": "array", + "description": "List of links to sources that provide more details on the data contract.", + "items": { + "$ref": "#/$defs/AuthoritativeDefinition" + } + }, + "description": { + "$ref": "#/$defs/Description" + }, + "customProperties": { + "type": "array", + "description": "A list of key/value pairs for custom properties.", + "items": { + "$ref": "#/$defs/CustomProperty" + } + }, + "tags": { + "$ref": "#/$defs/Tags" + }, + "inputPorts": { + "type": "array", + "description": "List of objects describing an input port. You need at least one as a data product needs to get data somewhere.", + "items": { + "$ref": "#/$defs/InputPort" + } + }, + "outputPorts": { + "type": "array", + "description": "List of objects describing an output port. You need at least one, as a data product without output is useless.", + "items": { + "$ref": "#/$defs/OutputPort" + } + }, + "managementPorts": { + "type": "array", + "description": "Management ports define access points for managing the data product.", + "items": { + "$ref": "#/$defs/ManagementPort" + } + }, + "support": { + "type": "array", + "description": "Support and communication channels.", + "items": { + "$ref": "#/$defs/Support" + } + }, + "team": { + "$ref": "#/$defs/Team" + }, + "productCreatedTs": { + "type": "string", + "format": "date-time", + "description": "Timestamp in UTC of when the data product was created, using ISO 8601." + } + }, + "$defs": { + "Tags": { + "type": "array", + "description": "A list of tags that may be assigned to the elements (object or property); the tags keyword may appear at any level. Tags may be used to better categorize an element. For example, `finance`, `sensitive`, `employee_record`.", + "examples": ["finance", "sensitive", "employee_record"], + "items": { + "type": "string" + } + }, + "Description": { + "type": "object", + "description": "Object containing the descriptions.", + "additionalProperties": false, + "properties": { + "purpose": { + "type": "string", + "description": "Intended purpose for the provided data." + }, + "limitations": { + "type": "string", + "description": "Technical, compliance, and legal limitations for data use." + }, + "usage": { + "type": "string", + "description": "Recommended usage of the data." + }, + "authoritativeDefinitions": { + "type": "array", + "description": "List of links to sources that provide more details on the data contract.", + "items": { + "$ref": "#/$defs/AuthoritativeDefinition" + } + }, + "customProperties": { + "type": "array", + "description": "A list of key/value pairs for custom properties.", + "items": { + "$ref": "#/$defs/CustomProperty" + } + } + } + }, + "CustomProperty": { + "type": "object", + "description": "A key/value pair for custom properties.", + "additionalProperties": false, + "properties": { + "id": { + "type": "string", + "description": "A unique identifier for the element used to create stable, refactor-safe references. Recommended for elements that will be referenced." + }, + "property": { + "type": "string", + "description": "The name of the key. Names should be in camel case, the same as if they were permanent properties in the contract." + }, + "value": { + "description": "The value of the key." + }, + "description": { + "type": "string", + "description": "Optional description." + }, + "vendor": { + "type": "string", + "description": "Identifies the vendor, provider, or external system associated with this custom property. SHOULD be a stable, lowercase identifier matching ^[a-z0-9][a-z0-9-]*$ (e.g. confluent, zeenea, atlan, soda). Tools MUST preserve unknown vendor values.", + "examples": ["confluent", "zeenea", "atlan", "soda"] + } + }, + "required": ["property", "value"] + }, + "Synonyms": { + "type": "array", + "description": "A list of alternative names for the object. See RFC 0041.", + "items": { + "$ref": "#/$defs/Synonym" + } + }, + "Synonym": { + "type": "object", + "description": "An alternative name for a named object, helping catalogs, AI/LLM tools, and natural language interfaces resolve business vocabulary to the underlying object. See RFC 0041.", + "additionalProperties": false, + "properties": { + "id": { + "type": "string", + "description": "A unique identifier for the synonym, useful when referencing or deduplicating synonyms across tools. Recommended for elements that will be referenced." + }, + "synonym": { + "type": "string", + "description": "The synonymous term." + }, + "description": { + "type": "string", + "description": "Short human-readable note about when or why this synonym is used." + }, + "locale": { + "type": "string", + "description": "BCP 47 language tag (e.g., `en-US`, `fr-FR`) when the synonym is language-specific." + }, + "source": { + "type": "string", + "description": "Origin of the synonym (e.g., `glossary`, `finance-team`, `legacy-system`)." + }, + "status": { + "type": "string", + "description": "Lifecycle status of the synonym (e.g., `active`, `deprecated`)." + }, + "customProperties": { + "type": "array", + "description": "Custom properties block.", + "items": { + "$ref": "#/$defs/CustomProperty" + } + } + }, + "required": ["synonym"] + }, + "AuthoritativeDefinition": { + "type": "object", + "description": "A type/link pair for authoritative definitions.", + "additionalProperties": false, + "properties": { + "id": { + "type": "string", + "description": "A unique identifier for the element used to create stable, refactor-safe references. Recommended for elements that will be referenced." + }, + "type": { + "type": "string", + "description": "Type of definition for authority.", + "examples": ["businessDefinition", "canonicalUrl", "glossary", "implementation", "ontology", "taxonomy", "transformationImplementation", "tutorial", "videoTutorial"] + }, + "url": { + "type": "string", + "format": "uri", + "description": "URL to the authority." + }, + "description": { + "type": "string", + "description": "Optional description." + } + }, + "required": ["type", "url"] + }, + "InputPort": { + "type": "object", + "description": "An input port describing expectations.", + "additionalProperties": false, + "properties": { + "deprecated": { + "type": "boolean", + "description": "Indicates this input port is deprecated and should not be used in new implementations. Defaults to false.", + "default": false + }, + "id": { + "type": "string", + "description": "A unique identifier for the element used to create stable, refactor-safe references. Recommended for elements that will be referenced." + }, + "name": { + "type": "string", + "description": "Name of the input port." + }, + "version": { + "type": "string", + "description": "Version of the input port." + }, + "contractId": { + "type": "string", + "description": "Contract ID for the input port." + }, + "tags": { + "$ref": "#/$defs/Tags" + }, + "customProperties": { + "type": "array", + "description": "Custom properties block.", + "items": { + "$ref": "#/$defs/CustomProperty" + } + }, + "authoritativeDefinitions": { + "type": "array", + "description": "Authoritative definitions block.", + "items": { + "$ref": "#/$defs/AuthoritativeDefinition" + } + } + }, + "required": ["name"] + }, + "OutputPort": { + "type": "object", + "description": "An output port describing promises.", + "additionalProperties": false, + "properties": { + "deprecated": { + "type": "boolean", + "description": "Indicates this output port is deprecated and should not be used in new implementations. Defaults to false.", + "default": false + }, + "id": { + "type": "string", + "description": "A unique identifier for the element used to create stable, refactor-safe references. Recommended for elements that will be referenced." + }, + "name": { + "type": "string", + "description": "Name of the output port." + }, + "description": { + "type": "string", + "description": "Human readable short description of the output port." + }, + "type": { + "type": "string", + "description": "There can be different types of output ports, each automated and handled differently. Here you can indicate the type." + }, + "version": { + "type": "string", + "description": "For each version, a different instance of the output port is listed. The combination of the name and version is the key." + }, + "contractId": { + "type": "string", + "description": "Contract ID for the output port." + }, + "sbom": { + "type": "array", + "description": "The SBOM can/should be at the version level.", + "items": { + "$ref": "#/$defs/SBOM" + } + }, + "inputContracts": { + "type": "array", + "description": "Dependencies or input contracts.", + "items": { + "$ref": "#/$defs/InputContract" + } + }, + "synonyms": { + "$ref": "#/$defs/Synonyms" + }, + "tags": { + "$ref": "#/$defs/Tags" + }, + "customProperties": { + "type": "array", + "description": "Custom properties block.", + "items": { + "$ref": "#/$defs/CustomProperty" + } + }, + "authoritativeDefinitions": { + "type": "array", + "description": "Authoritative definitions block.", + "items": { + "$ref": "#/$defs/AuthoritativeDefinition" + } + }, + "context": { + "$ref": "#/$defs/Context" + } + }, + "required": ["name"] + }, + "SBOM": { + "type": "object", + "description": "Software Bill of Materials.", + "additionalProperties": false, + "properties": { + "id": { + "type": "string", + "description": "A unique identifier for the element used to create stable, refactor-safe references. Recommended for elements that will be referenced." + }, + "type": { + "type": "string", + "default": "external", + "description": "Type of SBOM." + }, + "url": { + "type": "string", + "format": "uri", + "description": "URL to the SBOM." + }, + "tags": { "$ref": "#/$defs/Tags" }, + "customProperties": { + "type": "array", + "description": "Custom properties block.", + "items": { "$ref": "#/$defs/CustomProperty" } + }, + "authoritativeDefinitions": { + "type": "array", + "description": "Authoritative definitions block.", + "items": { "$ref": "#/$defs/AuthoritativeDefinition" } + } + } + }, + "InputContract": { + "type": "object", + "description": "Input contract dependency.", + "additionalProperties": false, + "properties": { + "id": { + "type": "string", + "description": "Contract ID or contractId." + }, + "version": { + "type": "string", + "description": "Version of the input contract." + } + }, + "required": ["id", "version"] + }, + "ManagementPort": { + "type": "object", + "description": "Management port for managing the data product.", + "additionalProperties": false, + "properties": { + "deprecated": { + "type": "boolean", + "description": "Indicates this management port is deprecated and should not be used in new implementations. Defaults to false.", + "default": false + }, + "id": { + "type": "string", + "description": "A unique identifier for the element used to create stable, refactor-safe references. Recommended for elements that will be referenced." + }, + "name": { + "type": "string", + "description": "Endpoint identifier or unique name." + }, + "content": { + "type": "string", + "description": "Content type.", + "examples": ["discoverability", "observability", "control", "dictionary"] + }, + "type": { + "type": "string", + "default": "rest", + "description": "Type: can be `rest` or `topic`. Default is `rest`.", + "examples": ["rest", "topic"] + }, + "url": { + "type": "string", + "format": "uri", + "description": "URL to access the endpoint." + }, + "channel": { + "type": "string", + "description": "Channel to communicate with the data product." + }, + "description": { + "type": "string", + "description": "Purpose and usage." + }, + "tags": { + "$ref": "#/$defs/Tags" + }, + "customProperties": { + "type": "array", + "description": "Custom properties block.", + "items": { + "$ref": "#/$defs/CustomProperty" + } + }, + "authoritativeDefinitions": { + "type": "array", + "description": "Authoritative definitions block.", + "items": { + "$ref": "#/$defs/AuthoritativeDefinition" + } + } + }, + "required": ["name", "content"] + }, + "Support": { + "type": "object", + "description": "Support channel.", + "additionalProperties": false, + "properties": { + "id": { + "type": "string", + "description": "A unique identifier for the element used to create stable, refactor-safe references. Recommended for elements that will be referenced." + }, + "channel": { + "type": "string", + "description": "Channel name or identifier." + }, + "url": { + "type": "string", + "format": "uri", + "description": "Access URL using normal URL scheme (https, mailto, etc.)." + }, + "description": { + "type": "string", + "description": "Description of the channel, free text." + }, + "tool": { + "type": "string", + "description": "Name of the tool.", + "examples": ["email", "slack", "teams", "discord", "ticket", "other"] + }, + "scope": { + "type": "string", + "description": "Scope can be: `interactive`, `announcements`, `issues`.", + "examples": ["interactive", "announcements", "issues"] + }, + "invitationUrl": { + "type": "string", + "format": "uri", + "description": "Some tools uses invitation URL for requesting or subscribing. Follows the URL scheme." + }, + "tags": { + "$ref": "#/$defs/Tags" + }, + "customProperties": { + "type": "array", + "description": "Custom properties block.", + "items": { + "$ref": "#/$defs/CustomProperty" + } + }, + "authoritativeDefinitions": { + "type": "array", + "description": "Authoritative definitions block.", + "items": { + "$ref": "#/$defs/AuthoritativeDefinition" + } + } + }, + "required": ["channel", "url"] + }, + "TeamMember": { + "type": "object", + "description": "Team member information.", + "additionalProperties": false, + "properties": { + "id": { + "type": "string", + "description": "A unique identifier for the element used to create stable, refactor-safe references. Recommended for elements that will be referenced." + }, + "username": { + "type": "string", + "description": "The user's username or email." + }, + "name": { + "type": "string", + "description": "The user's name." + }, + "description": { + "type": "string", + "description": "The user's description." + }, + "role": { + "type": "string", + "description": "The user's job role; Examples might be owner, data steward. There is no limit on the role." + }, + "dateIn": { + "type": "string", + "format": "date", + "description": "The date when the user joined the team." + }, + "dateOut": { + "type": "string", + "format": "date", + "description": "The date when the user ceased to be part of the team." + }, + "replacedByUsername": { + "type": "string", + "description": "The username of the user who replaced the previous user." + }, + "tags": { + "$ref": "#/$defs/Tags" + }, + "customProperties": { + "type": "array", + "description": "Custom properties block.", + "items": { + "$ref": "#/$defs/CustomProperty" + } + }, + "authoritativeDefinitions": { + "type": "array", + "description": "Authoritative definitions block.", + "items": { + "$ref": "#/$defs/AuthoritativeDefinition" + } + } + }, + "required": ["username"] + }, + "Context": { + "type": "object", + "description": "AI and semantic context block (RFC-0038). Provides structured guidance for AI agents, LLMs, BI tools, and semantic layer platforms. Optional and additive. Applicable at the data product (top level) and output port levels. Input ports refer to the linked ODCS contract's context instead.", + "additionalProperties": false, + "properties": { + "instructions": { + "type": "string", + "description": "Natural language guidance for AI agents and tools on how to use this entity. Equivalent to a system prompt scoped to this level." + }, + "verifiedStatements": { + "type": "array", + "description": "Canonical business questions, each with an optional curated answer. Entries with `answer` should be returned verbatim by AI agents when a query is semantically close; entries without `answer` serve as sample questions for text-to-SQL priming and disambiguation.", + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "id": { + "type": "string", + "description": "Stable identifier for the entry, useful when referencing or deduplicating verified statements across tools." + }, + "question": { + "type": "string", + "description": "The canonical question." + }, + "answer": { + "type": "string", + "description": "The expected response or result description. Optional — omit to signal an unanswered sample question." + }, + "authoritativeDefinitions": { + "type": "array", + "items": { "$ref": "#/$defs/AuthoritativeDefinition" } + }, + "tags": { "$ref": "#/$defs/Tags" }, + "customProperties": { + "type": "array", + "items": { "$ref": "#/$defs/CustomProperty" } + } + }, + "required": ["question"] + } + }, + "constraints": { + "type": "array", + "description": "Negative guidance: what AI agents must NOT do with this entity.", + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "id": { + "type": "string", + "description": "Stable identifier for the constraint." + }, + "constraint": { + "type": "string", + "description": "The constraint text (negative guidance for AI agents)." + }, + "authoritativeDefinitions": { + "type": "array", + "items": { "$ref": "#/$defs/AuthoritativeDefinition" } + }, + "tags": { "$ref": "#/$defs/Tags" }, + "customProperties": { + "type": "array", + "items": { "$ref": "#/$defs/CustomProperty" } + } + }, + "required": ["constraint"] + } + } + } + }, + "Team": { + "type": "object", + "description": "Team information.", + "additionalProperties": false, + "properties": { + "name": { + "type": "string", + "description": "Team name." + }, + "description": { + "type": "string", + "description": "Team description." + }, + "members": { + "type": "array", + "description": "List of members.", + "items": { + "$ref": "#/$defs/TeamMember" + } + }, + "tags": { + "$ref": "#/$defs/Tags" + }, + "customProperties": { + "type": "array", + "description": "Custom properties block.", + "items": { + "$ref": "#/$defs/CustomProperty" + } + }, + "authoritativeDefinitions": { + "type": "array", + "description": "Authoritative definitions block.", + "items": { + "$ref": "#/$defs/AuthoritativeDefinition" + } + } + } + } + } +} \ No newline at end of file diff --git a/fluid_build/providers/odps_standard/validation.py b/fluid_build/providers/odps_standard/validation.py new file mode 100644 index 00000000..f0b46b8b --- /dev/null +++ b/fluid_build/providers/odps_standard/validation.py @@ -0,0 +1,108 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +"""Bitol ODPS JSON Schema validation (version-aware) + round-trip diff. + +Two ODPS schema versions are vendored side by side: + +- ``v1.1.0`` — current upstream. Additive over v1.0.0: optional top-level + ``type`` / ``deprecated`` / ``context`` / ``synonyms``; ``status`` no + longer required; per-element ``id``; ``deprecated``/``context``/``synonyms`` + on ports; ``vendor`` on custom properties; SBOM ``url`` optional. +- ``v1.0.0`` — previous. Kept so a product that *declares* ``v1.0.0`` is + validated against exactly the spec it claims. + +Mirrors :mod:`fluid_build.providers.odcs.validation`; see there for the +version-resolution rules. Emit default stays at :data:`DEFAULT_API_VERSION`; +opt into the newer version with ``ODPS_API_VERSION`` or +``BitolOdpsProvider(api_version=...)``. +""" + +from __future__ import annotations + +import json +import logging +from collections.abc import Mapping +from pathlib import Path +from typing import Any, Dict, Optional, Tuple + +from fluid_build.providers.base import ProviderError + +# Reuse the deep-diff implementation and the version normaliser from the odcs +# validation module — the two providers share the same notion of "structural +# equality" and the same ``vX.Y.Z`` spelling rules. +from fluid_build.providers.odcs.validation import ( # noqa: F401 + api_version_tuple, + json_instance, + normalize_api_version, + roundtrip_check, +) + +LOG = logging.getLogger(__name__) + +SUPPORTED_API_VERSIONS: Tuple[str, ...] = ("v1.1.0", "v1.0.0") # newest first +DEFAULT_API_VERSION = "v1.0.0" # conservative emit default + +_SCHEMA_DIR = Path(__file__).parent / "schemas" +_SCHEMA_FILES: Dict[str, str] = {v: f"odps-product-{v}.json" for v in SUPPORTED_API_VERSIONS} +_SCHEMA_CACHE: Dict[str, Optional[Dict[str, Any]]] = {} + + +def resolve_schema_version(doc: Mapping[str, Any]) -> str: + """Declared+vendored ``apiVersion`` → that; otherwise newest vendored.""" + declared = normalize_api_version(doc.get("apiVersion") if isinstance(doc, Mapping) else None) + if declared in _SCHEMA_FILES: + return declared + return SUPPORTED_API_VERSIONS[0] + + +def load_schema(api_version: Optional[str] = None) -> Optional[Dict[str, Any]]: + """Load a vendored ODPS JSON Schema (default :data:`DEFAULT_API_VERSION`).""" + version = normalize_api_version(api_version) or DEFAULT_API_VERSION + if version not in _SCHEMA_FILES: + raise ProviderError( + f"Unsupported ODPS apiVersion {version!r}; " + f"vendored: {', '.join(SUPPORTED_API_VERSIONS)}" + ) + if version in _SCHEMA_CACHE: + return _SCHEMA_CACHE[version] + + schema_path = _SCHEMA_DIR / _SCHEMA_FILES[version] + schema: Optional[Dict[str, Any]] = None + if not schema_path.exists(): + LOG.warning("ODPS schema not found: %s", schema_path) + else: + try: + with open(schema_path) as f: + schema = json.load(f) + except Exception as exc: # pragma: no cover - defensive + LOG.error("Failed to load ODPS schema %s: %s", schema_path, exc) + _SCHEMA_CACHE[version] = schema + return schema + + +def validate(odps: Mapping[str, Any], schema: Optional[Mapping[str, Any]] = None) -> None: + """Validate an ODPS product against the JSON Schema. + + When ``schema`` is omitted the version is resolved from the document's + ``apiVersion`` so a v1.1.0 product is never judged by the v1.0.0 schema. + """ + try: + import jsonschema + except ImportError: + LOG.warning("jsonschema not installed, skipping ODPS validation") + return + if schema is None: + schema = load_schema(resolve_schema_version(odps)) + if schema is None: + LOG.warning("ODPS schema not available, skipping validation") + return + try: + # YAML dates are validated as the ISO strings JSON would carry. + jsonschema.validate(instance=json_instance(odps), schema=schema) + except jsonschema.ValidationError as exc: + raise ProviderError(f"ODPS validation failed: {exc.message}") from exc diff --git a/pyproject.toml b/pyproject.toml index b3ae7f28..44b37d0a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,6 +38,7 @@ dependencies = [ "typing-extensions>=4.8", "jinja2>=3.1", "httpx>=0.27", # For FluidCommandCenter catalog provider + "requests>=2.31", # Data Mesh Manager provider (`fluid dmm publish`) # Optional UX nicety on Windows; no-op elsewhere "colorama>=0.4.6; platform_system == 'Windows'", ] @@ -48,6 +49,25 @@ sdk = [ "fluid-provider-sdk>=0.1.0", ] +# Optional ODCS tooling borrowed from the ecosystem: +# - vowl (MIT, Bitol-listed vendor): second-pass native parser on every +# emitted ODCS contract via ``ODCS_VOWL_VALIDATE=true`` / ODPS strict mode. +# Skipped with a warning for apiVersions newer than the installed vowl +# (vowl>=0.0.6 supports ODCS v3.2.0). +# - open-data-contract-standard (innoq / datacontract.com): pydantic model of +# the full ODCS surface, tracking the spec version; powers +# ``OdcsProvider.to_model()`` for typed access + readable field errors. +# - deepdiff: backend for ``OdcsProvider.roundtrip_check()``. +# Every vowl and open-data-contract-standard release requires Python >= 3.10, +# so they are gated like the gcp/snowflake 3.10+ extras: on Python 3.9 the +# extra still resolves and installs only deepdiff (the vowl second pass is +# then skipped and ``to_model()`` reports the missing package). +odcs-strict = [ + "vowl>=0.0.2 ; python_version >= '3.10'", + "open-data-contract-standard>=3.2.0,<4 ; python_version >= '3.10'", + "deepdiff>=6", +] + # Local provider (DuckDB) local = [ "duckdb>=1.0", @@ -111,6 +131,7 @@ dev = [ "types-PyYAML>=6.0.12", "black==25.11.0", "requests>=2.32.5", + "deepdiff>=6", # roundtrip_check() backend (ODCS/ODPS tests) ] # One-shot install for contributors diff --git a/scripts/dmm_live_e2e.sh b/scripts/dmm_live_e2e.sh new file mode 100755 index 00000000..c91200fb --- /dev/null +++ b/scripts/dmm_live_e2e.sh @@ -0,0 +1,186 @@ +#!/usr/bin/env bash +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Live Entropy Data CE (Data Mesh Manager) end-to-end check for forge-cli's +# ODCS/ODPS export path. Opt-in and manual — never run by CI. +# +# What it verifies, against a real Entropy CE instance: +# 1. `fluid publish --catalog datamesh-manager` (CatalogTarget path) +# 2. `fluid dmm publish --with-contract` (direct DMM provider path) +# ...each at the default ODCS emit (v3.1.0) and with ODCS_API_VERSION=v3.2.0, +# asserting via the DMM REST API that the product + one contract per port +# landed, and that each contract carries the expected apiVersion. +# +# Last verified green: Entropy CE 2.1.x, both paths, v3.1.0 and v3.2.0. +# +# Prereqs: docker + compose plugin, python3, git, curl, and forge-cli installed +# from your checkout (`pip install -e .`). +# +# Usage: +# scripts/dmm_live_e2e.sh up # start Entropy CE (compose), wait until healthy +# scripts/dmm_live_e2e.sh run # needs DMM_API_KEY exported (see bootstrap below) +# scripts/dmm_live_e2e.sh down # stop + remove the stack (keeps images) +# +# Environment: +# DMM_API_KEY required for `run` — an organisation API key with write access +# DMM_API_URL default http://localhost:8081 +# CE_DIR where the entropy-data-ce compose repo is cloned +# FLUID_CHECKOUT forge-cli checkout (default: git toplevel of this script) +# DMM_PRODUCT_SPEC if set, passed as --data-product-spec on path 2 +# (default: unset, i.e. exercise fluid's own default — ODPS) +# +# Side effects: none on your home directory. The fluid commands run with HOME +# pointed at a private temp dir that holds the generated +# ~/.fluid/config.yaml, so your real ~/.fluid/config.yaml / ~/.fluidrc.yaml +# are never read or modified. The API key is passed to curl through a 0600 +# header file, never on the command line. +set -euo pipefail + +CE_REPO=https://github.com/entropy-data/entropy-data-ce.git +CE_DIR="${CE_DIR:-$HOME/.cache/entropy-data-ce}" +DMM_API_URL="${DMM_API_URL:-http://localhost:8081}" +FLUID_CHECKOUT="${FLUID_CHECKOUT:-$(git -C "$(dirname "$0")" rev-parse --show-toplevel 2>/dev/null || pwd)}" +FIXTURE="$FLUID_CHECKOUT/tests/fixtures/fluid/contract-multi-expose.fluid.yaml" +PRODUCT_ID="commerce.orders-product" +PORTS=(daily_orders hourly_orders) +# Private, unpredictable work dir (mktemp -d creates it 0700 and fails if it +# already exists — no symlink games in a shared /tmp). +WORK=$(mktemp -d "${TMPDIR:-/tmp}/dmm_live_e2e.XXXXXXXX") +trap 'rm -rf "$WORK"' EXIT +# Isolated HOME for the fluid commands; the generated config lives here. +FLUID_HOME="$WORK/home" +FLUID_CFG="$FLUID_HOME/.fluid/config.yaml" +HDR_FILE="$WORK/api-key.header" +# Keep `pip install --user` installs importable under the isolated HOME. +PY_USER_BASE="${PYTHONUSERBASE:-$(python3 -m site --user-base 2>/dev/null || true)}" + +log() { printf '\n\033[1;34m== %s\033[0m\n' "$*"; } +ok() { printf ' \033[1;32m✓\033[0m %s\n' "$*"; } +fail() { printf ' \033[1;31m✗ %s\033[0m\n' "$*"; FAILED=1; } +FAILED=0 + +write_api_key_header() { # header file read by `curl -H @file` (0600, printf is a builtin) + : "${DMM_API_KEY:?export DMM_API_KEY first}" + (umask 077; printf 'x-api-key: %s\n' "$DMM_API_KEY" > "$HDR_FILE") +} + +api() { # api → prints HTTP code; body in $WORK/body.json + local m=$1 p=$2 + [ -f "$HDR_FILE" ] || write_api_key_header + curl -sS -o "$WORK/body.json" -w '%{http_code}' -X "$m" \ + -H @"$HDR_FILE" \ + -H 'Accept: application/json' "$DMM_API_URL$p" +} + +fluid_isolated() { # run fluid with HOME pointed at the private config dir + HOME="$FLUID_HOME" PYTHONUSERBASE="$PY_USER_BASE" fluid "$@" +} + +cmd_up() { + log "Cloning/updating Entropy CE compose repo → $CE_DIR" + if [ -d "$CE_DIR/.git" ]; then git -C "$CE_DIR" pull -q; else git clone -q "$CE_REPO" "$CE_DIR"; fi + log "docker compose up (app :8081, pgvector pg16, mailhog :8025)" + docker compose -f "$CE_DIR/docker-compose.yaml" up -d + log "Waiting for the app (Spring Boot + DB init can take ~60–120s)" + for i in $(seq 1 60); do + if curl -sS -o /dev/null -w '%{http_code}' "$DMM_API_URL/" 2>/dev/null | grep -qE '^(200|302|401)$'; then ok "up after $((i*3))s"; break; fi + sleep 3; [ "$i" = 60 ] && { fail "app not reachable at $DMM_API_URL"; exit 1; } + done + cat < "$FLUID_CFG") < + local label=$1 want=$2 code + code=$(api GET "/api/dataproducts/$PRODUCT_ID") + [ "$code" = 200 ] && ok "$label: product $PRODUCT_ID → HTTP 200" || fail "$label: product GET → HTTP $code: $(head -c 300 "$WORK/body.json")" + for port in "${PORTS[@]}"; do + local cid="$PRODUCT_ID.$port" + code=$(api GET "/api/datacontracts/$cid") + if [ "$code" != 200 ]; then fail "$label: contract $cid GET → HTTP $code: $(head -c 300 "$WORK/body.json")"; continue; fi + local got; got=$(python3 -c "import json,sys;d=json.load(open(sys.argv[1]));print(d.get('apiVersion') or (d.get('spec') or {}).get('apiVersion') or '?')" "$WORK/body.json") + [ "$got" = "$want" ] && ok "$label: contract $cid → HTTP 200, apiVersion=$got" || fail "$label: contract $cid apiVersion=$got (wanted $want)" + done +} + +cleanup_product() { + api DELETE "/api/dataproducts/$PRODUCT_ID" >/dev/null 2>&1 || true + for port in "${PORTS[@]}"; do api DELETE "/api/datacontracts/$PRODUCT_ID.$port" >/dev/null 2>&1 || true; done +} + +cmd_run() { + : "${DMM_API_KEY:?export DMM_API_KEY first (see 'scripts/dmm_live_e2e.sh up' output)}" + [ -f "$FIXTURE" ] || { echo "fixture not found: $FIXTURE (set FLUID_CHECKOUT)"; exit 1; } + log "Sanity: DMM API reachable with this key" + code=$(api GET /api/dataproducts); [ "$code" = 200 ] && ok "GET /api/dataproducts → 200" || { fail "GET /api/dataproducts → $code: $(head -c 300 "$WORK/body.json")"; exit 1; } + log "fluid version"; fluid_isolated --version | head -1 + write_fluid_config + + # Path 2 exercises fluid's own default spec (ODPS) unless DMM_PRODUCT_SPEC pins one. + SPEC_FLAG=(); [ -n "${DMM_PRODUCT_SPEC:-}" ] && SPEC_FLAG=(--data-product-spec "$DMM_PRODUCT_SPEC") + + for ver in "" v3.2.0; do + label="ODCS ${ver:-v3.1.0 (default)}"; want="${ver:-v3.1.0}" + export ODCS_API_VERSION="$ver"; [ -z "$ver" ] && unset ODCS_API_VERSION + + log "[$label] path 1: fluid publish --catalog datamesh-manager" + cleanup_product + if fluid_isolated publish "$FIXTURE" --catalog datamesh-manager --format json | tee "$WORK/publish.json" \ + | python3 -c "import json,sys;r=json.load(sys.stdin);sys.exit(0 if all(x['success'] for x in r) else 1)"; then + ok "$label: fluid publish reported success"; else fail "$label: fluid publish reported failure — see $WORK/publish.json"; fi + assert_landed "$label / catalog" "$want" + + log "[$label] path 2: fluid dmm publish --with-contract (strict, fail-on-error)" + cleanup_product + if fluid_isolated dmm publish "$FIXTURE" --api-url "$DMM_API_URL" --with-contract --contract-format odcs \ + --validate-generated-contracts --validation-mode strict --fail-on-contract-error "${SPEC_FLAG[@]}"; then + ok "$label: fluid dmm publish exit 0"; else fail "$label: fluid dmm publish non-zero exit"; fi + assert_landed "$label / dmm" "$want" + done + + log "Result" + if [ "$FAILED" = 0 ]; then + echo "ALL CHECKS PASSED — Entropy CE accepted both paths at v3.1.0 and v3.2.0." + else + echo "SOME CHECKS FAILED (see ✗ lines)." + # Keep the response bodies for debugging, but never the API key. + rm -f "$HDR_FILE"; trap - EXIT; echo "Bodies kept in $WORK"; exit 1 + fi +} + +cmd_down() { docker compose -f "$CE_DIR/docker-compose.yaml" down; } + +case "${1:-}" in up) cmd_up ;; run) cmd_run ;; down) cmd_down ;; *) sed -n '16,48p' "$0"; exit 2 ;; esac diff --git a/tests/fixtures/fluid/contract-multi-expose.fluid.yaml b/tests/fixtures/fluid/contract-multi-expose.fluid.yaml new file mode 100644 index 00000000..4d7b15b6 --- /dev/null +++ b/tests/fixtures/fluid/contract-multi-expose.fluid.yaml @@ -0,0 +1,78 @@ +# Multi-expose FLUID contract — drives the Bitol ODPS export tests. +# Two output ports + one input port, with qos, owner, schema, and tags. +fluidVersion: "0.7.1" +kind: DataContract +id: commerce.orders-product +name: orders-product + +metadata: + version: "1.0.0" + name: "Orders Product" + description: "Orders data product with daily and hourly outputs" + status: active + domain: commerce + tenant: acme + tags: + - orders + - production + +owner: + team: commerce-team + name: Alice + email: alice@acme.com + role: data-owner + +exposes: + - id: daily_orders + exposeId: daily_orders + version: "1.0.0" + description: "Daily order aggregates" + binding: + platform: bigquery + location: + project: acme-prod + dataset: orders + contract: + schema: + - name: order_date + type: date + required: true + tags: [primary-key] + - name: order_count + type: bigint + required: true + - name: total_amount + type: decimal + required: true + qos: + availability: "99.9%" + freshnessSLO: "PT1H" + + - id: hourly_orders + exposeId: hourly_orders + version: "1.0.0" + description: "Hourly order stream" + binding: + platform: kafka + location: + host: kafka-prod.acme:9092 + contract: + schema: + - name: order_id + type: string + required: true + tags: [primary-key] + - name: customer_id + type: string + required: true + - name: amount + type: decimal + required: true + +expects: + - id: customers_raw + provider: snowflake + location: + account: company-prod + database: RAW_DB + schema: CUSTOMERS diff --git a/tests/fixtures/odcs/contract-full.yaml b/tests/fixtures/odcs/contract-full.yaml new file mode 100644 index 00000000..0d0a8e56 --- /dev/null +++ b/tests/fixtures/odcs/contract-full.yaml @@ -0,0 +1,102 @@ +# Comprehensive ODCS v3.1.0 fixture — populates every section the Phase 1 +# lossless round-trip needs to preserve. Used by tests/test_odcs_roundtrip.py. +version: "1.0.0" +apiVersion: v3.1.0 +kind: DataContract +id: data-product.orders +status: active +name: orders +domain: commerce +tenant: acme +description: + purpose: "Order data for downstream fulfilment" + limitations: "PII columns require approval before consumption" + usage: "Consume via the analytics warehouse only" +tags: + - orders + - production +team: + name: commerce-team + description: "Owners of the commerce domain data" + members: + - username: alice@acme.com + name: Alice + role: data-owner + dateIn: "2024-01-01" + - username: bob@acme.com + name: Bob + role: steward +schema: + - name: orders + logicalType: object + physicalType: table + physicalName: orders_v1 + description: "One row per order" + tags: + - core + properties: + - name: order_id + logicalType: string + physicalType: STRING + required: true + primaryKey: true + description: "Unique order identifier" + tags: + - id + quality: + - type: library + metric: duplicateValues + mustBe: 0 + dimension: uniqueness + - type: library + metric: nullValues + mustBe: 0 + dimension: completeness + - name: customer_id + logicalType: string + physicalType: STRING + required: true + description: "Foreign key to customers" + - name: amount + logicalType: number + physicalType: NUMERIC + required: true + description: "Order amount in dollars" + quality: + - type: library + metric: nullValues + mustBe: 0 + dimension: completeness + relationships: + - type: foreignKey + from: orders.customer_id + to: customers.id + quality: + - type: library + metric: rowCount + mustBeGreaterThan: 0 + dimension: completeness +servers: + - server: bq-prod + type: bigquery + project: acme-prod + dataset: orders +slaProperties: + - property: availability + value: 0.99 + - property: interval + value: "PT5M" + - property: retention + value: "365" + unit: d +support: + - channel: ops-orders + url: https://slack.acme.com/ops + tool: slack +price: + priceAmount: 100.0 + priceCurrency: USD + priceUnit: GB +customProperties: + - property: owner-cost-center + value: CC-42 diff --git a/tests/fixtures/odcs/contract-minimal.yaml b/tests/fixtures/odcs/contract-minimal.yaml new file mode 100644 index 00000000..7742819e --- /dev/null +++ b/tests/fixtures/odcs/contract-minimal.yaml @@ -0,0 +1,19 @@ +# Minimal ODCS contract — only required + name fields populated. +version: "1.0.0" +apiVersion: v3.1.0 +kind: DataContract +id: minimal.contract +status: draft +name: minimal +schema: + - name: minimal_table + logicalType: object + physicalType: table + properties: + - name: id + logicalType: string +servers: + - server: local + type: local + path: /tmp/minimal + format: parquet diff --git a/tests/fixtures/odps/contracts-only/daily_orders.odcs.yaml b/tests/fixtures/odps/contracts-only/daily_orders.odcs.yaml new file mode 100644 index 00000000..532ad04b --- /dev/null +++ b/tests/fixtures/odps/contracts-only/daily_orders.odcs.yaml @@ -0,0 +1,78 @@ +version: 1.0.0 +apiVersion: v3.1.0 +kind: DataContract +id: commerce.orders-product.daily_orders +status: active +name: Orders Product +description: + purpose: Orders data product with daily and hourly outputs +tags: +- orders +- production +domain: commerce +tenant: acme +team: + name: commerce-team + members: + - name: Alice + username: alice@acme.com + role: data-owner +schema: +- name: daily_orders + logicalType: object + physicalType: table + properties: + - name: order_date + logicalType: date + physicalType: DATE + required: true + tags: + - primary-key + quality: + - type: library + metric: nullValues + mustBe: 0 + dimension: completeness + description: Field 'order_date' must not contain null values + - type: library + metric: duplicateValues + mustBe: 0 + dimension: uniqueness + description: Primary key field 'order_date' must contain only unique values + primaryKey: true + - name: order_count + logicalType: integer + physicalType: INT64 + required: true + quality: + - type: library + metric: nullValues + mustBe: 0 + dimension: completeness + description: Field 'order_count' must not contain null values + - name: total_amount + logicalType: number + physicalType: NUMERIC + required: true + quality: + - type: library + metric: nullValues + mustBe: 0 + dimension: completeness + description: Field 'total_amount' must not contain null values + description: Daily order aggregates +servers: +- server: daily_orders + type: bigquery + project: acme-prod + dataset: orders +- server: customers_raw + type: snowflake + account: company-prod + database: RAW_DB + schema: CUSTOMERS +slaProperties: +- property: availability + value: 0.9990000000000001 +- property: interval + value: PT1H diff --git a/tests/fixtures/odps/contracts-only/hourly_orders.odcs.yaml b/tests/fixtures/odps/contracts-only/hourly_orders.odcs.yaml new file mode 100644 index 00000000..9a6fd294 --- /dev/null +++ b/tests/fixtures/odps/contracts-only/hourly_orders.odcs.yaml @@ -0,0 +1,72 @@ +version: 1.0.0 +apiVersion: v3.1.0 +kind: DataContract +id: commerce.orders-product.hourly_orders +status: active +name: Orders Product +description: + purpose: Orders data product with daily and hourly outputs +tags: +- orders +- production +domain: commerce +tenant: acme +team: + name: commerce-team + members: + - name: Alice + username: alice@acme.com + role: data-owner +schema: +- name: hourly_orders + logicalType: object + physicalType: topic + properties: + - name: order_id + logicalType: string + physicalType: string + required: true + tags: + - primary-key + quality: + - type: library + metric: nullValues + mustBe: 0 + dimension: completeness + description: Field 'order_id' must not contain null values + - type: library + metric: duplicateValues + mustBe: 0 + dimension: uniqueness + description: Primary key field 'order_id' must contain only unique values + primaryKey: true + - name: customer_id + logicalType: string + physicalType: string + required: true + quality: + - type: library + metric: nullValues + mustBe: 0 + dimension: completeness + description: Field 'customer_id' must not contain null values + - name: amount + logicalType: number + physicalType: number + required: true + quality: + - type: library + metric: nullValues + mustBe: 0 + dimension: completeness + description: Field 'amount' must not contain null values + description: Hourly order stream +servers: +- server: hourly_orders + type: kafka + host: kafka-prod.acme:9092 +- server: customers_raw + type: snowflake + account: company-prod + database: RAW_DB + schema: CUSTOMERS diff --git a/tests/fixtures/odps/product-bitol-broken/dangling-port.odps.yaml b/tests/fixtures/odps/product-bitol-broken/dangling-port.odps.yaml new file mode 100644 index 00000000..453d5be5 --- /dev/null +++ b/tests/fixtures/odps/product-bitol-broken/dangling-port.odps.yaml @@ -0,0 +1,12 @@ +# Broken: outputPort points at a contractId that has no corresponding ODCS file +# in the same directory. Used to exercise ContractNotFound. +apiVersion: v1.0.0 +kind: DataProduct +id: broken.dangling-port +name: broken-product +version: "1.0.0" +status: draft +outputPorts: + - name: missing_table + version: "1.0.0" + contractId: broken.dangling-port.missing_table diff --git a/tests/fixtures/odps/product-bitol-broken/two-odps-docs/product-a.odps.yaml b/tests/fixtures/odps/product-bitol-broken/two-odps-docs/product-a.odps.yaml new file mode 100644 index 00000000..a7830424 --- /dev/null +++ b/tests/fixtures/odps/product-bitol-broken/two-odps-docs/product-a.odps.yaml @@ -0,0 +1,6 @@ +apiVersion: v1.0.0 +kind: DataProduct +id: broken.product-a +name: product-a +version: "1.0.0" +status: draft diff --git a/tests/fixtures/odps/product-bitol-broken/two-odps-docs/product-b.odps.yaml b/tests/fixtures/odps/product-bitol-broken/two-odps-docs/product-b.odps.yaml new file mode 100644 index 00000000..0225a1b5 --- /dev/null +++ b/tests/fixtures/odps/product-bitol-broken/two-odps-docs/product-b.odps.yaml @@ -0,0 +1,6 @@ +apiVersion: v1.0.0 +kind: DataProduct +id: broken.product-b +name: product-b +version: "1.0.0" +status: draft diff --git a/tests/fixtures/odps/product-bitol/commerce.orders-product.daily_orders.odcs.yaml b/tests/fixtures/odps/product-bitol/commerce.orders-product.daily_orders.odcs.yaml new file mode 100644 index 00000000..532ad04b --- /dev/null +++ b/tests/fixtures/odps/product-bitol/commerce.orders-product.daily_orders.odcs.yaml @@ -0,0 +1,78 @@ +version: 1.0.0 +apiVersion: v3.1.0 +kind: DataContract +id: commerce.orders-product.daily_orders +status: active +name: Orders Product +description: + purpose: Orders data product with daily and hourly outputs +tags: +- orders +- production +domain: commerce +tenant: acme +team: + name: commerce-team + members: + - name: Alice + username: alice@acme.com + role: data-owner +schema: +- name: daily_orders + logicalType: object + physicalType: table + properties: + - name: order_date + logicalType: date + physicalType: DATE + required: true + tags: + - primary-key + quality: + - type: library + metric: nullValues + mustBe: 0 + dimension: completeness + description: Field 'order_date' must not contain null values + - type: library + metric: duplicateValues + mustBe: 0 + dimension: uniqueness + description: Primary key field 'order_date' must contain only unique values + primaryKey: true + - name: order_count + logicalType: integer + physicalType: INT64 + required: true + quality: + - type: library + metric: nullValues + mustBe: 0 + dimension: completeness + description: Field 'order_count' must not contain null values + - name: total_amount + logicalType: number + physicalType: NUMERIC + required: true + quality: + - type: library + metric: nullValues + mustBe: 0 + dimension: completeness + description: Field 'total_amount' must not contain null values + description: Daily order aggregates +servers: +- server: daily_orders + type: bigquery + project: acme-prod + dataset: orders +- server: customers_raw + type: snowflake + account: company-prod + database: RAW_DB + schema: CUSTOMERS +slaProperties: +- property: availability + value: 0.9990000000000001 +- property: interval + value: PT1H diff --git a/tests/fixtures/odps/product-bitol/commerce.orders-product.hourly_orders.odcs.yaml b/tests/fixtures/odps/product-bitol/commerce.orders-product.hourly_orders.odcs.yaml new file mode 100644 index 00000000..9a6fd294 --- /dev/null +++ b/tests/fixtures/odps/product-bitol/commerce.orders-product.hourly_orders.odcs.yaml @@ -0,0 +1,72 @@ +version: 1.0.0 +apiVersion: v3.1.0 +kind: DataContract +id: commerce.orders-product.hourly_orders +status: active +name: Orders Product +description: + purpose: Orders data product with daily and hourly outputs +tags: +- orders +- production +domain: commerce +tenant: acme +team: + name: commerce-team + members: + - name: Alice + username: alice@acme.com + role: data-owner +schema: +- name: hourly_orders + logicalType: object + physicalType: topic + properties: + - name: order_id + logicalType: string + physicalType: string + required: true + tags: + - primary-key + quality: + - type: library + metric: nullValues + mustBe: 0 + dimension: completeness + description: Field 'order_id' must not contain null values + - type: library + metric: duplicateValues + mustBe: 0 + dimension: uniqueness + description: Primary key field 'order_id' must contain only unique values + primaryKey: true + - name: customer_id + logicalType: string + physicalType: string + required: true + quality: + - type: library + metric: nullValues + mustBe: 0 + dimension: completeness + description: Field 'customer_id' must not contain null values + - name: amount + logicalType: number + physicalType: number + required: true + quality: + - type: library + metric: nullValues + mustBe: 0 + dimension: completeness + description: Field 'amount' must not contain null values + description: Hourly order stream +servers: +- server: hourly_orders + type: kafka + host: kafka-prod.acme:9092 +- server: customers_raw + type: snowflake + account: company-prod + database: RAW_DB + schema: CUSTOMERS diff --git a/tests/fixtures/odps/product-bitol/commerce.orders-product.odps.yaml b/tests/fixtures/odps/product-bitol/commerce.orders-product.odps.yaml new file mode 100644 index 00000000..7a1e3040 --- /dev/null +++ b/tests/fixtures/odps/product-bitol/commerce.orders-product.odps.yaml @@ -0,0 +1,34 @@ +apiVersion: v1.0.0 +kind: DataProduct +id: commerce.orders-product +name: Orders Product +version: 1.0.0 +status: active +description: + purpose: Orders data product with daily and hourly outputs +domain: commerce +tenant: acme +tags: +- orders +- production +team: + name: commerce-team + members: + - name: Alice + username: alice@acme.com + role: data-owner +outputPorts: +- name: daily_orders + version: 1.0.0 + contractId: commerce.orders-product.daily_orders + description: Daily order aggregates + type: bigquery +- name: hourly_orders + version: 1.0.0 + contractId: commerce.orders-product.hourly_orders + description: Hourly order stream + type: kafka +inputPorts: +- name: customers_raw + version: '1' + contractId: commerce.orders-product.input.customers_raw diff --git a/tests/fixtures/upstream/README.md b/tests/fixtures/upstream/README.md new file mode 100644 index 00000000..deb420b7 --- /dev/null +++ b/tests/fixtures/upstream/README.md @@ -0,0 +1,20 @@ +# Upstream Bitol example fixtures (gold standard) + +Verbatim copies of the example documents shipped by the Bitol specification +repositories, vendored so our import/export/validate paths are exercised +against real, maintainer-authored documents — not just hand-rolled fixtures. + +| File | Source | Declares | +|------|--------|----------| +| `odcs/full-example.v3.2.0.odcs.yaml` | bitol-io/open-data-contract-standard `docs/examples/all/full-example.odcs.yaml` | `apiVersion: v3.2.0` | +| `odcs/postgresql-adventureworks.v3.0.0.odcs.yaml` | bitol-io/open-data-contract-standard `docs/examples/all/postgresql-adventureworks-contract.odcs.yaml` | `apiVersion: v3.0.0` (older-than-vendored; exercises the superset fallback) | +| `odps/customer-data-product.v1.1.0.odps.yaml` | bitol-io/open-data-product-standard `docs/examples/customer-data-product.odps.yaml` | `v1.1.0` — uses `context`, `synonyms`, top-level `type` | +| `odps/deprecated-data-product.v1.1.0.odps.yaml` | … `docs/examples/deprecated-data-product.odps.yaml` | `v1.1.0` — uses `deprecated` | +| `odps/simple-data-product.v1.1.0.odps.yaml` | … `docs/examples/simple-data-product.odps.yaml` | `v1.1.0` — minimal (relaxed `required`) | +| `odps/variables.v1.1.0.odps.yaml` | … `docs/examples/variables.odps.yaml` | `v1.1.0` — `${VAR}` references (RFC 0050) | +| `odps/vendor-custom-properties.v1.1.0.odps.yaml` | … `docs/examples/vendor-custom-properties.odps.yaml` | `v1.1.0` — `customProperties[].vendor` | + +Both upstream repositories are Apache-2.0 licensed. Files are unmodified; +the `.vX.Y.Z.` infix in the filename is ours, recording the declared +`apiVersion` at vendoring time. Refresh with the fetch snippet in +`tests/test_spec_version_matrix.py`. diff --git a/tests/fixtures/upstream/odcs/full-example.v3.2.0.odcs.yaml b/tests/fixtures/upstream/odcs/full-example.v3.2.0.odcs.yaml new file mode 100644 index 00000000..f1c072bd --- /dev/null +++ b/tests/fixtures/upstream/odcs/full-example.v3.2.0.odcs.yaml @@ -0,0 +1,362 @@ +# Copyright 2026 The Bitol Contributors +# SPDX-License-Identifier: Apache-2.0 + +# What's this data contract about? +domain: seller # Domain +dataProduct: my quantum # Data product name +version: 1.1.0 # Version (follows semantic versioning) +status: active +id: 53581432-6c55-4ba2-a65f-72344a91553a + +authoritativeDefinitions: +- type: canonical + url: https://github.com/bitol-io/open-data-contract-standard/blob/main/docs/examples/all/full-example.odcs.yaml + description: Canonical URL to the latest version of the contract. + +# Lots of information +description: + purpose: Views built on top of the seller tables. + limitations: Data based on seller perspective, no buyer information + usage: Predict sales over time + authoritativeDefinitions: + - type: privacy-statement + url: https://example.com/gdpr.pdf +tenant: ClimateQuantumInc + +kind: DataContract +apiVersion: v3.2.0 # Standard version (follows semantic versioning) + +# Infrastructure & servers +servers: + - server: my-postgres + type: postgres + host: localhost + port: 5432 + database: pypl-edw + schema: pp_access_views + +# Dataset, schema, and quality +schema: + - id: tbl_obj + name: tbl + physicalName: tbl_1 + physicalType: table + businessName: Core Payment Metrics + description: Provides core payment metrics + authoritativeDefinitions: + - url: https://catalog.data.gov/dataset/air-quality + type: businessDefinition + - url: https://youtu.be/jbY1BKFj9ec + type: videoTutorial + tags: [ 'finance', 'payments'] + dataGranularityDescription: Aggregation on columns txn_ref_dt, pmt_txn_id + # Schema-level relationships (for composite keys) + relationships: + - type: foreignKey + from: + - tbl.rcvr_id + - tbl.rcvr_cntry_code + to: + - receivers.id + - receivers.country_code + customProperties: + - property: description + value: "Composite key linking to receivers table" + - property: cardinality + value: "many-to-one" + properties: + - id: txn_ref_dt_prop + name: transaction_reference_date + physicalName: txn_ref_dt + primaryKey: false + primaryKeyPosition: -1 + businessName: transaction reference date + logicalType: date + physicalType: date + required: false + description: Reference date for transaction + partitioned: true + partitionKeyPosition: 1 + criticalDataElement: false + tags: [ ] + classification: public + transformSourceObjects: + - table_name_1 + - table_name_2 + - table_name_3 + transformLogic: sel t1.txn_dt as txn_ref_dt from table_name_1 as t1, table_name_2 as t2, table_name_3 as t3 where t1.txn_dt=date-3 + transformDescription: defines the logic in business terms; logic for dummies + examples: + - "2022-10-03" + - "2020-01-28" + customProperties: + - property: anonymizationStrategy + value: none + - id: rcvr_id_prop + name: rcvr_id + primaryKey: true + primaryKeyPosition: 1 + businessName: receiver id + logicalType: string + physicalType: varchar(18) + required: false + description: A description for column rcvr_id. + partitioned: false + partitionKeyPosition: -1 + criticalDataElement: false + tags: [ 'uid' ] + classification: restricted + # Property-level relationship (from is implicit) + relationships: + - to: receivers.id + type: foreignKey + customProperties: + - property: description + value: "Links to receiver master data" + - id: rcvr_cntry_code_prop + name: rcvr_cntry_code + primaryKey: false + primaryKeyPosition: -1 + businessName: receiver country code + logicalType: string + physicalType: varchar(2) + required: false + description: Country code + partitioned: false + partitionKeyPosition: -1 + criticalDataElement: false + tags: [ ] + classification: public + # Enumeration of allowed values (RFC 0033, ODCS v3.2.0+) + enum: + - value: US + label: United States + description: ISO 3166-1 alpha-2 code for the United States. + - value: CA + label: Canada + - value: MX + label: Mexico + customProperties: + - property: languageEs + value: México + authoritativeDefinitions: + - url: https://collibra.com/asset/742b358f-71a5-4ab1-bda4-dcdba9418c25 + type: businessDefinition + - url: https://github.com/myorg/myrepo + type: transformationImplementation + - url: jdbc:postgresql://localhost:5432/adventureworks/tbl_1/rcvr_cntry_code + type: implementation + encryptedName: rcvr_cntry_code_encrypted + quality: + - metric: nullValues + mustBe: 0 + description: column should not contain null values + dimension: completeness + type: library + severity: error + businessImpact: operational + schedule: 0 20 * * * + scheduler: cron + customProperties: + - property: FIELD_NAME + value: + - property: COMPARE_TO + value: + - property: COMPARISON_TYPE + value: Greater than + quality: + - metric: rowCount + mustBeGreaterThan: 1000000 + type: library + description: Ensure row count is within expected volume range + dimension: completeness + method: reconciliation + severity: error + businessImpact: operational + schedule: 0 20 * * * + scheduler: cron + customProperties: + - property: business-key + value: + - txn_ref_dt + - rcvr_id + + # Receivers master table + - id: receivers_obj + name: receivers + physicalName: receivers_master + physicalType: table + businessName: Receivers Master Data + description: Master data for all receivers + tags: [ 'master-data', 'receivers' ] + properties: + - id: receiver_id_prop + name: id + primaryKey: true + primaryKeyPosition: 1 + businessName: receiver identifier + logicalType: string + physicalType: varchar(18) + required: true + description: Unique identifier for receiver + unique: true + classification: restricted + - id: country_code_prop + name: country_code + primaryKey: true + primaryKeyPosition: 2 + businessName: receiver country + logicalType: string + physicalType: varchar(2) + required: true + description: Country code of the receiver + classification: public + - id: receiver_name_prop + name: receiver_name + businessName: receiver name + logicalType: string + physicalType: varchar(255) + required: true + description: Name of the receiver + classification: restricted + - id: receiver_type_prop + name: receiver_type + businessName: receiver type + logicalType: string + physicalType: varchar(20) + required: false + description: Type of receiver (individual, business, etc.) + classification: public + # Example of nested reference + relationships: + - to: receiver_types.type_code + customProperties: + - property: description + value: "Links to receiver type definitions" + # Map-typed property (RFC 0030, ODCS v3.2.0+) + - id: receiver_attributes_prop + name: receiver_attributes + businessName: receiver attributes + logicalType: map + physicalType: "MAP" + required: false + description: Free-form attributes attached to a receiver, keyed by attribute name. + classification: restricted + map: + key: + logicalType: string + description: Attribute name. + value: + logicalType: string + description: Attribute value. + + +# Pricing +price: + priceAmount: 9.95 + priceCurrency: USD + priceUnit: megabyte + + +# Team +team: + name: my-team + description: The team owning the data contract + members: + - username: ceastwood + role: Data Scientist + dateIn: "2022-08-02" + dateOut: "2022-10-01" + replacedByUsername: mhopper + - username: mhopper + role: Data Scientist + dateIn: "2022-10-01" + - username: daustin + role: Owner + description: Keeper of the grail + dateIn: "2022-10-01" + + +# Roles +roles: + - role: microstrategy_user_opr + access: read + firstLevelApprovers: Reporting Manager + secondLevelApprovers: 'mandolorian' + - role: bq_queryman_user_opr + access: read + firstLevelApprovers: Reporting Manager + secondLevelApprovers: na + - role: risk_data_access_opr + access: read + firstLevelApprovers: Reporting Manager + secondLevelApprovers: 'dathvador' + - role: bq_unica_user_opr + access: write + firstLevelApprovers: Reporting Manager + secondLevelApprovers: 'mickey' + +# SLA +slaProperties: + - property: latency # Property, see list of values in DP QoS + value: 4 + unit: d # d, day, days for days; y, yr, years for years + element: tab1.txn_ref_dt # This would not be needed as it is the same table.column as the default one + - property: generalAvailability + value: "2022-05-12T09:30:10-08:00" + - property: endOfSupport + value: "2032-05-12T09:30:10-08:00" + - property: endOfLife + value: "2042-05-12T09:30:10-08:00" + - property: retention + value: 3 + unit: y + element: tab1.txn_ref_dt + - property: frequency + value: 1 + valueExt: 1 + unit: d + element: tab1.txn_ref_dt + - property: timeOfAvailability + value: 09:00-08:00 + element: tab1.txn_ref_dt + driver: regulatory # Describes the importance of the SLA: [regulatory|analytics|operational|...] + - property: timeOfAvailability + value: 08:00-08:00 + element: tab1.txn_ref_dt + driver: analytics + + +# Support +support: + - channel: '#product-help' # Simple Slack communication channel + tool: slack + - channel: datacontract-ann # Simple distribution list + tool: email + url: mailto:datacontract-ann@bitol.io + - channel: Feedback # Product Feedback + description: General Product Feedback (Public) + url: https://product-feedback.com + - channel: 'product-issues' + tool: teams + scope: issues + customProperties: + - property: servicehours + value: 9-5 CET + + +# Tags +tags: + - transactions + + +# Custom properties +customProperties: + - property: refRulesetName + value: gcsc.ruleset.name + - property: somePropertyName + value: property.value + - property: dataprocClusterName # Used for specific applications like Elevate + value: [ cluster name ] + +contractCreatedTs: "2022-11-15T02:59:43+00:00" diff --git a/tests/fixtures/upstream/odcs/postgresql-adventureworks.v3.0.0.odcs.yaml b/tests/fixtures/upstream/odcs/postgresql-adventureworks.v3.0.0.odcs.yaml new file mode 100644 index 00000000..304b51eb --- /dev/null +++ b/tests/fixtures/upstream/odcs/postgresql-adventureworks.v3.0.0.odcs.yaml @@ -0,0 +1,5329 @@ +# Copyright 2026 The Bitol Contributors +# SPDX-License-Identifier: Apache-2.0 + +version: "1.0.0" +apiVersion: "v3.0.0" +id: "6aeafdc1-ed62-4c8f-bf0a-da1061c98cdb" +status: active +kind: "DataContract" +description: {} +schema: + - name: "department" + physicalName: "department" + physicalType: "table" + description: "Lookup table containing the departments within the Adventure Works\ + \ Cycles company." + properties: + - name: "departmentid" + logicalType: "number" + physicalType: "serial" + description: "Primary key for Department records." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/department/departmentid" + criticalDataElement: false + primaryKey: true + required: false + - name: "name" + logicalType: "object" + physicalType: "Name" + description: "Name of the department." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/department/name" + criticalDataElement: false + primaryKey: false + required: false + - name: "groupname" + logicalType: "object" + physicalType: "Name" + description: "Name of the group to which the department belongs." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/department/groupname" + criticalDataElement: false + primaryKey: false + required: false + - name: "modifieddate" + logicalType: "date" + physicalType: "timestamp" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/department/modifieddate" + criticalDataElement: false + primaryKey: false + required: false + - name: "employee" + physicalName: "employee" + physicalType: "table" + description: "Employee information such as salary, department, and title." + properties: + - name: "businessentityid" + logicalType: "number" + physicalType: "int4" + description: "Primary key for Employee records. Foreign key to BusinessEntity.BusinessEntityID." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/employee/businessentityid" + criticalDataElement: false + primaryKey: true + required: false + - name: "nationalidnumber" + logicalType: "string" + physicalType: "varchar[15]" + description: "Unique national identification number such as a social security\ + \ number." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/employee/nationalidnumber" + criticalDataElement: false + primaryKey: false + required: false + tags: ['sensitive'] + - name: "loginid" + logicalType: "string" + physicalType: "varchar[256]" + description: "Network login." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/employee/loginid" + criticalDataElement: false + primaryKey: false + required: false + - name: "jobtitle" + logicalType: "string" + physicalType: "varchar[50]" + description: "Work title such as Buyer or Sales Representative." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/employee/jobtitle" + criticalDataElement: false + primaryKey: false + required: false + - name: "birthdate" + logicalType: "date" + physicalType: "date" + description: "Date of birth." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/employee/birthdate" + criticalDataElement: false + primaryKey: false + required: false + tags: ['sensitive'] + - name: "maritalstatus" + logicalType: "string" + physicalType: "bpchar" + description: "M = Married, S = Single" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/employee/maritalstatus" + criticalDataElement: false + primaryKey: false + required: false + - name: "gender" + logicalType: "string" + physicalType: "bpchar" + description: "M = Male, F = Female" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/employee/gender" + criticalDataElement: false + primaryKey: false + required: false + - name: "hiredate" + logicalType: "date" + physicalType: "date" + description: "Employee hired on this date." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/employee/hiredate" + criticalDataElement: false + primaryKey: false + required: false + - name: "salariedflag" + logicalType: "object" + physicalType: "Flag" + description: "Job classification. 0 = Hourly, not exempt from collective bargaining.\ + \ 1 = Salaried, exempt from collective bargaining." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/employee/salariedflag" + criticalDataElement: false + primaryKey: false + required: false + - name: "vacationhours" + logicalType: "number" + physicalType: "int2" + description: "Number of available vacation hours." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/employee/vacationhours" + criticalDataElement: false + primaryKey: false + required: false + - name: "sickleavehours" + logicalType: "number" + physicalType: "int2" + description: "Number of available sick leave hours." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/employee/sickleavehours" + criticalDataElement: false + primaryKey: false + required: false + - name: "currentflag" + logicalType: "object" + physicalType: "Flag" + description: "0 = Inactive, 1 = Active" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/employee/currentflag" + criticalDataElement: false + primaryKey: false + required: false + - name: "rowguid" + logicalType: "string" + physicalType: "uuid" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/employee/rowguid" + criticalDataElement: false + primaryKey: false + required: false + - name: "modifieddate" + logicalType: "date" + physicalType: "timestamp" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/employee/modifieddate" + criticalDataElement: false + primaryKey: false + required: false + - name: "organizationnode" + logicalType: "string" + physicalType: "varchar[2147483647]" + description: "Where the employee is located in corporate hierarchy." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/employee/organizationnode" + criticalDataElement: false + primaryKey: false + required: false + - name: "employeedepartmenthistory" + physicalName: "employeedepartmenthistory" + physicalType: "table" + description: "Employee department transfers." + properties: + - name: "businessentityid" + logicalType: "number" + physicalType: "int4" + description: "Employee identification number. Foreign key to Employee.BusinessEntityID." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/employeedepartmenthistory/businessentityid" + criticalDataElement: false + primaryKey: true + required: false + - name: "departmentid" + logicalType: "number" + physicalType: "int2" + description: "Department in which the employee worked including currently. Foreign\ + \ key to Department.DepartmentID." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/employeedepartmenthistory/departmentid" + criticalDataElement: false + primaryKey: true + required: false + - name: "shiftid" + logicalType: "number" + physicalType: "int2" + description: "Identifies which 8-hour shift the employee works. Foreign key\ + \ to Shift.Shift.ID." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/employeedepartmenthistory/shiftid" + criticalDataElement: false + primaryKey: true + required: false + - name: "startdate" + logicalType: "date" + physicalType: "date" + description: "Date the employee started work in the department." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/employeedepartmenthistory/startdate" + criticalDataElement: false + primaryKey: true + required: false + - name: "enddate" + logicalType: "date" + physicalType: "date" + description: "Date the employee left the department. NULL = Current department." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/employeedepartmenthistory/enddate" + criticalDataElement: false + primaryKey: false + required: false + - name: "modifieddate" + logicalType: "date" + physicalType: "timestamp" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/employeedepartmenthistory/modifieddate" + criticalDataElement: false + primaryKey: false + required: false + - name: "employeepayhistory" + physicalName: "employeepayhistory" + physicalType: "table" + description: "Employee pay history." + properties: + - name: "businessentityid" + logicalType: "number" + physicalType: "int4" + description: "Employee identification number. Foreign key to Employee.BusinessEntityID." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/employeepayhistory/businessentityid" + criticalDataElement: false + primaryKey: true + required: false + - name: "ratechangedate" + logicalType: "date" + physicalType: "timestamp" + description: "Date the change in pay is effective" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/employeepayhistory/ratechangedate" + criticalDataElement: false + primaryKey: true + required: false + - name: "rate" + logicalType: "number" + physicalType: "numeric" + description: "Salary hourly rate." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/employeepayhistory/rate" + criticalDataElement: false + primaryKey: false + required: false + - name: "payfrequency" + logicalType: "number" + physicalType: "int2" + description: "1 = Salary received monthly, 2 = Salary received biweekly" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/employeepayhistory/payfrequency" + criticalDataElement: false + primaryKey: false + required: false + - name: "modifieddate" + logicalType: "date" + physicalType: "timestamp" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/employeepayhistory/modifieddate" + criticalDataElement: false + primaryKey: false + required: false + - name: "jobcandidate" + physicalName: "jobcandidate" + physicalType: "table" + description: "Résumés submitted to Human Resources by job applicants." + properties: + - name: "jobcandidateid" + logicalType: "number" + physicalType: "serial" + description: "Primary key for JobCandidate records." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/jobcandidate/jobcandidateid" + criticalDataElement: false + primaryKey: true + required: false + - name: "businessentityid" + logicalType: "number" + physicalType: "int4" + description: "Employee identification number if applicant was hired. Foreign\ + \ key to Employee.BusinessEntityID." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/jobcandidate/businessentityid" + criticalDataElement: false + primaryKey: false + required: false + - name: "resume" + logicalType: "string" + physicalType: "xml" + description: "Résumé in XML format." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/jobcandidate/resume" + criticalDataElement: false + primaryKey: false + required: false + - name: "modifieddate" + logicalType: "date" + physicalType: "timestamp" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/jobcandidate/modifieddate" + criticalDataElement: false + primaryKey: false + required: false + - name: "shift" + physicalName: "shift" + physicalType: "table" + description: "Work shift lookup table." + properties: + - name: "shiftid" + logicalType: "number" + physicalType: "serial" + description: "Primary key for Shift records." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/shift/shiftid" + criticalDataElement: false + primaryKey: true + required: false + - name: "name" + logicalType: "object" + physicalType: "Name" + description: "Shift description." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/shift/name" + criticalDataElement: false + primaryKey: false + required: false + - name: "starttime" + logicalType: "date" + physicalType: "time" + description: "Shift start time." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/shift/starttime" + criticalDataElement: false + primaryKey: false + required: false + - name: "endtime" + logicalType: "date" + physicalType: "time" + description: "Shift end time." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/shift/endtime" + criticalDataElement: false + primaryKey: false + required: false + - name: "modifieddate" + logicalType: "date" + physicalType: "timestamp" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/shift/modifieddate" + criticalDataElement: false + primaryKey: false + required: false + - name: "address" + physicalName: "address" + physicalType: "table" + description: "Street address information for customers, employees, and vendors." + properties: + - name: "addressid" + logicalType: "number" + physicalType: "serial" + description: "Primary key for Address records." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/address/addressid" + criticalDataElement: false + primaryKey: true + required: false + - name: "addressline1" + logicalType: "string" + physicalType: "varchar[60]" + description: "First street address line." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/address/addressline1" + criticalDataElement: false + primaryKey: false + required: false + - name: "addressline2" + logicalType: "string" + physicalType: "varchar[60]" + description: "Second street address line." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/address/addressline2" + criticalDataElement: false + primaryKey: false + required: false + - name: "city" + logicalType: "string" + physicalType: "varchar[30]" + description: "Name of the city." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/address/city" + criticalDataElement: false + primaryKey: false + required: false + - name: "stateprovinceid" + logicalType: "number" + physicalType: "int4" + description: "Unique identification number for the state or province. Foreign\ + \ key to StateProvince table." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/address/stateprovinceid" + criticalDataElement: false + primaryKey: false + required: false + - name: "postalcode" + logicalType: "string" + physicalType: "varchar[15]" + description: "Postal code for the street address." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/address/postalcode" + criticalDataElement: false + primaryKey: false + required: false + - name: "spatiallocation" + logicalType: "string" + physicalType: "bytea" + description: "Latitude and longitude of this address." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/address/spatiallocation" + criticalDataElement: false + primaryKey: false + required: false + - name: "rowguid" + logicalType: "string" + physicalType: "uuid" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/address/rowguid" + criticalDataElement: false + primaryKey: false + required: false + - name: "modifieddate" + logicalType: "date" + physicalType: "timestamp" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/address/modifieddate" + criticalDataElement: false + primaryKey: false + required: false + - name: "addresstype" + physicalName: "addresstype" + physicalType: "table" + description: "Types of addresses stored in the Address table." + properties: + - name: "addresstypeid" + logicalType: "number" + physicalType: "serial" + description: "Primary key for AddressType records." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/addresstype/addresstypeid" + criticalDataElement: false + primaryKey: true + required: false + - name: "name" + logicalType: "object" + physicalType: "Name" + description: "Address type description. For example, Billing, Home, or Shipping." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/addresstype/name" + criticalDataElement: false + primaryKey: false + required: false + - name: "rowguid" + logicalType: "string" + physicalType: "uuid" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/addresstype/rowguid" + criticalDataElement: false + primaryKey: false + required: false + - name: "modifieddate" + logicalType: "date" + physicalType: "timestamp" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/addresstype/modifieddate" + criticalDataElement: false + primaryKey: false + required: false + - name: "businessentity" + physicalName: "businessentity" + physicalType: "table" + description: "Source of the ID that connects vendors, customers, and employees\ + \ with address and contact information." + properties: + - name: "businessentityid" + logicalType: "number" + physicalType: "serial" + description: "Primary key for all customers, vendors, and employees." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/businessentity/businessentityid" + criticalDataElement: false + primaryKey: true + required: false + - name: "rowguid" + logicalType: "string" + physicalType: "uuid" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/businessentity/rowguid" + criticalDataElement: false + primaryKey: false + required: false + - name: "modifieddate" + logicalType: "date" + physicalType: "timestamp" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/businessentity/modifieddate" + criticalDataElement: false + primaryKey: false + required: false + - name: "businessentityaddress" + physicalName: "businessentityaddress" + physicalType: "table" + description: "Cross-reference table mapping customers, vendors, and employees\ + \ to their addresses." + properties: + - name: "businessentityid" + logicalType: "number" + physicalType: "int4" + description: "Primary key. Foreign key to BusinessEntity.BusinessEntityID." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/businessentityaddress/businessentityid" + criticalDataElement: false + primaryKey: true + required: false + - name: "addressid" + logicalType: "number" + physicalType: "int4" + description: "Primary key. Foreign key to Address.AddressID." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/businessentityaddress/addressid" + criticalDataElement: false + primaryKey: true + required: false + - name: "addresstypeid" + logicalType: "number" + physicalType: "int4" + description: "Primary key. Foreign key to AddressType.AddressTypeID." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/businessentityaddress/addresstypeid" + criticalDataElement: false + primaryKey: true + required: false + - name: "rowguid" + logicalType: "string" + physicalType: "uuid" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/businessentityaddress/rowguid" + criticalDataElement: false + primaryKey: false + required: false + - name: "modifieddate" + logicalType: "date" + physicalType: "timestamp" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/businessentityaddress/modifieddate" + criticalDataElement: false + primaryKey: false + required: false + - name: "businessentitycontact" + physicalName: "businessentitycontact" + physicalType: "table" + description: "Cross-reference table mapping stores, vendors, and employees to\ + \ people" + properties: + - name: "businessentityid" + logicalType: "number" + physicalType: "int4" + description: "Primary key. Foreign key to BusinessEntity.BusinessEntityID." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/businessentitycontact/businessentityid" + criticalDataElement: false + primaryKey: true + required: false + - name: "personid" + logicalType: "number" + physicalType: "int4" + description: "Primary key. Foreign key to Person.BusinessEntityID." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/businessentitycontact/personid" + criticalDataElement: false + primaryKey: true + required: false + - name: "contacttypeid" + logicalType: "number" + physicalType: "int4" + description: "Primary key. Foreign key to ContactType.ContactTypeID." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/businessentitycontact/contacttypeid" + criticalDataElement: false + primaryKey: true + required: false + - name: "rowguid" + logicalType: "string" + physicalType: "uuid" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/businessentitycontact/rowguid" + criticalDataElement: false + primaryKey: false + required: false + - name: "modifieddate" + logicalType: "date" + physicalType: "timestamp" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/businessentitycontact/modifieddate" + criticalDataElement: false + primaryKey: false + required: false + - name: "contacttype" + physicalName: "contacttype" + physicalType: "table" + description: "Lookup table containing the types of business entity contacts." + properties: + - name: "contacttypeid" + logicalType: "number" + physicalType: "serial" + description: "Primary key for ContactType records." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/contacttype/contacttypeid" + criticalDataElement: false + primaryKey: true + required: false + - name: "name" + logicalType: "object" + physicalType: "Name" + description: "Contact type description." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/contacttype/name" + criticalDataElement: false + primaryKey: false + required: false + - name: "modifieddate" + logicalType: "date" + physicalType: "timestamp" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/contacttype/modifieddate" + criticalDataElement: false + primaryKey: false + required: false + - name: "countryregion" + physicalName: "countryregion" + physicalType: "table" + description: "Lookup table containing the ISO standard codes for countries and\ + \ regions." + properties: + - name: "countryregioncode" + logicalType: "string" + physicalType: "varchar[3]" + description: "ISO standard code for countries and regions." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/countryregion/countryregioncode" + criticalDataElement: false + primaryKey: true + required: false + - name: "name" + logicalType: "object" + physicalType: "Name" + description: "Country or region name." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/countryregion/name" + criticalDataElement: false + primaryKey: false + required: false + - name: "modifieddate" + logicalType: "date" + physicalType: "timestamp" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/countryregion/modifieddate" + criticalDataElement: false + primaryKey: false + required: false + - name: "emailaddress" + physicalName: "emailaddress" + physicalType: "table" + description: "Where to send a person email." + properties: + - name: "businessentityid" + logicalType: "number" + physicalType: "int4" + description: "Primary key. Person associated with this email address. Foreign\ + \ key to Person.BusinessEntityID" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/emailaddress/businessentityid" + criticalDataElement: false + primaryKey: true + required: false + - name: "emailaddressid" + logicalType: "number" + physicalType: "serial" + description: "Primary key. ID of this email address." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/emailaddress/emailaddressid" + criticalDataElement: false + primaryKey: true + required: false + - name: "emailaddress" + logicalType: "string" + physicalType: "varchar[50]" + description: "E-mail address for the person." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/emailaddress/emailaddress" + criticalDataElement: false + primaryKey: false + required: false + - name: "rowguid" + logicalType: "string" + physicalType: "uuid" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/emailaddress/rowguid" + criticalDataElement: false + primaryKey: false + required: false + - name: "modifieddate" + logicalType: "date" + physicalType: "timestamp" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/emailaddress/modifieddate" + criticalDataElement: false + primaryKey: false + required: false + - name: "password" + physicalName: "password" + physicalType: "table" + description: "One way hashed authentication information" + properties: + - name: "businessentityid" + logicalType: "number" + physicalType: "int4" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/password/businessentityid" + criticalDataElement: false + primaryKey: true + required: false + - name: "passwordhash" + logicalType: "string" + physicalType: "varchar[128]" + description: "Password for the e-mail account." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/password/passwordhash" + criticalDataElement: false + primaryKey: false + required: false + - name: "passwordsalt" + logicalType: "string" + physicalType: "varchar[10]" + description: "Random value concatenated with the password string before the\ + \ password is hashed." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/password/passwordsalt" + criticalDataElement: false + primaryKey: false + required: false + - name: "rowguid" + logicalType: "string" + physicalType: "uuid" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/password/rowguid" + criticalDataElement: false + primaryKey: false + required: false + - name: "modifieddate" + logicalType: "date" + physicalType: "timestamp" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/password/modifieddate" + criticalDataElement: false + primaryKey: false + required: false + - name: "person" + physicalName: "person" + physicalType: "table" + description: "Human beings involved with AdventureWorks: employees, customer contacts,\ + \ and vendor contacts." + properties: + - name: "businessentityid" + logicalType: "number" + physicalType: "int4" + description: "Primary key for Person records." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/person/businessentityid" + criticalDataElement: false + primaryKey: true + required: false + - name: "persontype" + logicalType: "string" + physicalType: "bpchar" + description: "Primary type of person: SC = Store Contact, IN = Individual (retail)\ + \ customer, SP = Sales person, EM = Employee (non-sales), VC = Vendor contact,\ + \ GC = General contact" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/person/persontype" + criticalDataElement: false + primaryKey: false + required: false + - name: "namestyle" + logicalType: "object" + physicalType: "NameStyle" + description: "0 = The data in FirstName and LastName are stored in western style\ + \ (first name, last name) order. 1 = Eastern style (last name, first name)\ + \ order." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/person/namestyle" + criticalDataElement: false + primaryKey: false + required: false + - name: "title" + logicalType: "string" + physicalType: "varchar[8]" + description: "A courtesy title. For example, Mr. or Ms." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/person/title" + criticalDataElement: false + primaryKey: false + required: false + - name: "firstname" + logicalType: "object" + physicalType: "Name" + description: "First name of the person." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/person/firstname" + criticalDataElement: false + primaryKey: false + required: false + - name: "middlename" + logicalType: "object" + physicalType: "Name" + description: "Middle name or middle initial of the person." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/person/middlename" + criticalDataElement: false + primaryKey: false + required: false + - name: "lastname" + logicalType: "object" + physicalType: "Name" + description: "Last name of the person." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/person/lastname" + criticalDataElement: false + primaryKey: false + required: false + - name: "suffix" + logicalType: "string" + physicalType: "varchar[10]" + description: "Surname suffix. For example, Sr. or Jr." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/person/suffix" + criticalDataElement: false + primaryKey: false + required: false + - name: "emailpromotion" + logicalType: "number" + physicalType: "int4" + description: "0 = Contact does not wish to receive e-mail promotions, 1 = Contact\ + \ does wish to receive e-mail promotions from AdventureWorks, 2 = Contact\ + \ does wish to receive e-mail promotions from AdventureWorks and selected\ + \ partners." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/person/emailpromotion" + criticalDataElement: false + primaryKey: false + required: false + - name: "additionalcontactinfo" + logicalType: "string" + physicalType: "xml" + description: "Additional contact information about the person stored in xml\ + \ format." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/person/additionalcontactinfo" + criticalDataElement: false + primaryKey: false + required: false + - name: "demographics" + logicalType: "string" + physicalType: "xml" + description: "Personal information such as hobbies, and income collected from\ + \ online shoppers. Used for sales analysis." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/person/demographics" + criticalDataElement: false + primaryKey: false + required: false + - name: "rowguid" + logicalType: "string" + physicalType: "uuid" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/person/rowguid" + criticalDataElement: false + primaryKey: false + required: false + - name: "modifieddate" + logicalType: "date" + physicalType: "timestamp" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/person/modifieddate" + criticalDataElement: false + primaryKey: false + required: false + - name: "personphone" + physicalName: "personphone" + physicalType: "table" + description: "Telephone number and type of a person." + properties: + - name: "businessentityid" + logicalType: "number" + physicalType: "int4" + description: "Business entity identification number. Foreign key to Person.BusinessEntityID." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/personphone/businessentityid" + criticalDataElement: false + primaryKey: true + required: false + - name: "phonenumber" + logicalType: "object" + physicalType: "Phone" + description: "Telephone number identification number." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/personphone/phonenumber" + criticalDataElement: false + primaryKey: true + required: false + - name: "phonenumbertypeid" + logicalType: "number" + physicalType: "int4" + description: "Kind of phone number. Foreign key to PhoneNumberType.PhoneNumberTypeID." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/personphone/phonenumbertypeid" + criticalDataElement: false + primaryKey: true + required: false + - name: "modifieddate" + logicalType: "date" + physicalType: "timestamp" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/personphone/modifieddate" + criticalDataElement: false + primaryKey: false + required: false + - name: "phonenumbertype" + physicalName: "phonenumbertype" + physicalType: "table" + description: "Type of phone number of a person." + properties: + - name: "phonenumbertypeid" + logicalType: "number" + physicalType: "serial" + description: "Primary key for telephone number type records." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/phonenumbertype/phonenumbertypeid" + criticalDataElement: false + primaryKey: true + required: false + - name: "name" + logicalType: "object" + physicalType: "Name" + description: "Name of the telephone number type" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/phonenumbertype/name" + criticalDataElement: false + primaryKey: false + required: false + - name: "modifieddate" + logicalType: "date" + physicalType: "timestamp" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/phonenumbertype/modifieddate" + criticalDataElement: false + primaryKey: false + required: false + - name: "stateprovince" + physicalName: "stateprovince" + physicalType: "table" + description: "State and province lookup table." + properties: + - name: "stateprovinceid" + logicalType: "number" + physicalType: "serial" + description: "Primary key for StateProvince records." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/stateprovince/stateprovinceid" + criticalDataElement: false + primaryKey: true + required: false + - name: "stateprovincecode" + logicalType: "string" + physicalType: "bpchar" + description: "ISO standard state or province code." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/stateprovince/stateprovincecode" + criticalDataElement: false + primaryKey: false + required: false + - name: "countryregioncode" + logicalType: "string" + physicalType: "varchar[3]" + description: "ISO standard country or region code. Foreign key to CountryRegion.CountryRegionCode." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/stateprovince/countryregioncode" + criticalDataElement: false + primaryKey: false + required: false + - name: "isonlystateprovinceflag" + logicalType: "object" + physicalType: "Flag" + description: "0 = StateProvinceCode exists. 1 = StateProvinceCode unavailable,\ + \ using CountryRegionCode." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/stateprovince/isonlystateprovinceflag" + criticalDataElement: false + primaryKey: false + required: false + - name: "name" + logicalType: "object" + physicalType: "Name" + description: "State or province description." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/stateprovince/name" + criticalDataElement: false + primaryKey: false + required: false + - name: "territoryid" + logicalType: "number" + physicalType: "int4" + description: "ID of the territory in which the state or province is located.\ + \ Foreign key to SalesTerritory.SalesTerritoryID." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/stateprovince/territoryid" + criticalDataElement: false + primaryKey: false + required: false + - name: "rowguid" + logicalType: "string" + physicalType: "uuid" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/stateprovince/rowguid" + criticalDataElement: false + primaryKey: false + required: false + - name: "modifieddate" + logicalType: "date" + physicalType: "timestamp" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/stateprovince/modifieddate" + criticalDataElement: false + primaryKey: false + required: false + - name: "billofmaterials" + physicalName: "billofmaterials" + physicalType: "table" + description: "Items required to make bicycles and bicycle subassemblies. It identifies\ + \ the heirarchical relationship between a parent product and its components." + properties: + - name: "billofmaterialsid" + logicalType: "number" + physicalType: "serial" + description: "Primary key for BillOfMaterials records." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/billofmaterials/billofmaterialsid" + criticalDataElement: false + primaryKey: true + required: false + - name: "productassemblyid" + logicalType: "number" + physicalType: "int4" + description: "Parent product identification number. Foreign key to Product.ProductID." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/billofmaterials/productassemblyid" + criticalDataElement: false + primaryKey: false + required: false + - name: "componentid" + logicalType: "number" + physicalType: "int4" + description: "Component identification number. Foreign key to Product.ProductID." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/billofmaterials/componentid" + criticalDataElement: false + primaryKey: false + required: false + - name: "startdate" + logicalType: "date" + physicalType: "timestamp" + description: "Date the component started being used in the assembly item." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/billofmaterials/startdate" + criticalDataElement: false + primaryKey: false + required: false + - name: "enddate" + logicalType: "date" + physicalType: "timestamp" + description: "Date the component stopped being used in the assembly item." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/billofmaterials/enddate" + criticalDataElement: false + primaryKey: false + required: false + - name: "unitmeasurecode" + logicalType: "string" + physicalType: "bpchar" + description: "Standard code identifying the unit of measure for the quantity." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/billofmaterials/unitmeasurecode" + criticalDataElement: false + primaryKey: false + required: false + - name: "bomlevel" + logicalType: "number" + physicalType: "int2" + description: "Indicates the depth the component is from its parent (AssemblyID)." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/billofmaterials/bomlevel" + criticalDataElement: false + primaryKey: false + required: false + - name: "perassemblyqty" + logicalType: "number" + physicalType: "numeric" + description: "Quantity of the component needed to create the assembly." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/billofmaterials/perassemblyqty" + criticalDataElement: false + primaryKey: false + required: false + - name: "modifieddate" + logicalType: "date" + physicalType: "timestamp" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/billofmaterials/modifieddate" + criticalDataElement: false + primaryKey: false + required: false + - name: "culture" + physicalName: "culture" + physicalType: "table" + description: "Lookup table containing the languages in which some AdventureWorks\ + \ data is stored." + properties: + - name: "cultureid" + logicalType: "string" + physicalType: "bpchar" + description: "Primary key for Culture records." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/culture/cultureid" + criticalDataElement: false + primaryKey: true + required: false + - name: "name" + logicalType: "object" + physicalType: "Name" + description: "Culture description." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/culture/name" + criticalDataElement: false + primaryKey: false + required: false + - name: "modifieddate" + logicalType: "date" + physicalType: "timestamp" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/culture/modifieddate" + criticalDataElement: false + primaryKey: false + required: false + - name: "document" + physicalName: "document" + physicalType: "table" + description: "Product maintenance documents." + properties: + - name: "title" + logicalType: "string" + physicalType: "varchar[50]" + description: "Title of the document." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/document/title" + criticalDataElement: false + primaryKey: false + required: false + - name: "owner" + logicalType: "number" + physicalType: "int4" + description: "Employee who controls the document. Foreign key to Employee.BusinessEntityID" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/document/owner" + criticalDataElement: false + primaryKey: false + required: false + - name: "folderflag" + logicalType: "object" + physicalType: "Flag" + description: "0 = This is a folder, 1 = This is a document." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/document/folderflag" + criticalDataElement: false + primaryKey: false + required: false + - name: "filename" + logicalType: "string" + physicalType: "varchar[400]" + description: "File name of the document" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/document/filename" + criticalDataElement: false + primaryKey: false + required: false + - name: "fileextension" + logicalType: "string" + physicalType: "varchar[8]" + description: "File extension indicating the document type. For example, .doc\ + \ or .txt." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/document/fileextension" + criticalDataElement: false + primaryKey: false + required: false + - name: "revision" + logicalType: "string" + physicalType: "bpchar" + description: "Revision number of the document." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/document/revision" + criticalDataElement: false + primaryKey: false + required: false + - name: "changenumber" + logicalType: "number" + physicalType: "int4" + description: "Engineering change approval number." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/document/changenumber" + criticalDataElement: false + primaryKey: false + required: false + - name: "status" + logicalType: "number" + physicalType: "int2" + description: "1 = Pending approval, 2 = Approved, 3 = Obsolete" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/document/status" + criticalDataElement: false + primaryKey: false + required: false + - name: "documentsummary" + logicalType: "string" + physicalType: "text" + description: "Document abstract." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/document/documentsummary" + criticalDataElement: false + primaryKey: false + required: false + - name: "document" + logicalType: "string" + physicalType: "bytea" + description: "Complete document." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/document/document" + criticalDataElement: false + primaryKey: false + required: false + - name: "rowguid" + logicalType: "string" + physicalType: "uuid" + description: "ROWGUIDCOL number uniquely identifying the record. Required for\ + \ FileStream." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/document/rowguid" + criticalDataElement: false + primaryKey: false + required: false + - name: "modifieddate" + logicalType: "date" + physicalType: "timestamp" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/document/modifieddate" + criticalDataElement: false + primaryKey: false + required: false + - name: "documentnode" + logicalType: "string" + physicalType: "varchar[2147483647]" + description: "Primary key for Document records." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/document/documentnode" + criticalDataElement: false + primaryKey: true + required: false + - name: "illustration" + physicalName: "illustration" + physicalType: "table" + description: "Bicycle assembly diagrams." + properties: + - name: "illustrationid" + logicalType: "number" + physicalType: "serial" + description: "Primary key for Illustration records." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/illustration/illustrationid" + criticalDataElement: false + primaryKey: true + required: false + - name: "diagram" + logicalType: "string" + physicalType: "xml" + description: "Illustrations used in manufacturing instructions. Stored as XML." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/illustration/diagram" + criticalDataElement: false + primaryKey: false + required: false + - name: "modifieddate" + logicalType: "date" + physicalType: "timestamp" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/illustration/modifieddate" + criticalDataElement: false + primaryKey: false + required: false + - name: "location" + physicalName: "location" + physicalType: "table" + description: "Product inventory and manufacturing locations." + properties: + - name: "locationid" + logicalType: "number" + physicalType: "serial" + description: "Primary key for Location records." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/location/locationid" + criticalDataElement: false + primaryKey: true + required: false + - name: "name" + logicalType: "object" + physicalType: "Name" + description: "Location description." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/location/name" + criticalDataElement: false + primaryKey: false + required: false + - name: "costrate" + logicalType: "number" + physicalType: "numeric" + description: "Standard hourly cost of the manufacturing location." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/location/costrate" + criticalDataElement: false + primaryKey: false + required: false + - name: "availability" + logicalType: "number" + physicalType: "numeric" + description: "Work capacity (in hours) of the manufacturing location." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/location/availability" + criticalDataElement: false + primaryKey: false + required: false + - name: "modifieddate" + logicalType: "date" + physicalType: "timestamp" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/location/modifieddate" + criticalDataElement: false + primaryKey: false + required: false + - name: "product" + physicalName: "product" + physicalType: "table" + description: "Products sold or used in the manfacturing of sold products." + properties: + - name: "productid" + logicalType: "number" + physicalType: "serial" + description: "Primary key for Product records." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/product/productid" + criticalDataElement: false + primaryKey: true + required: false + - name: "name" + logicalType: "object" + physicalType: "Name" + description: "Name of the product." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/product/name" + criticalDataElement: false + primaryKey: false + required: false + - name: "productnumber" + logicalType: "string" + physicalType: "varchar[25]" + description: "Unique product identification number." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/product/productnumber" + criticalDataElement: false + primaryKey: false + required: false + - name: "makeflag" + logicalType: "object" + physicalType: "Flag" + description: "0 = Product is purchased, 1 = Product is manufactured in-house." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/product/makeflag" + criticalDataElement: false + primaryKey: false + required: false + - name: "finishedgoodsflag" + logicalType: "object" + physicalType: "Flag" + description: "0 = Product is not a salable item. 1 = Product is salable." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/product/finishedgoodsflag" + criticalDataElement: false + primaryKey: false + required: false + - name: "color" + logicalType: "string" + physicalType: "varchar[15]" + description: "Product color." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/product/color" + criticalDataElement: false + primaryKey: false + required: false + - name: "safetystocklevel" + logicalType: "number" + physicalType: "int2" + description: "Minimum inventory quantity." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/product/safetystocklevel" + criticalDataElement: false + primaryKey: false + required: false + - name: "reorderpoint" + logicalType: "number" + physicalType: "int2" + description: "Inventory level that triggers a purchase order or work order." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/product/reorderpoint" + criticalDataElement: false + primaryKey: false + required: false + - name: "standardcost" + logicalType: "number" + physicalType: "numeric" + description: "Standard cost of the product." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/product/standardcost" + criticalDataElement: false + primaryKey: false + required: false + - name: "listprice" + logicalType: "number" + physicalType: "numeric" + description: "Selling price." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/product/listprice" + criticalDataElement: false + primaryKey: false + required: false + - name: "size" + logicalType: "string" + physicalType: "varchar[5]" + description: "Product size." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/product/size" + criticalDataElement: false + primaryKey: false + required: false + - name: "sizeunitmeasurecode" + logicalType: "string" + physicalType: "bpchar" + description: "Unit of measure for Size column." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/product/sizeunitmeasurecode" + criticalDataElement: false + primaryKey: false + required: false + - name: "weightunitmeasurecode" + logicalType: "string" + physicalType: "bpchar" + description: "Unit of measure for Weight column." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/product/weightunitmeasurecode" + criticalDataElement: false + primaryKey: false + required: false + - name: "weight" + logicalType: "number" + physicalType: "numeric" + description: "Product weight." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/product/weight" + criticalDataElement: false + primaryKey: false + required: false + - name: "daystomanufacture" + logicalType: "number" + physicalType: "int4" + description: "Number of days required to manufacture the product." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/product/daystomanufacture" + criticalDataElement: false + primaryKey: false + required: false + - name: "productline" + logicalType: "string" + physicalType: "bpchar" + description: "R = Road, M = Mountain, T = Touring, S = Standard" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/product/productline" + criticalDataElement: false + primaryKey: false + required: false + - name: "class" + logicalType: "string" + physicalType: "bpchar" + description: "H = High, M = Medium, L = Low" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/product/class" + criticalDataElement: false + primaryKey: false + required: false + - name: "style" + logicalType: "string" + physicalType: "bpchar" + description: "W = Womens, M = Mens, U = Universal" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/product/style" + criticalDataElement: false + primaryKey: false + required: false + - name: "productsubcategoryid" + logicalType: "number" + physicalType: "int4" + description: "Product is a member of this product subcategory. Foreign key to\ + \ ProductSubCategory.ProductSubCategoryID." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/product/productsubcategoryid" + criticalDataElement: false + primaryKey: false + required: false + - name: "productmodelid" + logicalType: "number" + physicalType: "int4" + description: "Product is a member of this product model. Foreign key to ProductModel.ProductModelID." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/product/productmodelid" + criticalDataElement: false + primaryKey: false + required: false + - name: "sellstartdate" + logicalType: "date" + physicalType: "timestamp" + description: "Date the product was available for sale." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/product/sellstartdate" + criticalDataElement: false + primaryKey: false + required: false + - name: "sellenddate" + logicalType: "date" + physicalType: "timestamp" + description: "Date the product was no longer available for sale." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/product/sellenddate" + criticalDataElement: false + primaryKey: false + required: false + - name: "discontinueddate" + logicalType: "date" + physicalType: "timestamp" + description: "Date the product was discontinued." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/product/discontinueddate" + criticalDataElement: false + primaryKey: false + required: false + - name: "rowguid" + logicalType: "string" + physicalType: "uuid" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/product/rowguid" + criticalDataElement: false + primaryKey: false + required: false + - name: "modifieddate" + logicalType: "date" + physicalType: "timestamp" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/product/modifieddate" + criticalDataElement: false + primaryKey: false + required: false + - name: "productcategory" + physicalName: "productcategory" + physicalType: "table" + description: "High-level product categorization." + properties: + - name: "productcategoryid" + logicalType: "number" + physicalType: "serial" + description: "Primary key for ProductCategory records." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/productcategory/productcategoryid" + criticalDataElement: false + primaryKey: true + required: false + - name: "name" + logicalType: "object" + physicalType: "Name" + description: "Category description." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/productcategory/name" + criticalDataElement: false + primaryKey: false + required: false + - name: "rowguid" + logicalType: "string" + physicalType: "uuid" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/productcategory/rowguid" + criticalDataElement: false + primaryKey: false + required: false + - name: "modifieddate" + logicalType: "date" + physicalType: "timestamp" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/productcategory/modifieddate" + criticalDataElement: false + primaryKey: false + required: false + - name: "productcosthistory" + physicalName: "productcosthistory" + physicalType: "table" + description: "Changes in the cost of a product over time." + properties: + - name: "productid" + logicalType: "number" + physicalType: "int4" + description: "Product identification number. Foreign key to Product.ProductID" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/productcosthistory/productid" + criticalDataElement: false + primaryKey: true + required: false + - name: "startdate" + logicalType: "date" + physicalType: "timestamp" + description: "Product cost start date." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/productcosthistory/startdate" + criticalDataElement: false + primaryKey: true + required: false + - name: "enddate" + logicalType: "date" + physicalType: "timestamp" + description: "Product cost end date." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/productcosthistory/enddate" + criticalDataElement: false + primaryKey: false + required: false + - name: "standardcost" + logicalType: "number" + physicalType: "numeric" + description: "Standard cost of the product." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/productcosthistory/standardcost" + criticalDataElement: false + primaryKey: false + required: false + - name: "modifieddate" + logicalType: "date" + physicalType: "timestamp" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/productcosthistory/modifieddate" + criticalDataElement: false + primaryKey: false + required: false + - name: "productdescription" + physicalName: "productdescription" + physicalType: "table" + description: "Product descriptions in several languages." + properties: + - name: "productdescriptionid" + logicalType: "number" + physicalType: "serial" + description: "Primary key for ProductDescription records." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/productdescription/productdescriptionid" + criticalDataElement: false + primaryKey: true + required: false + - name: "description" + logicalType: "string" + physicalType: "varchar[400]" + description: "Description of the product." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/productdescription/description" + criticalDataElement: false + primaryKey: false + required: false + - name: "rowguid" + logicalType: "string" + physicalType: "uuid" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/productdescription/rowguid" + criticalDataElement: false + primaryKey: false + required: false + - name: "modifieddate" + logicalType: "date" + physicalType: "timestamp" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/productdescription/modifieddate" + criticalDataElement: false + primaryKey: false + required: false + - name: "productdocument" + physicalName: "productdocument" + physicalType: "table" + description: "Cross-reference table mapping products to related product documents." + properties: + - name: "productid" + logicalType: "number" + physicalType: "int4" + description: "Product identification number. Foreign key to Product.ProductID." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/productdocument/productid" + criticalDataElement: false + primaryKey: true + required: false + - name: "modifieddate" + logicalType: "date" + physicalType: "timestamp" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/productdocument/modifieddate" + criticalDataElement: false + primaryKey: false + required: false + - name: "documentnode" + logicalType: "string" + physicalType: "varchar[2147483647]" + description: "Document identification number. Foreign key to Document.DocumentNode." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/productdocument/documentnode" + criticalDataElement: false + primaryKey: true + required: false + - name: "productinventory" + physicalName: "productinventory" + physicalType: "table" + description: "Product inventory information." + properties: + - name: "productid" + logicalType: "number" + physicalType: "int4" + description: "Product identification number. Foreign key to Product.ProductID." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/productinventory/productid" + criticalDataElement: false + primaryKey: true + required: false + - name: "locationid" + logicalType: "number" + physicalType: "int2" + description: "Inventory location identification number. Foreign key to Location.LocationID." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/productinventory/locationid" + criticalDataElement: false + primaryKey: true + required: false + - name: "shelf" + logicalType: "string" + physicalType: "varchar[10]" + description: "Storage compartment within an inventory location." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/productinventory/shelf" + criticalDataElement: false + primaryKey: false + required: false + - name: "bin" + logicalType: "number" + physicalType: "int2" + description: "Storage container on a shelf in an inventory location." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/productinventory/bin" + criticalDataElement: false + primaryKey: false + required: false + - name: "quantity" + logicalType: "number" + physicalType: "int2" + description: "Quantity of products in the inventory location." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/productinventory/quantity" + criticalDataElement: false + primaryKey: false + required: false + - name: "rowguid" + logicalType: "string" + physicalType: "uuid" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/productinventory/rowguid" + criticalDataElement: false + primaryKey: false + required: false + - name: "modifieddate" + logicalType: "date" + physicalType: "timestamp" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/productinventory/modifieddate" + criticalDataElement: false + primaryKey: false + required: false + - name: "productlistpricehistory" + physicalName: "productlistpricehistory" + physicalType: "table" + description: "Changes in the list price of a product over time." + properties: + - name: "productid" + logicalType: "number" + physicalType: "int4" + description: "Product identification number. Foreign key to Product.ProductID" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/productlistpricehistory/productid" + criticalDataElement: false + primaryKey: true + required: false + - name: "startdate" + logicalType: "date" + physicalType: "timestamp" + description: "List price start date." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/productlistpricehistory/startdate" + criticalDataElement: false + primaryKey: true + required: false + - name: "enddate" + logicalType: "date" + physicalType: "timestamp" + description: "List price end date" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/productlistpricehistory/enddate" + criticalDataElement: false + primaryKey: false + required: false + - name: "listprice" + logicalType: "number" + physicalType: "numeric" + description: "Product list price." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/productlistpricehistory/listprice" + criticalDataElement: false + primaryKey: false + required: false + - name: "modifieddate" + logicalType: "date" + physicalType: "timestamp" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/productlistpricehistory/modifieddate" + criticalDataElement: false + primaryKey: false + required: false + - name: "productmodel" + physicalName: "productmodel" + physicalType: "table" + description: "Product model classification." + properties: + - name: "productmodelid" + logicalType: "number" + physicalType: "serial" + description: "Primary key for ProductModel records." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/productmodel/productmodelid" + criticalDataElement: false + primaryKey: true + required: false + - name: "name" + logicalType: "object" + physicalType: "Name" + description: "Product model description." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/productmodel/name" + criticalDataElement: false + primaryKey: false + required: false + - name: "catalogdescription" + logicalType: "string" + physicalType: "xml" + description: "Detailed product catalog information in xml format." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/productmodel/catalogdescription" + criticalDataElement: false + primaryKey: false + required: false + - name: "instructions" + logicalType: "string" + physicalType: "xml" + description: "Manufacturing instructions in xml format." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/productmodel/instructions" + criticalDataElement: false + primaryKey: false + required: false + - name: "rowguid" + logicalType: "string" + physicalType: "uuid" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/productmodel/rowguid" + criticalDataElement: false + primaryKey: false + required: false + - name: "modifieddate" + logicalType: "date" + physicalType: "timestamp" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/productmodel/modifieddate" + criticalDataElement: false + primaryKey: false + required: false + - name: "productmodelillustration" + physicalName: "productmodelillustration" + physicalType: "table" + description: "Cross-reference table mapping product models and illustrations." + properties: + - name: "productmodelid" + logicalType: "number" + physicalType: "int4" + description: "Primary key. Foreign key to ProductModel.ProductModelID." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/productmodelillustration/productmodelid" + criticalDataElement: false + primaryKey: true + required: false + - name: "illustrationid" + logicalType: "number" + physicalType: "int4" + description: "Primary key. Foreign key to Illustration.IllustrationID." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/productmodelillustration/illustrationid" + criticalDataElement: false + primaryKey: true + required: false + - name: "modifieddate" + logicalType: "date" + physicalType: "timestamp" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/productmodelillustration/modifieddate" + criticalDataElement: false + primaryKey: false + required: false + - name: "productmodelproductdescriptionculture" + physicalName: "productmodelproductdescriptionculture" + physicalType: "table" + description: "Cross-reference table mapping product descriptions and the language\ + \ the description is written in." + properties: + - name: "productmodelid" + logicalType: "number" + physicalType: "int4" + description: "Primary key. Foreign key to ProductModel.ProductModelID." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/productmodelproductdescriptionculture/productmodelid" + criticalDataElement: false + primaryKey: true + required: false + - name: "productdescriptionid" + logicalType: "number" + physicalType: "int4" + description: "Primary key. Foreign key to ProductDescription.ProductDescriptionID." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/productmodelproductdescriptionculture/productdescriptionid" + criticalDataElement: false + primaryKey: true + required: false + - name: "cultureid" + logicalType: "string" + physicalType: "bpchar" + description: "Culture identification number. Foreign key to Culture.CultureID." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/productmodelproductdescriptionculture/cultureid" + criticalDataElement: false + primaryKey: true + required: false + - name: "modifieddate" + logicalType: "date" + physicalType: "timestamp" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/productmodelproductdescriptionculture/modifieddate" + criticalDataElement: false + primaryKey: false + required: false + - name: "productphoto" + physicalName: "productphoto" + physicalType: "table" + description: "Product images." + properties: + - name: "productphotoid" + logicalType: "number" + physicalType: "serial" + description: "Primary key for ProductPhoto records." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/productphoto/productphotoid" + criticalDataElement: false + primaryKey: true + required: false + - name: "thumbnailphoto" + logicalType: "string" + physicalType: "bytea" + description: "Small image of the product." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/productphoto/thumbnailphoto" + criticalDataElement: false + primaryKey: false + required: false + - name: "thumbnailphotofilename" + logicalType: "string" + physicalType: "varchar[50]" + description: "Small image file name." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/productphoto/thumbnailphotofilename" + criticalDataElement: false + primaryKey: false + required: false + - name: "largephoto" + logicalType: "string" + physicalType: "bytea" + description: "Large image of the product." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/productphoto/largephoto" + criticalDataElement: false + primaryKey: false + required: false + - name: "largephotofilename" + logicalType: "string" + physicalType: "varchar[50]" + description: "Large image file name." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/productphoto/largephotofilename" + criticalDataElement: false + primaryKey: false + required: false + - name: "modifieddate" + logicalType: "date" + physicalType: "timestamp" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/productphoto/modifieddate" + criticalDataElement: false + primaryKey: false + required: false + - name: "productproductphoto" + physicalName: "productproductphoto" + physicalType: "table" + description: "Cross-reference table mapping products and product photos." + properties: + - name: "productid" + logicalType: "number" + physicalType: "int4" + description: "Product identification number. Foreign key to Product.ProductID." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/productproductphoto/productid" + criticalDataElement: false + primaryKey: true + required: false + - name: "productphotoid" + logicalType: "number" + physicalType: "int4" + description: "Product photo identification number. Foreign key to ProductPhoto.ProductPhotoID." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/productproductphoto/productphotoid" + criticalDataElement: false + primaryKey: true + required: false + - name: "primary" + logicalType: "object" + physicalType: "Flag" + description: "0 = Photo is not the principal image. 1 = Photo is the principal\ + \ image." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/productproductphoto/primary" + criticalDataElement: false + primaryKey: false + required: false + - name: "modifieddate" + logicalType: "date" + physicalType: "timestamp" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/productproductphoto/modifieddate" + criticalDataElement: false + primaryKey: false + required: false + - name: "productreview" + physicalName: "productreview" + physicalType: "table" + description: "Customer reviews of products they have purchased." + properties: + - name: "productreviewid" + logicalType: "number" + physicalType: "serial" + description: "Primary key for ProductReview records." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/productreview/productreviewid" + criticalDataElement: false + primaryKey: true + required: false + - name: "productid" + logicalType: "number" + physicalType: "int4" + description: "Product identification number. Foreign key to Product.ProductID." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/productreview/productid" + criticalDataElement: false + primaryKey: false + required: false + - name: "reviewername" + logicalType: "object" + physicalType: "Name" + description: "Name of the reviewer." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/productreview/reviewername" + criticalDataElement: false + primaryKey: false + required: false + - name: "reviewdate" + logicalType: "date" + physicalType: "timestamp" + description: "Date review was submitted." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/productreview/reviewdate" + criticalDataElement: false + primaryKey: false + required: false + - name: "emailaddress" + logicalType: "string" + physicalType: "varchar[50]" + description: "Reviewer's e-mail address." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/productreview/emailaddress" + criticalDataElement: false + primaryKey: false + required: false + - name: "rating" + logicalType: "number" + physicalType: "int4" + description: "Product rating given by the reviewer. Scale is 1 to 5 with 5 as\ + \ the highest rating." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/productreview/rating" + criticalDataElement: false + primaryKey: false + required: false + - name: "comments" + logicalType: "string" + physicalType: "varchar[3850]" + description: "Reviewer's comments" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/productreview/comments" + criticalDataElement: false + primaryKey: false + required: false + - name: "modifieddate" + logicalType: "date" + physicalType: "timestamp" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/productreview/modifieddate" + criticalDataElement: false + primaryKey: false + required: false + - name: "productsubcategory" + physicalName: "productsubcategory" + physicalType: "table" + description: "Product subcategories. See ProductCategory table." + properties: + - name: "productsubcategoryid" + logicalType: "number" + physicalType: "serial" + description: "Primary key for ProductSubcategory records." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/productsubcategory/productsubcategoryid" + criticalDataElement: false + primaryKey: true + required: false + - name: "productcategoryid" + logicalType: "number" + physicalType: "int4" + description: "Product category identification number. Foreign key to ProductCategory.ProductCategoryID." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/productsubcategory/productcategoryid" + criticalDataElement: false + primaryKey: false + required: false + - name: "name" + logicalType: "object" + physicalType: "Name" + description: "Subcategory description." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/productsubcategory/name" + criticalDataElement: false + primaryKey: false + required: false + - name: "rowguid" + logicalType: "string" + physicalType: "uuid" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/productsubcategory/rowguid" + criticalDataElement: false + primaryKey: false + required: false + - name: "modifieddate" + logicalType: "date" + physicalType: "timestamp" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/productsubcategory/modifieddate" + criticalDataElement: false + primaryKey: false + required: false + - name: "scrapreason" + physicalName: "scrapreason" + physicalType: "table" + description: "Manufacturing failure reasons lookup table." + properties: + - name: "scrapreasonid" + logicalType: "number" + physicalType: "serial" + description: "Primary key for ScrapReason records." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/scrapreason/scrapreasonid" + criticalDataElement: false + primaryKey: true + required: false + - name: "name" + logicalType: "object" + physicalType: "Name" + description: "Failure description." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/scrapreason/name" + criticalDataElement: false + primaryKey: false + required: false + - name: "modifieddate" + logicalType: "date" + physicalType: "timestamp" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/scrapreason/modifieddate" + criticalDataElement: false + primaryKey: false + required: false + - name: "transactionhistory" + physicalName: "transactionhistory" + physicalType: "table" + description: "Record of each purchase order, sales order, or work order transaction\ + \ year to date." + properties: + - name: "transactionid" + logicalType: "number" + physicalType: "serial" + description: "Primary key for TransactionHistory records." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/transactionhistory/transactionid" + criticalDataElement: false + primaryKey: true + required: false + - name: "productid" + logicalType: "number" + physicalType: "int4" + description: "Product identification number. Foreign key to Product.ProductID." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/transactionhistory/productid" + criticalDataElement: false + primaryKey: false + required: false + - name: "referenceorderid" + logicalType: "number" + physicalType: "int4" + description: "Purchase order, sales order, or work order identification number." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/transactionhistory/referenceorderid" + criticalDataElement: false + primaryKey: false + required: false + - name: "referenceorderlineid" + logicalType: "number" + physicalType: "int4" + description: "Line number associated with the purchase order, sales order, or\ + \ work order." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/transactionhistory/referenceorderlineid" + criticalDataElement: false + primaryKey: false + required: false + - name: "transactiondate" + logicalType: "date" + physicalType: "timestamp" + description: "Date and time of the transaction." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/transactionhistory/transactiondate" + criticalDataElement: false + primaryKey: false + required: false + - name: "transactiontype" + logicalType: "string" + physicalType: "bpchar" + description: "W = WorkOrder, S = SalesOrder, P = PurchaseOrder" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/transactionhistory/transactiontype" + criticalDataElement: false + primaryKey: false + required: false + - name: "quantity" + logicalType: "number" + physicalType: "int4" + description: "Product quantity." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/transactionhistory/quantity" + criticalDataElement: false + primaryKey: false + required: false + - name: "actualcost" + logicalType: "number" + physicalType: "numeric" + description: "Product cost." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/transactionhistory/actualcost" + criticalDataElement: false + primaryKey: false + required: false + - name: "modifieddate" + logicalType: "date" + physicalType: "timestamp" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/transactionhistory/modifieddate" + criticalDataElement: false + primaryKey: false + required: false + - name: "transactionhistoryarchive" + physicalName: "transactionhistoryarchive" + physicalType: "table" + description: "Transactions for previous years." + properties: + - name: "transactionid" + logicalType: "number" + physicalType: "int4" + description: "Primary key for TransactionHistoryArchive records." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/transactionhistoryarchive/transactionid" + criticalDataElement: false + primaryKey: true + required: false + - name: "productid" + logicalType: "number" + physicalType: "int4" + description: "Product identification number. Foreign key to Product.ProductID." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/transactionhistoryarchive/productid" + criticalDataElement: false + primaryKey: false + required: false + - name: "referenceorderid" + logicalType: "number" + physicalType: "int4" + description: "Purchase order, sales order, or work order identification number." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/transactionhistoryarchive/referenceorderid" + criticalDataElement: false + primaryKey: false + required: false + - name: "referenceorderlineid" + logicalType: "number" + physicalType: "int4" + description: "Line number associated with the purchase order, sales order, or\ + \ work order." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/transactionhistoryarchive/referenceorderlineid" + criticalDataElement: false + primaryKey: false + required: false + - name: "transactiondate" + logicalType: "date" + physicalType: "timestamp" + description: "Date and time of the transaction." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/transactionhistoryarchive/transactiondate" + criticalDataElement: false + primaryKey: false + required: false + - name: "transactiontype" + logicalType: "string" + physicalType: "bpchar" + description: "W = Work Order, S = Sales Order, P = Purchase Order" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/transactionhistoryarchive/transactiontype" + criticalDataElement: false + primaryKey: false + required: false + - name: "quantity" + logicalType: "number" + physicalType: "int4" + description: "Product quantity." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/transactionhistoryarchive/quantity" + criticalDataElement: false + primaryKey: false + required: false + - name: "actualcost" + logicalType: "number" + physicalType: "numeric" + description: "Product cost." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/transactionhistoryarchive/actualcost" + criticalDataElement: false + primaryKey: false + required: false + - name: "modifieddate" + logicalType: "date" + physicalType: "timestamp" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/transactionhistoryarchive/modifieddate" + criticalDataElement: false + primaryKey: false + required: false + - name: "unitmeasure" + physicalName: "unitmeasure" + physicalType: "table" + description: "Unit of measure lookup table." + properties: + - name: "unitmeasurecode" + logicalType: "string" + physicalType: "bpchar" + description: "Primary key." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/unitmeasure/unitmeasurecode" + criticalDataElement: false + primaryKey: true + required: false + - name: "name" + logicalType: "object" + physicalType: "Name" + description: "Unit of measure description." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/unitmeasure/name" + criticalDataElement: false + primaryKey: false + required: false + - name: "modifieddate" + logicalType: "date" + physicalType: "timestamp" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/unitmeasure/modifieddate" + criticalDataElement: false + primaryKey: false + required: false + - name: "workorder" + physicalName: "workorder" + physicalType: "table" + description: "Manufacturing work orders." + properties: + - name: "workorderid" + logicalType: "number" + physicalType: "serial" + description: "Primary key for WorkOrder records." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/workorder/workorderid" + criticalDataElement: false + primaryKey: true + required: false + - name: "productid" + logicalType: "number" + physicalType: "int4" + description: "Product identification number. Foreign key to Product.ProductID." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/workorder/productid" + criticalDataElement: false + primaryKey: false + required: false + - name: "orderqty" + logicalType: "number" + physicalType: "int4" + description: "Product quantity to build." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/workorder/orderqty" + criticalDataElement: false + primaryKey: false + required: false + - name: "scrappedqty" + logicalType: "number" + physicalType: "int2" + description: "Quantity that failed inspection." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/workorder/scrappedqty" + criticalDataElement: false + primaryKey: false + required: false + - name: "startdate" + logicalType: "date" + physicalType: "timestamp" + description: "Work order start date." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/workorder/startdate" + criticalDataElement: false + primaryKey: false + required: false + - name: "enddate" + logicalType: "date" + physicalType: "timestamp" + description: "Work order end date." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/workorder/enddate" + criticalDataElement: false + primaryKey: false + required: false + - name: "duedate" + logicalType: "date" + physicalType: "timestamp" + description: "Work order due date." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/workorder/duedate" + criticalDataElement: false + primaryKey: false + required: false + - name: "scrapreasonid" + logicalType: "number" + physicalType: "int2" + description: "Reason for inspection failure." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/workorder/scrapreasonid" + criticalDataElement: false + primaryKey: false + required: false + - name: "modifieddate" + logicalType: "date" + physicalType: "timestamp" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/workorder/modifieddate" + criticalDataElement: false + primaryKey: false + required: false + - name: "workorderrouting" + physicalName: "workorderrouting" + physicalType: "table" + description: "Work order details." + properties: + - name: "workorderid" + logicalType: "number" + physicalType: "int4" + description: "Primary key. Foreign key to WorkOrder.WorkOrderID." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/workorderrouting/workorderid" + criticalDataElement: false + primaryKey: true + required: false + - name: "productid" + logicalType: "number" + physicalType: "int4" + description: "Primary key. Foreign key to Product.ProductID." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/workorderrouting/productid" + criticalDataElement: false + primaryKey: true + required: false + - name: "operationsequence" + logicalType: "number" + physicalType: "int2" + description: "Primary key. Indicates the manufacturing process sequence." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/workorderrouting/operationsequence" + criticalDataElement: false + primaryKey: true + required: false + - name: "locationid" + logicalType: "number" + physicalType: "int2" + description: "Manufacturing location where the part is processed. Foreign key\ + \ to Location.LocationID." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/workorderrouting/locationid" + criticalDataElement: false + primaryKey: false + required: false + - name: "scheduledstartdate" + logicalType: "date" + physicalType: "timestamp" + description: "Planned manufacturing start date." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/workorderrouting/scheduledstartdate" + criticalDataElement: false + primaryKey: false + required: false + - name: "scheduledenddate" + logicalType: "date" + physicalType: "timestamp" + description: "Planned manufacturing end date." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/workorderrouting/scheduledenddate" + criticalDataElement: false + primaryKey: false + required: false + - name: "actualstartdate" + logicalType: "date" + physicalType: "timestamp" + description: "Actual start date." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/workorderrouting/actualstartdate" + criticalDataElement: false + primaryKey: false + required: false + - name: "actualenddate" + logicalType: "date" + physicalType: "timestamp" + description: "Actual end date." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/workorderrouting/actualenddate" + criticalDataElement: false + primaryKey: false + required: false + - name: "actualresourcehrs" + logicalType: "number" + physicalType: "numeric" + description: "Number of manufacturing hours used." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/workorderrouting/actualresourcehrs" + criticalDataElement: false + primaryKey: false + required: false + - name: "plannedcost" + logicalType: "number" + physicalType: "numeric" + description: "Estimated manufacturing cost." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/workorderrouting/plannedcost" + criticalDataElement: false + primaryKey: false + required: false + - name: "actualcost" + logicalType: "number" + physicalType: "numeric" + description: "Actual manufacturing cost." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/workorderrouting/actualcost" + criticalDataElement: false + primaryKey: false + required: false + - name: "modifieddate" + logicalType: "date" + physicalType: "timestamp" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/workorderrouting/modifieddate" + criticalDataElement: false + primaryKey: false + required: false + - name: "productvendor" + physicalName: "productvendor" + physicalType: "table" + description: "Cross-reference table mapping vendors with the products they supply." + properties: + - name: "productid" + logicalType: "number" + physicalType: "int4" + description: "Primary key. Foreign key to Product.ProductID." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/productvendor/productid" + criticalDataElement: false + primaryKey: true + required: false + - name: "businessentityid" + logicalType: "number" + physicalType: "int4" + description: "Primary key. Foreign key to Vendor.BusinessEntityID." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/productvendor/businessentityid" + criticalDataElement: false + primaryKey: true + required: false + - name: "averageleadtime" + logicalType: "number" + physicalType: "int4" + description: "The average span of time (in days) between placing an order with\ + \ the vendor and receiving the purchased product." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/productvendor/averageleadtime" + criticalDataElement: false + primaryKey: false + required: false + - name: "standardprice" + logicalType: "number" + physicalType: "numeric" + description: "The vendor's usual selling price." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/productvendor/standardprice" + criticalDataElement: false + primaryKey: false + required: false + - name: "lastreceiptcost" + logicalType: "number" + physicalType: "numeric" + description: "The selling price when last purchased." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/productvendor/lastreceiptcost" + criticalDataElement: false + primaryKey: false + required: false + - name: "lastreceiptdate" + logicalType: "date" + physicalType: "timestamp" + description: "Date the product was last received by the vendor." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/productvendor/lastreceiptdate" + criticalDataElement: false + primaryKey: false + required: false + - name: "minorderqty" + logicalType: "number" + physicalType: "int4" + description: "The maximum quantity that should be ordered." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/productvendor/minorderqty" + criticalDataElement: false + primaryKey: false + required: false + - name: "maxorderqty" + logicalType: "number" + physicalType: "int4" + description: "The minimum quantity that should be ordered." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/productvendor/maxorderqty" + criticalDataElement: false + primaryKey: false + required: false + - name: "onorderqty" + logicalType: "number" + physicalType: "int4" + description: "The quantity currently on order." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/productvendor/onorderqty" + criticalDataElement: false + primaryKey: false + required: false + - name: "unitmeasurecode" + logicalType: "string" + physicalType: "bpchar" + description: "The product's unit of measure." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/productvendor/unitmeasurecode" + criticalDataElement: false + primaryKey: false + required: false + - name: "modifieddate" + logicalType: "date" + physicalType: "timestamp" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/productvendor/modifieddate" + criticalDataElement: false + primaryKey: false + required: false + - name: "purchaseorderdetail" + physicalName: "purchaseorderdetail" + physicalType: "table" + description: "Individual products associated with a specific purchase order. See\ + \ PurchaseOrderHeader." + properties: + - name: "purchaseorderid" + logicalType: "number" + physicalType: "int4" + description: "Primary key. Foreign key to PurchaseOrderHeader.PurchaseOrderID." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/purchaseorderdetail/purchaseorderid" + criticalDataElement: false + primaryKey: true + required: false + - name: "purchaseorderdetailid" + logicalType: "number" + physicalType: "serial" + description: "Primary key. One line number per purchased product." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/purchaseorderdetail/purchaseorderdetailid" + criticalDataElement: false + primaryKey: true + required: false + - name: "duedate" + logicalType: "date" + physicalType: "timestamp" + description: "Date the product is expected to be received." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/purchaseorderdetail/duedate" + criticalDataElement: false + primaryKey: false + required: false + - name: "orderqty" + logicalType: "number" + physicalType: "int2" + description: "Quantity ordered." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/purchaseorderdetail/orderqty" + criticalDataElement: false + primaryKey: false + required: false + - name: "productid" + logicalType: "number" + physicalType: "int4" + description: "Product identification number. Foreign key to Product.ProductID." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/purchaseorderdetail/productid" + criticalDataElement: false + primaryKey: false + required: false + - name: "unitprice" + logicalType: "number" + physicalType: "numeric" + description: "Vendor's selling price of a single product." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/purchaseorderdetail/unitprice" + criticalDataElement: false + primaryKey: false + required: false + - name: "receivedqty" + logicalType: "number" + physicalType: "numeric" + description: "Quantity actually received from the vendor." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/purchaseorderdetail/receivedqty" + criticalDataElement: false + primaryKey: false + required: false + - name: "rejectedqty" + logicalType: "number" + physicalType: "numeric" + description: "Quantity rejected during inspection." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/purchaseorderdetail/rejectedqty" + criticalDataElement: false + primaryKey: false + required: false + - name: "modifieddate" + logicalType: "date" + physicalType: "timestamp" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/purchaseorderdetail/modifieddate" + criticalDataElement: false + primaryKey: false + required: false + - name: "purchaseorderheader" + physicalName: "purchaseorderheader" + physicalType: "table" + description: "General purchase order information. See PurchaseOrderDetail." + properties: + - name: "purchaseorderid" + logicalType: "number" + physicalType: "serial" + description: "Primary key." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/purchaseorderheader/purchaseorderid" + criticalDataElement: false + primaryKey: true + required: false + - name: "revisionnumber" + logicalType: "number" + physicalType: "int2" + description: "Incremental number to track changes to the purchase order over\ + \ time." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/purchaseorderheader/revisionnumber" + criticalDataElement: false + primaryKey: false + required: false + - name: "status" + logicalType: "number" + physicalType: "int2" + description: "Order current status. 1 = Pending; 2 = Approved; 3 = Rejected;\ + \ 4 = Complete" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/purchaseorderheader/status" + criticalDataElement: false + primaryKey: false + required: false + - name: "employeeid" + logicalType: "number" + physicalType: "int4" + description: "Employee who created the purchase order. Foreign key to Employee.BusinessEntityID." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/purchaseorderheader/employeeid" + criticalDataElement: false + primaryKey: false + required: false + - name: "vendorid" + logicalType: "number" + physicalType: "int4" + description: "Vendor with whom the purchase order is placed. Foreign key to\ + \ Vendor.BusinessEntityID." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/purchaseorderheader/vendorid" + criticalDataElement: false + primaryKey: false + required: false + - name: "shipmethodid" + logicalType: "number" + physicalType: "int4" + description: "Shipping method. Foreign key to ShipMethod.ShipMethodID." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/purchaseorderheader/shipmethodid" + criticalDataElement: false + primaryKey: false + required: false + - name: "orderdate" + logicalType: "date" + physicalType: "timestamp" + description: "Purchase order creation date." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/purchaseorderheader/orderdate" + criticalDataElement: false + primaryKey: false + required: false + - name: "shipdate" + logicalType: "date" + physicalType: "timestamp" + description: "Estimated shipment date from the vendor." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/purchaseorderheader/shipdate" + criticalDataElement: false + primaryKey: false + required: false + - name: "subtotal" + logicalType: "number" + physicalType: "numeric" + description: "Purchase order subtotal. Computed as SUM(PurchaseOrderDetail.LineTotal)for\ + \ the appropriate PurchaseOrderID." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/purchaseorderheader/subtotal" + criticalDataElement: false + primaryKey: false + required: false + - name: "taxamt" + logicalType: "number" + physicalType: "numeric" + description: "Tax amount." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/purchaseorderheader/taxamt" + criticalDataElement: false + primaryKey: false + required: false + - name: "freight" + logicalType: "number" + physicalType: "numeric" + description: "Shipping cost." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/purchaseorderheader/freight" + criticalDataElement: false + primaryKey: false + required: false + - name: "modifieddate" + logicalType: "date" + physicalType: "timestamp" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/purchaseorderheader/modifieddate" + criticalDataElement: false + primaryKey: false + required: false + - name: "shipmethod" + physicalName: "shipmethod" + physicalType: "table" + description: "Shipping company lookup table." + properties: + - name: "shipmethodid" + logicalType: "number" + physicalType: "serial" + description: "Primary key for ShipMethod records." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/shipmethod/shipmethodid" + criticalDataElement: false + primaryKey: true + required: false + - name: "name" + logicalType: "object" + physicalType: "Name" + description: "Shipping company name." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/shipmethod/name" + criticalDataElement: false + primaryKey: false + required: false + - name: "shipbase" + logicalType: "number" + physicalType: "numeric" + description: "Minimum shipping charge." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/shipmethod/shipbase" + criticalDataElement: false + primaryKey: false + required: false + - name: "shiprate" + logicalType: "number" + physicalType: "numeric" + description: "Shipping charge per pound." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/shipmethod/shiprate" + criticalDataElement: false + primaryKey: false + required: false + - name: "rowguid" + logicalType: "string" + physicalType: "uuid" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/shipmethod/rowguid" + criticalDataElement: false + primaryKey: false + required: false + - name: "modifieddate" + logicalType: "date" + physicalType: "timestamp" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/shipmethod/modifieddate" + criticalDataElement: false + primaryKey: false + required: false + - name: "vendor" + physicalName: "vendor" + physicalType: "table" + description: "Companies from whom Adventure Works Cycles purchases parts or other\ + \ goods." + properties: + - name: "businessentityid" + logicalType: "number" + physicalType: "int4" + description: "Primary key for Vendor records. Foreign key to BusinessEntity.BusinessEntityID" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/vendor/businessentityid" + criticalDataElement: false + primaryKey: true + required: false + - name: "accountnumber" + logicalType: "object" + physicalType: "AccountNumber" + description: "Vendor account (identification) number." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/vendor/accountnumber" + criticalDataElement: false + primaryKey: false + required: false + - name: "name" + logicalType: "object" + physicalType: "Name" + description: "Company name." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/vendor/name" + criticalDataElement: false + primaryKey: false + required: false + - name: "creditrating" + logicalType: "number" + physicalType: "int2" + description: "1 = Superior, 2 = Excellent, 3 = Above average, 4 = Average, 5\ + \ = Below average" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/vendor/creditrating" + criticalDataElement: false + primaryKey: false + required: false + - name: "preferredvendorstatus" + logicalType: "object" + physicalType: "Flag" + description: "0 = Do not use if another vendor is available. 1 = Preferred over\ + \ other vendors supplying the same product." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/vendor/preferredvendorstatus" + criticalDataElement: false + primaryKey: false + required: false + - name: "activeflag" + logicalType: "object" + physicalType: "Flag" + description: "0 = Vendor no longer used. 1 = Vendor is actively used." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/vendor/activeflag" + criticalDataElement: false + primaryKey: false + required: false + - name: "purchasingwebserviceurl" + logicalType: "string" + physicalType: "varchar[1024]" + description: "Vendor URL." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/vendor/purchasingwebserviceurl" + criticalDataElement: false + primaryKey: false + required: false + - name: "modifieddate" + logicalType: "date" + physicalType: "timestamp" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/vendor/modifieddate" + criticalDataElement: false + primaryKey: false + required: false + - name: "countryregioncurrency" + physicalName: "countryregioncurrency" + physicalType: "table" + description: "Cross-reference table mapping ISO currency codes to a country or\ + \ region." + properties: + - name: "countryregioncode" + logicalType: "string" + physicalType: "varchar[3]" + description: "ISO code for countries and regions. Foreign key to CountryRegion.CountryRegionCode." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/countryregioncurrency/countryregioncode" + criticalDataElement: false + primaryKey: true + required: false + - name: "currencycode" + logicalType: "string" + physicalType: "bpchar" + description: "ISO standard currency code. Foreign key to Currency.CurrencyCode." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/countryregioncurrency/currencycode" + criticalDataElement: false + primaryKey: true + required: false + - name: "modifieddate" + logicalType: "date" + physicalType: "timestamp" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/countryregioncurrency/modifieddate" + criticalDataElement: false + primaryKey: false + required: false + - name: "creditcard" + physicalName: "creditcard" + physicalType: "table" + description: "Customer credit card information." + properties: + - name: "creditcardid" + logicalType: "number" + physicalType: "serial" + description: "Primary key for CreditCard records." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/creditcard/creditcardid" + criticalDataElement: false + primaryKey: true + required: false + - name: "cardtype" + logicalType: "string" + physicalType: "varchar[50]" + description: "Credit card name." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/creditcard/cardtype" + criticalDataElement: false + primaryKey: false + required: false + - name: "cardnumber" + logicalType: "string" + physicalType: "varchar[25]" + description: "Credit card number." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/creditcard/cardnumber" + criticalDataElement: false + primaryKey: false + required: false + - name: "expmonth" + logicalType: "number" + physicalType: "int2" + description: "Credit card expiration month." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/creditcard/expmonth" + criticalDataElement: false + primaryKey: false + required: false + - name: "expyear" + logicalType: "number" + physicalType: "int2" + description: "Credit card expiration year." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/creditcard/expyear" + criticalDataElement: false + primaryKey: false + required: false + - name: "modifieddate" + logicalType: "date" + physicalType: "timestamp" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/creditcard/modifieddate" + criticalDataElement: false + primaryKey: false + required: false + - name: "currency" + physicalName: "currency" + physicalType: "table" + description: "Lookup table containing standard ISO currencies." + properties: + - name: "currencycode" + logicalType: "string" + physicalType: "bpchar" + description: "The ISO code for the Currency." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/currency/currencycode" + criticalDataElement: false + primaryKey: true + required: false + - name: "name" + logicalType: "object" + physicalType: "Name" + description: "Currency name." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/currency/name" + criticalDataElement: false + primaryKey: false + required: false + - name: "modifieddate" + logicalType: "date" + physicalType: "timestamp" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/currency/modifieddate" + criticalDataElement: false + primaryKey: false + required: false + - name: "currencyrate" + physicalName: "currencyrate" + physicalType: "table" + description: "Currency exchange rates." + properties: + - name: "currencyrateid" + logicalType: "number" + physicalType: "serial" + description: "Primary key for CurrencyRate records." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/currencyrate/currencyrateid" + criticalDataElement: false + primaryKey: true + required: false + - name: "currencyratedate" + logicalType: "date" + physicalType: "timestamp" + description: "Date and time the exchange rate was obtained." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/currencyrate/currencyratedate" + criticalDataElement: false + primaryKey: false + required: false + - name: "fromcurrencycode" + logicalType: "string" + physicalType: "bpchar" + description: "Exchange rate was converted from this currency code." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/currencyrate/fromcurrencycode" + criticalDataElement: false + primaryKey: false + required: false + - name: "tocurrencycode" + logicalType: "string" + physicalType: "bpchar" + description: "Exchange rate was converted to this currency code." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/currencyrate/tocurrencycode" + criticalDataElement: false + primaryKey: false + required: false + - name: "averagerate" + logicalType: "number" + physicalType: "numeric" + description: "Average exchange rate for the day." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/currencyrate/averagerate" + criticalDataElement: false + primaryKey: false + required: false + - name: "endofdayrate" + logicalType: "number" + physicalType: "numeric" + description: "Final exchange rate for the day." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/currencyrate/endofdayrate" + criticalDataElement: false + primaryKey: false + required: false + - name: "modifieddate" + logicalType: "date" + physicalType: "timestamp" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/currencyrate/modifieddate" + criticalDataElement: false + primaryKey: false + required: false + - name: "customer" + physicalName: "customer" + physicalType: "table" + description: "Current customer information. Also see the Person and Store tables." + properties: + - name: "customerid" + logicalType: "number" + physicalType: "serial" + description: "Primary key." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/customer/customerid" + criticalDataElement: false + primaryKey: true + required: false + - name: "personid" + logicalType: "number" + physicalType: "int4" + description: "Foreign key to Person.BusinessEntityID" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/customer/personid" + criticalDataElement: false + primaryKey: false + required: false + - name: "storeid" + logicalType: "number" + physicalType: "int4" + description: "Foreign key to Store.BusinessEntityID" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/customer/storeid" + criticalDataElement: false + primaryKey: false + required: false + - name: "territoryid" + logicalType: "number" + physicalType: "int4" + description: "ID of the territory in which the customer is located. Foreign\ + \ key to SalesTerritory.SalesTerritoryID." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/customer/territoryid" + criticalDataElement: false + primaryKey: false + required: false + - name: "rowguid" + logicalType: "string" + physicalType: "uuid" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/customer/rowguid" + criticalDataElement: false + primaryKey: false + required: false + - name: "modifieddate" + logicalType: "date" + physicalType: "timestamp" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/customer/modifieddate" + criticalDataElement: false + primaryKey: false + required: false + - name: "personcreditcard" + physicalName: "personcreditcard" + physicalType: "table" + description: "Cross-reference table mapping people to their credit card information\ + \ in the CreditCard table." + properties: + - name: "businessentityid" + logicalType: "number" + physicalType: "int4" + description: "Business entity identification number. Foreign key to Person.BusinessEntityID." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/personcreditcard/businessentityid" + criticalDataElement: false + primaryKey: true + required: false + - name: "creditcardid" + logicalType: "number" + physicalType: "int4" + description: "Credit card identification number. Foreign key to CreditCard.CreditCardID." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/personcreditcard/creditcardid" + criticalDataElement: false + primaryKey: true + required: false + - name: "modifieddate" + logicalType: "date" + physicalType: "timestamp" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/personcreditcard/modifieddate" + criticalDataElement: false + primaryKey: false + required: false + - name: "salesorderdetail" + physicalName: "salesorderdetail" + physicalType: "table" + description: "Individual products associated with a specific sales order. See\ + \ SalesOrderHeader." + properties: + - name: "salesorderid" + logicalType: "number" + physicalType: "int4" + description: "Primary key. Foreign key to SalesOrderHeader.SalesOrderID." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/salesorderdetail/salesorderid" + criticalDataElement: false + primaryKey: true + required: false + - name: "salesorderdetailid" + logicalType: "number" + physicalType: "serial" + description: "Primary key. One incremental unique number per product sold." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/salesorderdetail/salesorderdetailid" + criticalDataElement: false + primaryKey: true + required: false + - name: "carriertrackingnumber" + logicalType: "string" + physicalType: "varchar[25]" + description: "Shipment tracking number supplied by the shipper." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/salesorderdetail/carriertrackingnumber" + criticalDataElement: false + primaryKey: false + required: false + - name: "orderqty" + logicalType: "number" + physicalType: "int2" + description: "Quantity ordered per product." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/salesorderdetail/orderqty" + criticalDataElement: false + primaryKey: false + required: false + - name: "productid" + logicalType: "number" + physicalType: "int4" + description: "Product sold to customer. Foreign key to Product.ProductID." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/salesorderdetail/productid" + criticalDataElement: false + primaryKey: false + required: false + - name: "specialofferid" + logicalType: "number" + physicalType: "int4" + description: "Promotional code. Foreign key to SpecialOffer.SpecialOfferID." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/salesorderdetail/specialofferid" + criticalDataElement: false + primaryKey: false + required: false + - name: "unitprice" + logicalType: "number" + physicalType: "numeric" + description: "Selling price of a single product." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/salesorderdetail/unitprice" + criticalDataElement: false + primaryKey: false + required: false + - name: "unitpricediscount" + logicalType: "number" + physicalType: "numeric" + description: "Discount amount." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/salesorderdetail/unitpricediscount" + criticalDataElement: false + primaryKey: false + required: false + - name: "rowguid" + logicalType: "string" + physicalType: "uuid" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/salesorderdetail/rowguid" + criticalDataElement: false + primaryKey: false + required: false + - name: "modifieddate" + logicalType: "date" + physicalType: "timestamp" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/salesorderdetail/modifieddate" + criticalDataElement: false + primaryKey: false + required: false + - name: "salesorderheader" + physicalName: "salesorderheader" + physicalType: "table" + description: "General sales order information." + properties: + - name: "salesorderid" + logicalType: "number" + physicalType: "serial" + description: "Primary key." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/salesorderheader/salesorderid" + criticalDataElement: false + primaryKey: true + required: false + - name: "revisionnumber" + logicalType: "number" + physicalType: "int2" + description: "Incremental number to track changes to the sales order over time." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/salesorderheader/revisionnumber" + criticalDataElement: false + primaryKey: false + required: false + - name: "orderdate" + logicalType: "date" + physicalType: "timestamp" + description: "Dates the sales order was created." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/salesorderheader/orderdate" + criticalDataElement: false + primaryKey: false + required: false + - name: "duedate" + logicalType: "date" + physicalType: "timestamp" + description: "Date the order is due to the customer." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/salesorderheader/duedate" + criticalDataElement: false + primaryKey: false + required: false + - name: "shipdate" + logicalType: "date" + physicalType: "timestamp" + description: "Date the order was shipped to the customer." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/salesorderheader/shipdate" + criticalDataElement: false + primaryKey: false + required: false + - name: "status" + logicalType: "number" + physicalType: "int2" + description: "Order current status. 1 = In process; 2 = Approved; 3 = Backordered;\ + \ 4 = Rejected; 5 = Shipped; 6 = Cancelled" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/salesorderheader/status" + criticalDataElement: false + primaryKey: false + required: false + - name: "onlineorderflag" + logicalType: "object" + physicalType: "Flag" + description: "0 = Order placed by sales person. 1 = Order placed online by customer." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/salesorderheader/onlineorderflag" + criticalDataElement: false + primaryKey: false + required: false + - name: "purchaseordernumber" + logicalType: "object" + physicalType: "OrderNumber" + description: "Customer purchase order number reference." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/salesorderheader/purchaseordernumber" + criticalDataElement: false + primaryKey: false + required: false + - name: "accountnumber" + logicalType: "object" + physicalType: "AccountNumber" + description: "Financial accounting number reference." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/salesorderheader/accountnumber" + criticalDataElement: false + primaryKey: false + required: false + - name: "customerid" + logicalType: "number" + physicalType: "int4" + description: "Customer identification number. Foreign key to Customer.BusinessEntityID." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/salesorderheader/customerid" + criticalDataElement: false + primaryKey: false + required: false + - name: "salespersonid" + logicalType: "number" + physicalType: "int4" + description: "Sales person who created the sales order. Foreign key to SalesPerson.BusinessEntityID." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/salesorderheader/salespersonid" + criticalDataElement: false + primaryKey: false + required: false + - name: "territoryid" + logicalType: "number" + physicalType: "int4" + description: "Territory in which the sale was made. Foreign key to SalesTerritory.SalesTerritoryID." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/salesorderheader/territoryid" + criticalDataElement: false + primaryKey: false + required: false + - name: "billtoaddressid" + logicalType: "number" + physicalType: "int4" + description: "Customer billing address. Foreign key to Address.AddressID." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/salesorderheader/billtoaddressid" + criticalDataElement: false + primaryKey: false + required: false + - name: "shiptoaddressid" + logicalType: "number" + physicalType: "int4" + description: "Customer shipping address. Foreign key to Address.AddressID." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/salesorderheader/shiptoaddressid" + criticalDataElement: false + primaryKey: false + required: false + - name: "shipmethodid" + logicalType: "number" + physicalType: "int4" + description: "Shipping method. Foreign key to ShipMethod.ShipMethodID." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/salesorderheader/shipmethodid" + criticalDataElement: false + primaryKey: false + required: false + - name: "creditcardid" + logicalType: "number" + physicalType: "int4" + description: "Credit card identification number. Foreign key to CreditCard.CreditCardID." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/salesorderheader/creditcardid" + criticalDataElement: false + primaryKey: false + required: false + - name: "creditcardapprovalcode" + logicalType: "string" + physicalType: "varchar[15]" + description: "Approval code provided by the credit card company." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/salesorderheader/creditcardapprovalcode" + criticalDataElement: false + primaryKey: false + required: false + - name: "currencyrateid" + logicalType: "number" + physicalType: "int4" + description: "Currency exchange rate used. Foreign key to CurrencyRate.CurrencyRateID." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/salesorderheader/currencyrateid" + criticalDataElement: false + primaryKey: false + required: false + - name: "subtotal" + logicalType: "number" + physicalType: "numeric" + description: "Sales subtotal. Computed as SUM(SalesOrderDetail.LineTotal)for\ + \ the appropriate SalesOrderID." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/salesorderheader/subtotal" + criticalDataElement: false + primaryKey: false + required: false + - name: "taxamt" + logicalType: "number" + physicalType: "numeric" + description: "Tax amount." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/salesorderheader/taxamt" + criticalDataElement: false + primaryKey: false + required: false + - name: "freight" + logicalType: "number" + physicalType: "numeric" + description: "Shipping cost." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/salesorderheader/freight" + criticalDataElement: false + primaryKey: false + required: false + - name: "totaldue" + logicalType: "number" + physicalType: "numeric" + description: "Total due from customer. Computed as Subtotal + TaxAmt + Freight." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/salesorderheader/totaldue" + criticalDataElement: false + primaryKey: false + required: false + - name: "comment" + logicalType: "string" + physicalType: "varchar[128]" + description: "Sales representative comments." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/salesorderheader/comment" + criticalDataElement: false + primaryKey: false + required: false + - name: "rowguid" + logicalType: "string" + physicalType: "uuid" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/salesorderheader/rowguid" + criticalDataElement: false + primaryKey: false + required: false + - name: "modifieddate" + logicalType: "date" + physicalType: "timestamp" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/salesorderheader/modifieddate" + criticalDataElement: false + primaryKey: false + required: false + - name: "salesorderheadersalesreason" + physicalName: "salesorderheadersalesreason" + physicalType: "table" + description: "Cross-reference table mapping sales orders to sales reason codes." + properties: + - name: "salesorderid" + logicalType: "number" + physicalType: "int4" + description: "Primary key. Foreign key to SalesOrderHeader.SalesOrderID." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/salesorderheadersalesreason/salesorderid" + criticalDataElement: false + primaryKey: true + required: false + - name: "salesreasonid" + logicalType: "number" + physicalType: "int4" + description: "Primary key. Foreign key to SalesReason.SalesReasonID." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/salesorderheadersalesreason/salesreasonid" + criticalDataElement: false + primaryKey: true + required: false + - name: "modifieddate" + logicalType: "date" + physicalType: "timestamp" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/salesorderheadersalesreason/modifieddate" + criticalDataElement: false + primaryKey: false + required: false + - name: "salesperson" + physicalName: "salesperson" + physicalType: "table" + description: "Sales representative current information." + properties: + - name: "businessentityid" + logicalType: "number" + physicalType: "int4" + description: "Primary key for SalesPerson records. Foreign key to Employee.BusinessEntityID" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/salesperson/businessentityid" + criticalDataElement: false + primaryKey: true + required: false + - name: "territoryid" + logicalType: "number" + physicalType: "int4" + description: "Territory currently assigned to. Foreign key to SalesTerritory.SalesTerritoryID." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/salesperson/territoryid" + criticalDataElement: false + primaryKey: false + required: false + - name: "salesquota" + logicalType: "number" + physicalType: "numeric" + description: "Projected yearly sales." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/salesperson/salesquota" + criticalDataElement: false + primaryKey: false + required: false + - name: "bonus" + logicalType: "number" + physicalType: "numeric" + description: "Bonus due if quota is met." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/salesperson/bonus" + criticalDataElement: false + primaryKey: false + required: false + - name: "commissionpct" + logicalType: "number" + physicalType: "numeric" + description: "Commision percent received per sale." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/salesperson/commissionpct" + criticalDataElement: false + primaryKey: false + required: false + - name: "salesytd" + logicalType: "number" + physicalType: "numeric" + description: "Sales total year to date." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/salesperson/salesytd" + criticalDataElement: false + primaryKey: false + required: false + - name: "saleslastyear" + logicalType: "number" + physicalType: "numeric" + description: "Sales total of previous year." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/salesperson/saleslastyear" + criticalDataElement: false + primaryKey: false + required: false + - name: "rowguid" + logicalType: "string" + physicalType: "uuid" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/salesperson/rowguid" + criticalDataElement: false + primaryKey: false + required: false + - name: "modifieddate" + logicalType: "date" + physicalType: "timestamp" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/salesperson/modifieddate" + criticalDataElement: false + primaryKey: false + required: false + - name: "salespersonquotahistory" + physicalName: "salespersonquotahistory" + physicalType: "table" + description: "Sales performance tracking." + properties: + - name: "businessentityid" + logicalType: "number" + physicalType: "int4" + description: "Sales person identification number. Foreign key to SalesPerson.BusinessEntityID." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/salespersonquotahistory/businessentityid" + criticalDataElement: false + primaryKey: true + required: false + - name: "quotadate" + logicalType: "date" + physicalType: "timestamp" + description: "Sales quota date." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/salespersonquotahistory/quotadate" + criticalDataElement: false + primaryKey: true + required: false + - name: "salesquota" + logicalType: "number" + physicalType: "numeric" + description: "Sales quota amount." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/salespersonquotahistory/salesquota" + criticalDataElement: false + primaryKey: false + required: false + - name: "rowguid" + logicalType: "string" + physicalType: "uuid" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/salespersonquotahistory/rowguid" + criticalDataElement: false + primaryKey: false + required: false + - name: "modifieddate" + logicalType: "date" + physicalType: "timestamp" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/salespersonquotahistory/modifieddate" + criticalDataElement: false + primaryKey: false + required: false + - name: "salesreason" + physicalName: "salesreason" + physicalType: "table" + description: "Lookup table of customer purchase reasons." + properties: + - name: "salesreasonid" + logicalType: "number" + physicalType: "serial" + description: "Primary key for SalesReason records." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/salesreason/salesreasonid" + criticalDataElement: false + primaryKey: true + required: false + - name: "name" + logicalType: "object" + physicalType: "Name" + description: "Sales reason description." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/salesreason/name" + criticalDataElement: false + primaryKey: false + required: false + - name: "reasontype" + logicalType: "object" + physicalType: "Name" + description: "Category the sales reason belongs to." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/salesreason/reasontype" + criticalDataElement: false + primaryKey: false + required: false + - name: "modifieddate" + logicalType: "date" + physicalType: "timestamp" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/salesreason/modifieddate" + criticalDataElement: false + primaryKey: false + required: false + - name: "salestaxrate" + physicalName: "salestaxrate" + physicalType: "table" + description: "Tax rate lookup table." + properties: + - name: "salestaxrateid" + logicalType: "number" + physicalType: "serial" + description: "Primary key for SalesTaxRate records." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/salestaxrate/salestaxrateid" + criticalDataElement: false + primaryKey: true + required: false + - name: "stateprovinceid" + logicalType: "number" + physicalType: "int4" + description: "State, province, or country/region the sales tax applies to." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/salestaxrate/stateprovinceid" + criticalDataElement: false + primaryKey: false + required: false + - name: "taxtype" + logicalType: "number" + physicalType: "int2" + description: "1 = Tax applied to retail transactions, 2 = Tax applied to wholesale\ + \ transactions, 3 = Tax applied to all sales (retail and wholesale) transactions." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/salestaxrate/taxtype" + criticalDataElement: false + primaryKey: false + required: false + - name: "taxrate" + logicalType: "number" + physicalType: "numeric" + description: "Tax rate amount." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/salestaxrate/taxrate" + criticalDataElement: false + primaryKey: false + required: false + - name: "name" + logicalType: "object" + physicalType: "Name" + description: "Tax rate description." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/salestaxrate/name" + criticalDataElement: false + primaryKey: false + required: false + - name: "rowguid" + logicalType: "string" + physicalType: "uuid" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/salestaxrate/rowguid" + criticalDataElement: false + primaryKey: false + required: false + - name: "modifieddate" + logicalType: "date" + physicalType: "timestamp" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/salestaxrate/modifieddate" + criticalDataElement: false + primaryKey: false + required: false + - name: "salesterritory" + physicalName: "salesterritory" + physicalType: "table" + description: "Sales territory lookup table." + properties: + - name: "territoryid" + logicalType: "number" + physicalType: "serial" + description: "Primary key for SalesTerritory records." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/salesterritory/territoryid" + criticalDataElement: false + primaryKey: true + required: false + - name: "name" + logicalType: "object" + physicalType: "Name" + description: "Sales territory description" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/salesterritory/name" + criticalDataElement: false + primaryKey: false + required: false + - name: "countryregioncode" + logicalType: "string" + physicalType: "varchar[3]" + description: "ISO standard country or region code. Foreign key to CountryRegion.CountryRegionCode." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/salesterritory/countryregioncode" + criticalDataElement: false + primaryKey: false + required: false + - name: "group" + logicalType: "string" + physicalType: "varchar[50]" + description: "Geographic area to which the sales territory belong." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/salesterritory/group" + criticalDataElement: false + primaryKey: false + required: false + - name: "salesytd" + logicalType: "number" + physicalType: "numeric" + description: "Sales in the territory year to date." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/salesterritory/salesytd" + criticalDataElement: false + primaryKey: false + required: false + - name: "saleslastyear" + logicalType: "number" + physicalType: "numeric" + description: "Sales in the territory the previous year." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/salesterritory/saleslastyear" + criticalDataElement: false + primaryKey: false + required: false + - name: "costytd" + logicalType: "number" + physicalType: "numeric" + description: "Business costs in the territory year to date." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/salesterritory/costytd" + criticalDataElement: false + primaryKey: false + required: false + - name: "costlastyear" + logicalType: "number" + physicalType: "numeric" + description: "Business costs in the territory the previous year." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/salesterritory/costlastyear" + criticalDataElement: false + primaryKey: false + required: false + - name: "rowguid" + logicalType: "string" + physicalType: "uuid" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/salesterritory/rowguid" + criticalDataElement: false + primaryKey: false + required: false + - name: "modifieddate" + logicalType: "date" + physicalType: "timestamp" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/salesterritory/modifieddate" + criticalDataElement: false + primaryKey: false + required: false + - name: "salesterritoryhistory" + physicalName: "salesterritoryhistory" + physicalType: "table" + description: "Sales representative transfers to other sales territories." + properties: + - name: "businessentityid" + logicalType: "number" + physicalType: "int4" + description: "Primary key. The sales rep. Foreign key to SalesPerson.BusinessEntityID." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/salesterritoryhistory/businessentityid" + criticalDataElement: false + primaryKey: true + required: false + - name: "territoryid" + logicalType: "number" + physicalType: "int4" + description: "Primary key. Territory identification number. Foreign key to SalesTerritory.SalesTerritoryID." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/salesterritoryhistory/territoryid" + criticalDataElement: false + primaryKey: true + required: false + - name: "startdate" + logicalType: "date" + physicalType: "timestamp" + description: "Primary key. Date the sales representive started work in the territory." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/salesterritoryhistory/startdate" + criticalDataElement: false + primaryKey: true + required: false + - name: "enddate" + logicalType: "date" + physicalType: "timestamp" + description: "Date the sales representative left work in the territory." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/salesterritoryhistory/enddate" + criticalDataElement: false + primaryKey: false + required: false + - name: "rowguid" + logicalType: "string" + physicalType: "uuid" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/salesterritoryhistory/rowguid" + criticalDataElement: false + primaryKey: false + required: false + - name: "modifieddate" + logicalType: "date" + physicalType: "timestamp" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/salesterritoryhistory/modifieddate" + criticalDataElement: false + primaryKey: false + required: false + - name: "shoppingcartitem" + physicalName: "shoppingcartitem" + physicalType: "table" + description: "Contains online customer orders until the order is submitted or\ + \ cancelled." + properties: + - name: "shoppingcartitemid" + logicalType: "number" + physicalType: "serial" + description: "Primary key for ShoppingCartItem records." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/shoppingcartitem/shoppingcartitemid" + criticalDataElement: false + primaryKey: true + required: false + - name: "shoppingcartid" + logicalType: "string" + physicalType: "varchar[50]" + description: "Shopping cart identification number." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/shoppingcartitem/shoppingcartid" + criticalDataElement: false + primaryKey: false + required: false + - name: "quantity" + logicalType: "number" + physicalType: "int4" + description: "Product quantity ordered." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/shoppingcartitem/quantity" + criticalDataElement: false + primaryKey: false + required: false + - name: "productid" + logicalType: "number" + physicalType: "int4" + description: "Product ordered. Foreign key to Product.ProductID." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/shoppingcartitem/productid" + criticalDataElement: false + primaryKey: false + required: false + - name: "datecreated" + logicalType: "date" + physicalType: "timestamp" + description: "Date the time the record was created." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/shoppingcartitem/datecreated" + criticalDataElement: false + primaryKey: false + required: false + - name: "modifieddate" + logicalType: "date" + physicalType: "timestamp" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/shoppingcartitem/modifieddate" + criticalDataElement: false + primaryKey: false + required: false + - name: "specialoffer" + physicalName: "specialoffer" + physicalType: "table" + description: "Sale discounts lookup table." + properties: + - name: "specialofferid" + logicalType: "number" + physicalType: "serial" + description: "Primary key for SpecialOffer records." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/specialoffer/specialofferid" + criticalDataElement: false + primaryKey: true + required: false + - name: "description" + logicalType: "string" + physicalType: "varchar[255]" + description: "Discount description." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/specialoffer/description" + criticalDataElement: false + primaryKey: false + required: false + - name: "discountpct" + logicalType: "number" + physicalType: "numeric" + description: "Discount precentage." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/specialoffer/discountpct" + criticalDataElement: false + primaryKey: false + required: false + - name: "type" + logicalType: "string" + physicalType: "varchar[50]" + description: "Discount type category." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/specialoffer/type" + criticalDataElement: false + primaryKey: false + required: false + - name: "category" + logicalType: "string" + physicalType: "varchar[50]" + description: "Group the discount applies to such as Reseller or Customer." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/specialoffer/category" + criticalDataElement: false + primaryKey: false + required: false + - name: "startdate" + logicalType: "date" + physicalType: "timestamp" + description: "Discount start date." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/specialoffer/startdate" + criticalDataElement: false + primaryKey: false + required: false + - name: "enddate" + logicalType: "date" + physicalType: "timestamp" + description: "Discount end date." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/specialoffer/enddate" + criticalDataElement: false + primaryKey: false + required: false + - name: "minqty" + logicalType: "number" + physicalType: "int4" + description: "Minimum discount percent allowed." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/specialoffer/minqty" + criticalDataElement: false + primaryKey: false + required: false + - name: "maxqty" + logicalType: "number" + physicalType: "int4" + description: "Maximum discount percent allowed." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/specialoffer/maxqty" + criticalDataElement: false + primaryKey: false + required: false + - name: "rowguid" + logicalType: "string" + physicalType: "uuid" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/specialoffer/rowguid" + criticalDataElement: false + primaryKey: false + required: false + - name: "modifieddate" + logicalType: "date" + physicalType: "timestamp" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/specialoffer/modifieddate" + criticalDataElement: false + primaryKey: false + required: false + - name: "specialofferproduct" + physicalName: "specialofferproduct" + physicalType: "table" + description: "Cross-reference table mapping products to special offer discounts." + properties: + - name: "specialofferid" + logicalType: "number" + physicalType: "int4" + description: "Primary key for SpecialOfferProduct records." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/specialofferproduct/specialofferid" + criticalDataElement: false + primaryKey: true + required: false + - name: "productid" + logicalType: "number" + physicalType: "int4" + description: "Product identification number. Foreign key to Product.ProductID." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/specialofferproduct/productid" + criticalDataElement: false + primaryKey: true + required: false + - name: "rowguid" + logicalType: "string" + physicalType: "uuid" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/specialofferproduct/rowguid" + criticalDataElement: false + primaryKey: false + required: false + - name: "modifieddate" + logicalType: "date" + physicalType: "timestamp" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/specialofferproduct/modifieddate" + criticalDataElement: false + primaryKey: false + required: false + - name: "store" + physicalName: "store" + physicalType: "table" + description: "Customers (resellers) of Adventure Works products." + properties: + - name: "businessentityid" + logicalType: "number" + physicalType: "int4" + description: "Primary key. Foreign key to Customer.BusinessEntityID." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/store/businessentityid" + criticalDataElement: false + primaryKey: true + required: false + - name: "name" + logicalType: "object" + physicalType: "Name" + description: "Name of the store." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/store/name" + criticalDataElement: false + primaryKey: false + required: false + - name: "salespersonid" + logicalType: "number" + physicalType: "int4" + description: "ID of the sales person assigned to the customer. Foreign key to\ + \ SalesPerson.BusinessEntityID." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/store/salespersonid" + criticalDataElement: false + primaryKey: false + required: false + - name: "demographics" + logicalType: "string" + physicalType: "xml" + description: "Demographic informationg about the store such as the number of\ + \ employees, annual sales and store type." + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/store/demographics" + criticalDataElement: false + primaryKey: false + required: false + - name: "rowguid" + logicalType: "string" + physicalType: "uuid" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/store/rowguid" + criticalDataElement: false + primaryKey: false + required: false + - name: "modifieddate" + logicalType: "date" + physicalType: "timestamp" + partitioned: false + authoritativeDefinitions: + - type: "implementation" + url: "jdbc:postgresql://localhost:5432/adventureworks/store/modifieddate" + criticalDataElement: false + primaryKey: false + required: false +contractCreatedTs: "2023-09-28T20:24:49.331+00:00" + diff --git a/tests/fixtures/upstream/odps/customer-data-product.v1.1.0.odps.yaml b/tests/fixtures/upstream/odps/customer-data-product.v1.1.0.odps.yaml new file mode 100644 index 00000000..6a1708ae --- /dev/null +++ b/tests/fixtures/upstream/odps/customer-data-product.v1.1.0.odps.yaml @@ -0,0 +1,186 @@ +# Copyright 2026 The Bitol Contributors +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: v1.1.0 +kind: DataProduct + +name: Customer Data Product +version: v1.1.0 +id: fbe8d147-28db-4f1d-bedf-a3fe9f458427 +type: aggregate +domain: seller +status: draft +tenant: RetailCorp + +synonyms: + - synonym: Customer 360 + - synonym: Golden customer record + source: data-governance + - id: client-fr + synonym: Référentiel client + locale: fr-FR + description: French name used by EU subsidiaries. + +description: + purpose: Enterprise view of a customer. + limitations: No known limitations. + usage: Check the various artefacts for their own description. + +tags: ['customer'] + +context: + instructions: > + This data product exposes an enterprise view of a customer for self-service + analytics and AI agents. Use the `consolidatedtransactions` output port for + aggregated, joined transaction history; use `rawtransactions` only when + audit-level granularity is required. Refresh latency is 4 hours. + verifiedStatements: + - question: "What was the total customer spend last quarter?" + - question: "How many returning customers placed an order this month?" + - id: lifetime-value + question: "What is the lifetime value of a customer?" + answer: "Use the `consolidatedtransactions` output port (v2.0.0) and sum `total_amount` grouped by `customer_id`." + constraints: + - id: no-pii-exposure + constraint: "Do not expose individual customer PII; always aggregate to at least country level." + tags: ['gdpr', 'pii'] + - constraint: "Do not use for real-time decisions; data latency is 4 hours." + +inputPorts: +- name: payments + version: 1.0.0 + contractId: dbb7b1eb-7628-436e-8914-2a00638ba6db +- name: payments + version: 2.0.0 + contractId: dbb7b1eb-7628-436e-8914-2a00638ba6da +- name: onlinetransactions + version: 1.0.0 + contractId: ec2a112d-5cfe-49f3-8760-f9cfb4597544 +- name: onlinetransactions + version: 1.1.0 + contractId: ec2a112d-5cfe-49f3-8760-f9cfb4597547 + tags: ["transactions"] + customProperties: + - property: "transactions_version" + value: "1.1.0" + authoritativeDefinitions: + - type: "data_dictionary" + url: "https://mydata/dictionary" + +outputPorts: +- name: rawtransactions + description: "Raw Transactions" + type: tables + version: 1.0.0 + contractId: c2798941-1b7e-4b03-9e0d-955b1a872b32 +- name: rawtransactions + description: "Raw Transactions" + type: tables + version: 2.0.0 + contractId: c2798941-1b7e-4b03-9e0d-955b1a872b33 + tags: ["transactions"] + customProperties: + - property: "transactions_version" + value: "2.0.0" + authoritativeDefinitions: + - type: "data_dictionary" + url: "https://mydata/dictionary" + sbom: + - id: sbom-runtime + type: "external" + url: "https://mysbomserver/transactions/runtime.cdx.json" + tags: ["runtime", "cyclonedx"] + customProperties: + - property: sbomFormat + value: CycloneDX 1.6 + - property: buildId + value: build-8842 + authoritativeDefinitions: + - type: implementation + url: "https://ci.acme.com/builds/8842" + - id: sbom-model + type: "external" + url: "https://mysbomserver/transactions/model.spdx.json" + tags: ["model", "spdx"] + customProperties: + - property: sbomFormat + value: SPDX 2.3 + inputContracts: + - id: dbb7b1eb-7628-436e-8914-2a00638ba6db + version: 2.0.0 + - id: ec2a112d-5cfe-49f3-8760-f9cfb4597544 + version: 1.0.0 + +- name: consolidatedtransactions + description: "Consolidated transactions" + type: tables + version: 1.0.0 + contractId: a44978be-1fe0-4226-b840-1b715bc25c63 + synonyms: + - synonym: Consolidated transactions + - synonym: Joined transactions + locale: en-US + - synonym: Transactions consolidées + locale: fr-FR + context: + instructions: > + Pre-joined and pre-aggregated transactions across the payments and + onlinetransactions sources. Prefer this port over `rawtransactions` + for any reporting or AI-driven analytics. Filter by `transaction_ts` + when querying time ranges. + constraints: + - id: no-writes + constraint: "Do not write INSERT, UPDATE, or DELETE statements against this port." + - constraint: "Do not query more than 90 days of data in a single request without pagination." + +- name: fulltransactionswithreturns + description: "Full transactions with returns" + type: tables + version: 0.3.0 + contractId: ef769969-0cbe-4188-876f-bb00abadaee4 + +managementPorts: +- content: dictionary + type: topic + name: tpc-dict-update + description: Kafka topic for dictionary updates + tags: ["kafka"] + customProperties: + - property: "kafka_topic" + value: "true" + authoritativeDefinitions: + - type: "kafka_topic" + url: "https://mykafka/topic" + +support: +- channel: "Data Team Slack" + url: "https://retailcorp.slack.com/archives/C1234567890" + description: "Primary support channel for data product questions" + tool: "slack" + scope: "interactive" +- channel: "Email Support" + url: "mailto:data-support@retailcorp.com" + description: "Email support for urgent issues" + tool: "email" + scope: "issues" + +team: + name: "Data Team" + description: "The Data Team is responsible for the data product." + tags: ["data", "team"] + customProperties: + - property: "data_team" + value: "true" + members: + - username: "john.doe@retailcorp.com" + name: "John Doe" + description: "Data Product Owner" + role: "owner" + dateIn: "2023-01-15" + - username: "jane.smith@retailcorp.com" + name: "Jane Smith" + description: "Data Steward" + role: "data steward" + dateIn: "2023-02-01" + +productCreatedTs: "2023-01-15T10:30:00Z" diff --git a/tests/fixtures/upstream/odps/deprecated-data-product.v1.1.0.odps.yaml b/tests/fixtures/upstream/odps/deprecated-data-product.v1.1.0.odps.yaml new file mode 100644 index 00000000..c2da3711 --- /dev/null +++ b/tests/fixtures/upstream/odps/deprecated-data-product.v1.1.0.odps.yaml @@ -0,0 +1,42 @@ +# Copyright 2026 The Bitol Contributors +# SPDX-License-Identifier: Apache-2.0 + +# Demonstrates RFC-0051: the optional `deprecated` boolean on the data product +# and on input, output, and management ports. Defaults to false. + +apiVersion: v1.1.0 +kind: DataProduct +id: 064c4630-8aad-4dc0-ba95-0f69940e6b18 +status: active +name: Customer 360 +version: v2.0.0 + +description: + purpose: Unified customer view + usage: Analytics and reporting + +inputPorts: +- name: legacy-source + version: 1.0.0 + contractId: 12345678-1234-1234-1234-123456789abc + deprecated: true + +outputPorts: +- name: legacy-export + description: "DEPRECATED: use the 'tables' output port instead." + type: file + version: 1.0.0 + contractId: 87654321-4321-4321-4321-cba987654321 + deprecated: true +- name: tables + description: "Current tabular output" + type: tables + version: 2.0.0 + contractId: 11112222-3333-4444-5555-666677778888 + +managementPorts: +- name: deprecated-admin-api + content: control + type: rest + deprecated: true + description: "DEPRECATED: replaced by the v2 management API." diff --git a/tests/fixtures/upstream/odps/simple-data-product.v1.1.0.odps.yaml b/tests/fixtures/upstream/odps/simple-data-product.v1.1.0.odps.yaml new file mode 100644 index 00000000..a5e29bbb --- /dev/null +++ b/tests/fixtures/upstream/odps/simple-data-product.v1.1.0.odps.yaml @@ -0,0 +1,28 @@ +# Copyright 2026 The Bitol Contributors +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: v1.1.0 +kind: DataProduct +id: 064c4630-8aad-4dc0-ba95-0f69940e6b18 +status: active +name: Simple Data Product +version: v1.0.0 + +description: + purpose: Simple test data product + limitations: None + usage: Testing purposes only + +tags: ['test', 'simple'] + +inputPorts: +- name: source-data + version: 1.0.0 + contractId: 12345678-1234-1234-1234-123456789abc + +outputPorts: +- name: processed-data + description: "Processed output data" + type: tables + version: 1.0.0 + contractId: 87654321-4321-4321-4321-cba987654321 \ No newline at end of file diff --git a/tests/fixtures/upstream/odps/variables.v1.1.0.odps.yaml b/tests/fixtures/upstream/odps/variables.v1.1.0.odps.yaml new file mode 100644 index 00000000..d74807ba --- /dev/null +++ b/tests/fixtures/upstream/odps/variables.v1.1.0.odps.yaml @@ -0,0 +1,27 @@ +# Copyright 2026 The Bitol Contributors +# SPDX-License-Identifier: Apache-2.0 + +# Demonstrates RFC-0050: any string value MAY contain ${VAR_NAME} references, +# resolved at runtime by tooling. The POSIX ${VAR_NAME:-default} form supplies +# an inline default used when the variable is unset or empty. + +apiVersion: v1.1.0 +kind: DataProduct +id: 2f6b9a84-7c3d-4e5f-9a1b-8c0d2e4f6a90 +status: active +name: Orders Data Product +version: v1.0.0 +tenant: ${TENANT:-RetailCorp} + +description: + purpose: Orders data for ${TARGET_ENVIRONMENT:-prod} consumers. + +customProperties: + - property: dataLakePath + value: s3://${DATA_BUCKET}/orders/ + +outputPorts: +- name: orders + type: tables + version: 1.0.0 + contractId: 87654321-4321-4321-4321-cba987654321 diff --git a/tests/fixtures/upstream/odps/vendor-custom-properties.v1.1.0.odps.yaml b/tests/fixtures/upstream/odps/vendor-custom-properties.v1.1.0.odps.yaml new file mode 100644 index 00000000..87804948 --- /dev/null +++ b/tests/fixtures/upstream/odps/vendor-custom-properties.v1.1.0.odps.yaml @@ -0,0 +1,32 @@ +# Copyright 2026 The Bitol Contributors +# SPDX-License-Identifier: Apache-2.0 + +# Demonstrates RFC-0035: the optional `vendor` field on customProperties, +# attributing a custom property to a specific vendor or external system. + +apiVersion: v1.1.0 +kind: DataProduct +id: 064c4630-8aad-4dc0-ba95-0f69940e6b18 +status: active +name: Customer 360 +version: v1.0.0 + +description: + purpose: Unified customer view + +customProperties: + - property: catalogId + value: cat-12345 + vendor: zeenea + - property: clusterName + value: prod-cluster + description: Cluster name for specific applications + vendor: confluent + - property: genericFlag + value: true + +outputPorts: +- name: tables + type: tables + version: 1.0.0 + contractId: 87654321-4321-4321-4321-cba987654321 diff --git a/tests/test_config_manager.py b/tests/test_config_manager.py index a9d1ea14..52b96dcd 100644 --- a/tests/test_config_manager.py +++ b/tests/test_config_manager.py @@ -515,3 +515,91 @@ def test_reset_clears_singleton(self): reset_config() b = get_config() assert a is not b + + +# ===================================================================== +# USER CONFIG PATH PROBES + ENV VAR EXPANSION +# (regression tests for bugs surfaced by the snowflake-biz-lab integration) +# ===================================================================== + + +class TestUserConfigPathProbes: + """``~/.fluid/config.yaml`` must be on the user-config probe list. + + The publish CLI's ``--help`` text and the snowflake-biz-lab bootstrap + script both write user config here; previously the loader only probed + ``~/.fluidrc.yaml``, ``~/.fluidrc``, and ``~/.config/fluid/config.yaml`` + so the catalog config never loaded. + """ + + def test_loads_from_dot_fluid_directory(self, tmp_path, monkeypatch): + from fluid_build.config_manager import FluidConfig + + fake_home = tmp_path + cfg_dir = fake_home / ".fluid" + cfg_dir.mkdir() + (cfg_dir / "config.yaml").write_text( + "catalogs:\n datamesh-manager:\n endpoint: http://example.test:1234\n" + ) + monkeypatch.setattr("pathlib.Path.home", lambda: fake_home) + + cfg = FluidConfig() + assert ( + cfg.get_catalog_config("datamesh-manager").get("endpoint") == "http://example.test:1234" + ) + + +class TestEnvVarExpansion: + """``${VAR}`` placeholders in YAML config values must expand from os.environ. + + The biz-lab bootstrap writes ``api_key: ${DMM_API_KEY}``; without expansion + the literal placeholder gets passed to the catalog provider and DMM auth + fails silently as "endpoint not accessible". + """ + + def test_string_value_expanded(self, tmp_path, monkeypatch): + from fluid_build.config_manager import FluidConfig + + fake_home = tmp_path + (fake_home / ".fluid").mkdir() + (fake_home / ".fluid" / "config.yaml").write_text( + "catalogs:\n datamesh-manager:\n" + " endpoint: http://localhost:8095\n" + " auth:\n type: api_key\n api_key: ${DMM_API_KEY}\n" + ) + monkeypatch.setattr("pathlib.Path.home", lambda: fake_home) + monkeypatch.setenv("DMM_API_KEY", "ed_live_orga_TESTKEY") + + cfg = FluidConfig() + catalog = cfg.get_catalog_config("datamesh-manager") + assert catalog["auth"]["api_key"] == "ed_live_orga_TESTKEY" + + def test_undefined_var_kept_literal(self, tmp_path, monkeypatch): + """Misconfigurations must be visible — undefined refs stay as ``${VAR}``.""" + from fluid_build.config_manager import FluidConfig + + fake_home = tmp_path + (fake_home / ".fluid").mkdir() + (fake_home / ".fluid" / "config.yaml").write_text( + "catalogs:\n c1:\n secret: ${THIS_IS_NOT_SET}\n" + ) + monkeypatch.setattr("pathlib.Path.home", lambda: fake_home) + monkeypatch.delenv("THIS_IS_NOT_SET", raising=False) + + cfg = FluidConfig() + assert cfg.get_catalog_config("c1")["secret"] == "${THIS_IS_NOT_SET}" + + def test_expansion_recurses_into_lists(self, tmp_path, monkeypatch): + from fluid_build.config_manager import FluidConfig + + fake_home = tmp_path + (fake_home / ".fluid").mkdir() + (fake_home / ".fluid" / "config.yaml").write_text( + "catalogs:\n c1:\n allow_hosts:\n - ${HOST_A}\n - ${HOST_B}\n" + ) + monkeypatch.setattr("pathlib.Path.home", lambda: fake_home) + monkeypatch.setenv("HOST_A", "a.example") + monkeypatch.setenv("HOST_B", "b.example") + + cfg = FluidConfig() + assert cfg.get_catalog_config("c1")["allow_hosts"] == ["a.example", "b.example"] diff --git a/tests/test_datamesh_manager_publish_spec.py b/tests/test_datamesh_manager_publish_spec.py index bedbba1d..87897e93 100644 --- a/tests/test_datamesh_manager_publish_spec.py +++ b/tests/test_datamesh_manager_publish_spec.py @@ -101,12 +101,62 @@ def _sample_odps_binding_platform_contract(): } -def test_apply_dry_run_defaults_to_dps_spec(): +def test_apply_dry_run_defaults_to_odps_spec(monkeypatch): + """ODPS is the default: current Entropy CE organisations reject the + legacy DPS shape with HTTP 400 ("Specification type 'dps' is not + supported ... Supported types: odps"). Verified against a live + Entropy CE 2.1.x via scripts/dmm_live_e2e.sh. + """ + monkeypatch.delenv("DMM_DATA_PRODUCT_SPEC", raising=False) provider = DataMeshManagerProvider(api_key="dummy", api_url="https://api.entropy-data.com") result = provider.apply(_sample_contract(), dry_run=True) + payload = result["payload"] + assert payload["kind"] == "DataProduct" + assert "apiVersion" in payload + assert "dataProductSpecification" not in payload + + +def test_apply_dry_run_explicit_dps_spec_keeps_legacy_shape(): + """``--data-product-spec 0.0.1`` is the opt-out back to the legacy DPS.""" + provider = DataMeshManagerProvider(api_key="dummy", api_url="https://api.entropy-data.com") + + result = provider.apply(_sample_contract(), dry_run=True, data_product_specification="0.0.1") + assert result["payload"]["dataProductSpecification"] == "0.0.1" + assert "info" in result["payload"] + + +def test_apply_dry_run_dps_provider_hint_keeps_legacy_shape(): + """``provider_hint="dps"`` (the catalog adapter's override path) also + selects the legacy DPS shape.""" + provider = DataMeshManagerProvider(api_key="dummy", api_url="https://api.entropy-data.com") + + result = provider.apply(_sample_contract(), dry_run=True, provider_hint="dps") + + assert result["payload"]["dataProductSpecification"] == "0.0.1" + + +def test_apply_dry_run_env_var_sets_default_spec(monkeypatch): + """``DMM_DATA_PRODUCT_SPEC`` changes the default without touching the + contract or the CLI invocation; explicit values still win over it.""" + monkeypatch.setenv("DMM_DATA_PRODUCT_SPEC", "0.0.1") + provider = DataMeshManagerProvider(api_key="dummy", api_url="https://api.entropy-data.com") + # Schema-valid FLUID data products always declare ``kind: DataProduct``; + # the env-var opt-out must still apply to them. + contract = _sample_contract() + contract["kind"] = "DataProduct" + + result = provider.apply(contract, dry_run=True) + assert result["payload"]["dataProductSpecification"] == "0.0.1" + assert provider.plan(contract)[0]["payload"]["dataProductSpecification"] == "0.0.1" + + result = provider.apply(contract, dry_run=True, provider_hint="odps") + assert result["payload"]["kind"] == "DataProduct" + + result = provider.apply(contract, dry_run=True, data_product_specification="4.1.0") + assert result["payload"]["dataProductSpecification"] == "4.1.0" def test_apply_dry_run_uses_odps_spec_when_provider_hint_is_odps(): @@ -120,6 +170,103 @@ def test_apply_dry_run_uses_odps_spec_when_provider_hint_is_odps(): assert "info" not in payload +def test_apply_dry_run_uses_odps_spec_when_contract_kind_is_dataproduct(monkeypatch): + """Bitol-style FLUID contracts declare ``kind: DataProduct``; the publish + payload must therefore default to ODPS so Entropy CE (configured for + ``odps`` only) accepts it. Regression test for the issue surfaced by + running ``publish:pre`` against the snowflake-biz-lab repo. + """ + monkeypatch.delenv("DMM_DATA_PRODUCT_SPEC", raising=False) + provider = DataMeshManagerProvider(api_key="dummy", api_url="https://api.entropy-data.com") + contract = _sample_contract() + contract["kind"] = "DataProduct" + + result = provider.apply(contract, dry_run=True) + + payload = result["payload"] + # ODPS path emits apiVersion + kind, not the DPS dataProductSpecification field + assert payload.get("kind") == "DataProduct" + assert "apiVersion" in payload + + +def test_plan_uses_same_default_spec_as_apply(monkeypatch): + """``plan()`` previews exactly what ``apply()`` would send — ODPS by + default — so ``fluid dmm publish --dry-run`` is a faithful preview.""" + monkeypatch.delenv("DMM_DATA_PRODUCT_SPEC", raising=False) + provider = DataMeshManagerProvider(api_key="dummy", api_url="https://api.entropy-data.com") + + actions = provider.plan(_sample_contract()) + + assert actions[0]["payload"]["kind"] == "DataProduct" + assert "apiVersion" in actions[0]["payload"] + + +def test_ensure_team_payload_includes_type_field(): + """Entropy CE 2.0.x's ``PUT /api/teams/{id}`` requires a non-null ``type`` + field. The fluid CLI's team payload must include it (default ``internal``) + or the publish fails with HTTP 400 "Failed to read request". + """ + provider = DataMeshManagerProvider(api_key="dummy", api_url="https://api.entropy-data.com") + + captured: dict = {} + + def fake_request(method, path, json_body=None, **kw): + captured["method"] = method + captured["path"] = path + captured["json_body"] = json_body + + class _Resp: + status_code = 200 + + return _Resp() + + fake_session = MagicMock() + # Force the GET to return 404 so _ensure_team falls through to PUT + fake_session.get.return_value = SimpleNamespace(status_code=404) + + with ( + patch.object(provider, "_session", return_value=fake_session), + patch.object(provider, "_request", side_effect=fake_request), + ): + provider._ensure_team( + {"owner": {"team": "telco-data-platform", "email": "data-platform@example.com"}}, + "telco-data-platform", + ) + + assert captured["method"] == "PUT" + assert captured["path"] == "/api/teams/telco-data-platform" + assert captured["json_body"]["type"] == "internal" + assert captured["json_body"]["id"] == "telco-data-platform" + + +def test_ensure_team_type_override_via_owner_team_type(): + """``owner.team_type`` overrides the default ``internal``.""" + provider = DataMeshManagerProvider(api_key="dummy", api_url="https://api.entropy-data.com") + captured: dict = {} + + def fake_request(method, path, json_body=None, **kw): + captured["json_body"] = json_body + + class _Resp: + status_code = 200 + + return _Resp() + + fake_session = MagicMock() + fake_session.get.return_value = SimpleNamespace(status_code=404) + + with ( + patch.object(provider, "_session", return_value=fake_session), + patch.object(provider, "_request", side_effect=fake_request), + ): + provider._ensure_team( + {"owner": {"team": "bu-analytics", "team_type": "business-unit"}}, + "bu-analytics", + ) + + assert captured["json_body"]["type"] == "business-unit" + + def test_apply_dry_run_allows_explicit_spec_override(): provider = DataMeshManagerProvider(api_key="dummy", api_url="https://api.entropy-data.com") @@ -136,7 +283,12 @@ def test_apply_dry_run_allows_explicit_spec_override(): def test_apply_dry_run_keeps_per_expose_data_contract_ids_for_dps(): provider = DataMeshManagerProvider(api_key="dummy", api_url="https://api.entropy-data.com") - result = provider.apply(_sample_contract_with_exposes(), dry_run=True, publish_contract=True) + result = provider.apply( + _sample_contract_with_exposes(), + dry_run=True, + publish_contract=True, + data_product_specification="0.0.1", + ) output_ports = result["payload"].get("outputPorts", []) data_contract_ids = [port.get("dataContractId") for port in output_ports] @@ -403,3 +555,156 @@ class _Resp: assert "validation_error" in results[0] assert "schema_objects" in results[0] assert "schema_properties" in results[0] + + +# ===================================================================== +# Catalog adapter: FLUID dict pass-through (bug 6 — post-redesign) +# ===================================================================== + + +class TestCatalogFluidPassthrough: + """The catalog-provider publish path threads the full FLUID dict through + so multi-port contracts emit one ODCS per output port. + + Regression test for the issue surfaced by running ``task publish:pre`` + against the local snowflake-biz-lab DMM stack — the redesign kills the + flat CatalogAsset intermediate (``_asset_to_fluid``) that used to + collapse multi-port contracts into a single wrapper. + """ + + def test_dmm_catalog_provider_forwards_full_fluid(self, monkeypatch): + """The new ``publish(fluid, target)`` signature hands the underlying + DMM provider the raw FLUID dict — every output port is preserved. + """ + import asyncio + + from fluid_build.providers.catalogs import CatalogTarget + from fluid_build.providers.catalogs.datamesh_manager import ( + DataMeshManagerCatalogProvider, + ) + + provider = DataMeshManagerCatalogProvider( + {"endpoint": "http://localhost", "auth": {"api_key": "k"}} + ) + captured: dict = {} + + def fake_apply(fluid, **kwargs): + captured["fluid"] = fluid + captured["kwargs"] = kwargs + return {"id": fluid.get("id"), "url": "http://x"} + + provider._provider.apply = fake_apply # type: ignore[assignment] + + full = { + "id": "bronze.test", + "name": "Test", + "kind": "DataProduct", + "exposes": [ + {"id": "port_a", "binding": {"platform": "snowflake"}}, + {"id": "port_b", "binding": {"platform": "snowflake"}}, + ], + } + target = CatalogTarget.from_fluid(full) + + loop = asyncio.new_event_loop() + try: + result = loop.run_until_complete(provider.publish(full, target)) + finally: + loop.close() + assert result.success + assert result.asset_id == "bronze.test" + # The underlying provider must have seen the full multi-port FLUID. + assert len(captured["fluid"]["exposes"]) == 2 + # No hard-coded hint: the adapter defers to the provider's resolver + # (explicit spec → hint → DMM_DATA_PRODUCT_SPEC → ODPS default) so the + # env-var / catalog-config opt-outs keep working on this path. + assert captured["kwargs"].get("provider_hint") is None + assert captured["kwargs"].get("data_product_specification") is None + monkeypatch.delenv("DMM_DATA_PRODUCT_SPEC", raising=False) + assert ( + provider._provider._resolve_data_product_specification(None) + == provider._provider.DATA_PRODUCT_SPEC_ODPS + ) + assert captured["kwargs"].get("publish_contract") is True + + def test_dmm_catalog_provider_respects_provider_hint_override(self): + """``catalog_hints['provider_hint']`` overrides the odps default.""" + import asyncio + + from fluid_build.providers.catalogs import CatalogTarget + from fluid_build.providers.catalogs.datamesh_manager import ( + DataMeshManagerCatalogProvider, + ) + + provider = DataMeshManagerCatalogProvider( + {"endpoint": "http://localhost", "auth": {"api_key": "k"}} + ) + captured: dict = {} + + def fake_apply(fluid, **kwargs): + captured["kwargs"] = kwargs + return {"id": fluid.get("id")} + + provider._provider.apply = fake_apply # type: ignore[assignment] + + full = {"id": "x", "name": "X", "exposes": []} + target = CatalogTarget.from_fluid(full, catalog_hints={"provider_hint": "dps"}) + + loop = asyncio.new_event_loop() + try: + loop.run_until_complete(provider.publish(full, target)) + finally: + loop.close() + assert captured["kwargs"].get("provider_hint") == "dps" + + def test_deprecated_map_contract_to_asset_still_works(self): + """The CatalogAsset builder stays available for one release with a + DeprecationWarning — external callers that build a CatalogAsset by + hand still round-trip through the ``publish_asset`` shim. + """ + import warnings + + from fluid_build.providers.catalogs import BaseCatalogProvider + + class _Stub(BaseCatalogProvider): + async def publish(self, fluid, target): ... + async def update(self, fluid, target): ... + async def verify(self, contract_id): ... + async def health_check(self): ... + + provider = _Stub({}) + contract = { + "id": "bronze.test", + "name": "Test", + "kind": "DataProduct", + "exposes": [ + {"id": "port_a", "binding": {"platform": "snowflake"}}, + {"id": "port_b", "binding": {"platform": "snowflake"}}, + ], + } + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + asset = provider.map_contract_to_asset(contract) + assert any(issubclass(w.category, DeprecationWarning) for w in caught) + assert asset.raw_contract is not None + assert len(asset.raw_contract["exposes"]) == 2 + + # ...and the publish_asset shim really forwards the full FLUID dict. + import asyncio + + seen: dict = {} + + async def _record(fluid, target=None): + seen["fluid"], seen["target"] = fluid, target + + provider.publish = _record # type: ignore[assignment] + loop = asyncio.new_event_loop() + try: + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + loop.run_until_complete(provider.publish_asset(asset)) + finally: + loop.close() + assert any(issubclass(w.category, DeprecationWarning) for w in caught) + assert len(seen["fluid"]["exposes"]) == 2 + assert seen["target"].contract_id == "bronze.test" diff --git a/tests/test_forge_copilot_seed.py b/tests/test_forge_copilot_seed.py new file mode 100644 index 00000000..ef6d0971 --- /dev/null +++ b/tests/test_forge_copilot_seed.py @@ -0,0 +1,159 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +"""Phase 7 — ``fluid forge --seed-from`` pre-processor tests. + +The seed pre-processor (``fluid_build.cli.forge_copilot_seed``) accepts the +same three input shapes as ``fluid opds import`` — lone ODCS, single ODPS +file, or directory — and produces a structural FLUID skeleton the LLM uses +as ground truth. These tests prove each shape produces an equivalent +skeleton with the expected provenance, and that the ground-truth diff +guard catches schema mutations. +""" + +from __future__ import annotations + +import copy +from pathlib import Path + +import pytest + +from fluid_build.cli.forge_copilot_seed import ( + SHAPE_DIRECTORY, + SHAPE_ODCS_FILE, + SHAPE_ODCS_ONLY_DIRECTORY, + SHAPE_ODPS_FILE, + diff_against_seed, + load_seed, + resolve_at_path, +) +from fluid_build.providers.base import ProviderError + +FIXTURES = Path(__file__).parent / "fixtures" +BUNDLE_DIR = FIXTURES / "odps" / "product-bitol" +CONTRACTS_ONLY = FIXTURES / "odps" / "contracts-only" + + +# --------------------------------------------------------------------------- +# Shape detection + skeleton generation +# --------------------------------------------------------------------------- + + +class TestShapeDetection: + def test_lone_odcs_file(self) -> None: + odcs = next(BUNDLE_DIR.glob("*.odcs.yaml")) + seed = load_seed(odcs) + assert seed.shape == SHAPE_ODCS_FILE + assert len(seed.fluid["exposes"]) == 1 + + def test_single_odps_file(self) -> None: + odps = next(BUNDLE_DIR.glob("*.odps.yaml")) + seed = load_seed(odps) + assert seed.shape == SHAPE_ODPS_FILE + assert len(seed.fluid["exposes"]) == 2 + + def test_directory_bundle(self) -> None: + seed = load_seed(BUNDLE_DIR) + assert seed.shape == SHAPE_DIRECTORY + assert len(seed.fluid["exposes"]) == 2 + + def test_directory_odcs_only(self) -> None: + seed = load_seed(CONTRACTS_ONLY) + assert seed.shape == SHAPE_ODCS_ONLY_DIRECTORY + # Two ODCS files → two exposes, no product wrapper + n_files = len(list(CONTRACTS_ONLY.glob("*.odcs.yaml"))) + assert len(seed.fluid["exposes"]) == n_files + + +class TestEquivalence: + """ODPS-file and directory imports should produce the same skeleton.""" + + def test_file_and_directory_seeds_are_equivalent(self) -> None: + odps = next(BUNDLE_DIR.glob("*.odps.yaml")) + seed_file = load_seed(odps) + seed_dir = load_seed(BUNDLE_DIR) + assert seed_file.fluid == seed_dir.fluid + + +# --------------------------------------------------------------------------- +# Provenance + missing input +# --------------------------------------------------------------------------- + + +class TestProvenance: + def test_lone_odcs_provenance_records_path(self) -> None: + odcs = next(BUNDLE_DIR.glob("*.odcs.yaml")) + seed = load_seed(odcs) + assert seed.provenance[0]["origin"] == str(odcs) + + def test_directory_provenance_enumerates_odcs_files(self) -> None: + seed = load_seed(BUNDLE_DIR) + origins = [entry["origin"] for entry in seed.provenance] + for child in BUNDLE_DIR.glob("*.odcs.yaml"): + assert str(child) in origins + + +class TestMissingPath: + def test_nonexistent_path_raises(self) -> None: + with pytest.raises(ProviderError, match="not found"): + load_seed("/does/not/exist.odps.yaml") + + +# --------------------------------------------------------------------------- +# Ground-truth diff guard +# --------------------------------------------------------------------------- + + +class TestGroundTruthGuard: + def test_no_mutation_returns_empty_diff(self) -> None: + seed = load_seed(BUNDLE_DIR) + # Candidate is the seed itself — must produce zero mismatches + assert diff_against_seed(seed, seed.fluid) == [] + + def test_schema_mutation_is_detected(self) -> None: + seed = load_seed(BUNDLE_DIR) + mutated = copy.deepcopy(seed.fluid) + # Remove one field from the first expose's schema — the guard must catch it + mutated["exposes"][0]["contract"]["schema"].pop() + mismatches = diff_against_seed(seed, mutated) + assert any("exposes[0].contract.schema" in m["path"] for m in mismatches) + + def test_qos_mutation_is_detected(self) -> None: + seed = load_seed(BUNDLE_DIR) + mutated = copy.deepcopy(seed.fluid) + # Find an expose with qos and mutate it + for expose in mutated.get("exposes", []): + if "qos" in expose: + expose["qos"]["availability"] = "0%" + break + else: + pytest.skip("seed fixture had no qos to mutate") + mismatches = diff_against_seed(seed, mutated) + assert any(".qos" in m["path"] for m in mismatches) + + def test_adding_unrelated_fields_does_not_trigger_diff(self) -> None: + seed = load_seed(BUNDLE_DIR) + mutated = copy.deepcopy(seed.fluid) + # The LLM is allowed to add builds/executes/governance + mutated["builds"] = {"transform": "SELECT * FROM source"} + mutated["executes"] = {"schedule": "0 5 * * *"} + mismatches = diff_against_seed(seed, mutated) + assert mismatches == [] + + +# --------------------------------------------------------------------------- +# Path resolver helper +# --------------------------------------------------------------------------- + + +class TestResolveAtPath: + def test_resolves_array_index(self) -> None: + data = {"exposes": [{"id": "a"}, {"id": "b"}]} + assert resolve_at_path(data, "exposes[1].id") == "b" + + def test_returns_none_for_missing_path(self) -> None: + assert resolve_at_path({}, "exposes[0].id") is None diff --git a/tests/test_forge_seed_import_matrix.py b/tests/test_forge_seed_import_matrix.py new file mode 100644 index 00000000..d9382021 --- /dev/null +++ b/tests/test_forge_seed_import_matrix.py @@ -0,0 +1,257 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +"""Combination matrix: forge ``--seed-from`` × ``opds import`` × input shapes. + +Cross-product test that for every supported input shape: + + - S1: single Bitol ODPS file + - S2: lone ODCS file + - S3: directory bundle (ODPS + sibling ODCS files) + - S4: ODCS-only directory (no product wrapper) + +both the **import** path (``BitolOdpsProvider.import_*`` / ``OdcsProvider. +import_contract``) and the **forge seed** path +(``fluid_build.cli.forge_copilot_seed.load_seed``) produce an equivalent +FLUID skeleton — same number of exposes, same number of schema fields per +expose. + +Why both: the import path is what ``fluid opds import`` runs to produce a +final FLUID contract; the seed path is what ``fluid forge --seed-from`` +runs to give the LLM a structural starting point. They share most of the +underlying machinery but the seed path also computes provenance + ground- +truth paths. Drift between them is silent and catastrophic — these tests +make any drift loud. +""" + +from __future__ import annotations + +import logging +import shutil +from pathlib import Path +from typing import Tuple + +import pytest +import yaml + +from fluid_build.cli.forge_copilot_seed import ( + SHAPE_DIRECTORY, + SHAPE_ODCS_FILE, + SHAPE_ODCS_ONLY_DIRECTORY, + SHAPE_ODPS_FILE, + diff_against_seed, + load_seed, +) +from fluid_build.providers.odcs import OdcsProvider +from fluid_build.providers.odps_standard import BitolOdpsProvider + +# Quiet the resolver "tried 8 paths" warning that fires for the synthetic +# ``commerce.orders-product.input.customers_raw`` input port in our fixture +# — every contract in the matrix has unresolvable input ports because we're +# not feeding their upstream contracts. +logging.getLogger("fluid_build.providers.odps_standard.provider.BitolOdpsProvider").setLevel( + logging.ERROR +) + + +_FIXTURES = Path(__file__).parent / "fixtures" + + +# --------------------------------------------------------------------------- +# Cross-product fixture matrix +# --------------------------------------------------------------------------- + + +def _render_bundle(tmp_path: Path, fluid_path: Path) -> Path: + """Render the FLUID contract to a Bitol bundle so we have physical + S1/S2/S3/S4 shapes on disk.""" + with open(fluid_path) as f: + fluid = yaml.safe_load(f) + prov = BitolOdpsProvider() + prov.strict_validation = False + prov.render(fluid, out_dir=tmp_path) + return tmp_path + + +# Single small fixture from the repo's own tests/fixtures/ — keeps the test +# hermetic. The biz-lab integration is exercised by the broader live E2E in +# the docker-up sandbox; this test pins the matrix shape against a known +# multi-port contract. +FLUID_FIXTURE = _FIXTURES / "fluid" / "contract-multi-expose.fluid.yaml" + + +SHAPES = [ + pytest.param("S1-odps-file", SHAPE_ODPS_FILE, id="S1-odps-file"), + pytest.param("S2-odcs-file", SHAPE_ODCS_FILE, id="S2-odcs-file"), + pytest.param("S3-dir-bundle", SHAPE_DIRECTORY, id="S3-dir-bundle"), + pytest.param("S4-odcs-only", SHAPE_ODCS_ONLY_DIRECTORY, id="S4-odcs-only"), +] + + +@pytest.fixture +def bundle(tmp_path: Path) -> Path: + """Per-test bundle so writes don't bleed across cases.""" + return _render_bundle(tmp_path, FLUID_FIXTURE) + + +def _input_for_shape(bundle_dir: Path, shape_label: str, tmp_path: Path) -> Path: + """Materialise the right input path for each shape, derived from the + same bundle so all four shapes describe the same underlying contract. + """ + if shape_label == "S1-odps-file": + return next(bundle_dir.glob("*.odps.yaml")) + if shape_label == "S2-odcs-file": + return sorted(bundle_dir.glob("*.odcs.yaml"))[0] + if shape_label == "S3-dir-bundle": + return bundle_dir + if shape_label == "S4-odcs-only": + odcs_only = tmp_path / "odcs-only" + odcs_only.mkdir(exist_ok=True) + for odcs_file in bundle_dir.glob("*.odcs.yaml"): + shutil.copy(odcs_file, odcs_only) + return odcs_only + raise ValueError(shape_label) + + +def _import_fluid(input_path: Path) -> dict: + """OP1: what ``fluid opds import`` runs to produce a final FLUID.""" + prov = BitolOdpsProvider() + prov.strict_validation = False + if input_path.is_dir(): + return prov.import_directory(input_path) + if input_path.name.endswith(".odcs.yaml"): + return OdcsProvider().import_contract(input_path) + return prov.import_contract(input_path) + + +def _seed_fluid(input_path: Path) -> dict: + """OP2: what ``fluid forge --seed-from`` runs (the pre-processor).""" + return load_seed(input_path, allow_remote=False).fluid + + +def _counts(fluid: dict) -> Tuple[int, int]: + """Two numbers we care about across both paths: number of exposes and + total schema fields summed across exposes.""" + n_exposes = len(fluid.get("exposes") or []) + n_fields = sum(len(e.get("contract", {}).get("schema", [])) for e in fluid.get("exposes") or []) + return n_exposes, n_fields + + +# --------------------------------------------------------------------------- +# OP1 — opds import: every shape produces a valid FLUID +# --------------------------------------------------------------------------- + + +class TestImportEveryShape: + @pytest.mark.parametrize("shape_label,_", SHAPES) + def test_import_yields_well_formed_fluid( + self, bundle: Path, tmp_path: Path, shape_label: str, _: str + ) -> None: + input_path = _input_for_shape(bundle, shape_label, tmp_path) + fluid = _import_fluid(input_path) + assert "exposes" in fluid + assert isinstance(fluid["exposes"], list) + # Every shape must produce at least one expose with a populated schema + n_exposes, n_fields = _counts(fluid) + assert n_exposes >= 1, f"{shape_label}: no exposes produced" + assert n_fields >= 1, f"{shape_label}: no schema fields imported" + + +# --------------------------------------------------------------------------- +# OP2 — forge --seed-from: every shape produces a seed with the right shape +# label, provenance, and a working diff guard +# --------------------------------------------------------------------------- + + +class TestSeedEveryShape: + @pytest.mark.parametrize("shape_label,expected_shape", SHAPES) + def test_seed_shape_detected( + self, + bundle: Path, + tmp_path: Path, + shape_label: str, + expected_shape: str, + ) -> None: + input_path = _input_for_shape(bundle, shape_label, tmp_path) + seed = load_seed(input_path, allow_remote=False) + assert seed.shape == expected_shape + + @pytest.mark.parametrize("shape_label,_", SHAPES) + def test_seed_provenance_non_empty( + self, bundle: Path, tmp_path: Path, shape_label: str, _: str + ) -> None: + input_path = _input_for_shape(bundle, shape_label, tmp_path) + seed = load_seed(input_path, allow_remote=False) + # Every successful seed has at least one provenance entry + assert len(seed.provenance) >= 1 + + @pytest.mark.parametrize("shape_label,_", SHAPES) + def test_seed_ground_truth_paths_match_exposes( + self, bundle: Path, tmp_path: Path, shape_label: str, _: str + ) -> None: + """``ground_truth_paths`` covers every expose's + ``contract.schema``.""" + input_path = _input_for_shape(bundle, shape_label, tmp_path) + seed = load_seed(input_path, allow_remote=False) + n_exposes = len(seed.fluid.get("exposes", [])) + schema_paths = [p for p in seed.ground_truth_paths if p.endswith(".contract.schema")] + assert len(schema_paths) == n_exposes + + @pytest.mark.parametrize("shape_label,_", SHAPES) + def test_seed_diff_guard_catches_schema_mutation( + self, bundle: Path, tmp_path: Path, shape_label: str, _: str + ) -> None: + """Wiping the first expose's schema must trip the ground-truth diff.""" + input_path = _input_for_shape(bundle, shape_label, tmp_path) + seed = load_seed(input_path, allow_remote=False) + if not seed.fluid.get("exposes"): + pytest.skip("no exposes to mutate") + mutated = yaml.safe_load(yaml.dump(seed.fluid)) + mutated["exposes"][0].setdefault("contract", {})["schema"] = [] + mismatches = diff_against_seed(seed, mutated) + assert mismatches, f"{shape_label}: diff guard failed to catch schema mutation" + + @pytest.mark.parametrize("shape_label,_", SHAPES) + def test_seed_diff_guard_identity_is_clean( + self, bundle: Path, tmp_path: Path, shape_label: str, _: str + ) -> None: + """Feeding the seed itself back through the diff must produce zero + mismatches.""" + input_path = _input_for_shape(bundle, shape_label, tmp_path) + seed = load_seed(input_path, allow_remote=False) + assert diff_against_seed(seed, seed.fluid) == [] + + +# --------------------------------------------------------------------------- +# Cross-check: import and seed paths agree on (#exposes, #fields) +# --------------------------------------------------------------------------- + + +class TestImportAndSeedAgree: + """The two paths share their underlying mappers — drift between them + would mean the LLM gets a different view of the contract than ``opds + import`` produces. These tests pin them together. + """ + + @pytest.mark.parametrize("shape_label,_", SHAPES) + def test_counts_match(self, bundle: Path, tmp_path: Path, shape_label: str, _: str) -> None: + input_path = _input_for_shape(bundle, shape_label, tmp_path) + op1 = _counts(_import_fluid(input_path)) + op2 = _counts(_seed_fluid(input_path)) + assert op1 == op2, ( + f"{shape_label}: import vs seed disagree. " + f"import={op1[0]}exposes/{op1[1]}fields " + f"seed={op2[0]}exposes/{op2[1]}fields" + ) + + @pytest.mark.parametrize("shape_label,_", SHAPES) + def test_expose_ids_match(self, bundle: Path, tmp_path: Path, shape_label: str, _: str) -> None: + """Same expose ids in same order — a stronger pin than just counts.""" + input_path = _input_for_shape(bundle, shape_label, tmp_path) + op1_ids = [e.get("id") for e in _import_fluid(input_path).get("exposes", [])] + op2_ids = [e.get("id") for e in _seed_fluid(input_path).get("exposes", [])] + assert op1_ids == op2_ids diff --git a/tests/test_forge_seed_real_llm.py b/tests/test_forge_seed_real_llm.py new file mode 100644 index 00000000..55deb395 --- /dev/null +++ b/tests/test_forge_seed_real_llm.py @@ -0,0 +1,237 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +"""Real-LLM end-to-end tests for the forge seed → ground-truth-guard loop. + +Skipped unless **all** of these are set in the environment: + - ``ANTHROPIC_API_KEY`` or ``ANTHROPIC_AUTH_TOKEN`` (auth) + - ``FLUID_REAL_LLM_TESTS=1`` (opt-in cost gate) + - ``FLUID_REAL_LLM_MODEL`` (optional; default ``claude-haiku-4-5-20251001``. + Haiku is the cheap CI adversary; run ``claude-sonnet-5`` for the + production-default happy path. Claude 5 responses lead with a + ``ThinkingBlock`` — see :func:`_text_of`.) + +The default test suite runs none of these — the Anthropic API is real and +consumes tokens. Engineers wanting to verify the full loop against a live +model set both env vars and run:: + + pytest tests/test_forge_seed_real_llm.py -v -s + +Each test runs one or two LLM calls (~5–25k tokens each on claude-haiku-4-5). +""" + +from __future__ import annotations + +import json +import logging +import os +import shutil +import sys +from pathlib import Path + +import pytest +import yaml + +from fluid_build.cli.forge_copilot_seed import ( + diff_against_seed, + format_ground_truth_prompt_block, + load_seed, +) +from fluid_build.providers.odps_standard import BitolOdpsProvider + +logging.getLogger("fluid_build.providers.odps_standard.provider.BitolOdpsProvider").setLevel( + logging.ERROR +) + + +# --------------------------------------------------------------------------- +# Gate: skip unless the operator opted in +# --------------------------------------------------------------------------- + + +pytestmark = [ + pytest.mark.skipif( + os.environ.get("FLUID_REAL_LLM_TESTS") != "1", + reason="Real-LLM tests are opt-in; set FLUID_REAL_LLM_TESTS=1 to enable", + ), + pytest.mark.skipif( + not (os.environ.get("ANTHROPIC_API_KEY") or os.environ.get("ANTHROPIC_AUTH_TOKEN")), + reason="Need ANTHROPIC_API_KEY or ANTHROPIC_AUTH_TOKEN to call the API", + ), +] + + +MODEL = os.environ.get("FLUID_REAL_LLM_MODEL", "claude-haiku-4-5-20251001") +MAX_TOKENS = 16000 + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture(scope="module") +def anthropic_client(): + anthropic = pytest.importorskip("anthropic") + if os.environ.get("ANTHROPIC_API_KEY"): + return anthropic.Anthropic(api_key=os.environ["ANTHROPIC_API_KEY"]) + # Session-ingress JWT — accepted as ``Bearer`` via ``auth_token`` + return anthropic.Anthropic( + auth_token=os.environ["ANTHROPIC_AUTH_TOKEN"], + base_url=os.environ.get("ANTHROPIC_BASE_URL", "https://api.anthropic.com"), + ) + + +@pytest.fixture(scope="module") +def seed_bundle(tmp_path_factory): + """Render a multi-port Bitol bundle from the in-tree multi-expose + fixture so the test is hermetic (no biz-lab dependency).""" + bundle_dir = tmp_path_factory.mktemp("seed_bundle") + fluid_path = Path(__file__).parent / "fixtures" / "fluid" / "contract-multi-expose.fluid.yaml" + with open(fluid_path) as f: + fluid = yaml.safe_load(f) + prov = BitolOdpsProvider() + prov.strict_validation = False + prov.render(fluid, out_dir=bundle_dir) + return bundle_dir + + +def _seed_view(seed) -> dict: + """The exact ground-truth block the production copilot injects — so this + E2E measures the real prompt (stash buckets stripped, FLUID-native only).""" + return format_ground_truth_prompt_block(seed)["ground_truth"] + + +def _text_of(msg) -> str: + """First ``text`` block of a response. + + Claude 5 models emit a ``ThinkingBlock`` before the text, so indexing + ``content[0]`` breaks; select by block type instead. + """ + for block in msg.content: + if getattr(block, "type", None) == "text": + return block.text + raise AssertionError( + f"no text block in response: {[getattr(b, 'type', None) for b in msg.content]}" + ) + + +def _parse_json_response(text: str) -> dict: + """LLMs sometimes wrap JSON in markdown fences; strip and parse.""" + text = text.strip() + if text.startswith("```"): + text = text.split("```", 2)[1].split("\n", 1)[1].rsplit("```", 1)[0] + return json.loads(text) + + +SYSTEM_PRESERVE = ( + "You scaffold FLUID data product contracts. The user gives you a SEED " + "with the schema and qos pinned by an upstream Bitol ODPS contract. " + "You MUST preserve those fields VERBATIM in your output — never add, " + "remove, or modify schema columns, qos values, or relationships. You " + "ONLY fill in `builds`, `executes`, and `governance`. Return a single " + "JSON object with top-level keys: `id`, `metadata`, `exposes`, " + "`builds`, `executes`, `governance`. Use the seed's `id`, `metadata`, " + "and `exposes` exactly as given." +) + + +# --------------------------------------------------------------------------- +# Tests +# --------------------------------------------------------------------------- + + +def test_seed_to_llm_preserves_ground_truth(anthropic_client, seed_bundle): + """Happy path: the LLM, given a preserve-verbatim system prompt and a + real seed, returns a FLUID dict that respects every ground-truth path. + """ + seed = load_seed(seed_bundle, allow_remote=False) + user = ( + f"SEED:\n```json\n{json.dumps(_seed_view(seed), indent=2)}\n```\n\n" + "INTENT: daily refresh, dbt builds, governance for the analytics " + "team. Return ONLY the JSON object." + ) + msg = anthropic_client.messages.create( + model=MODEL, + max_tokens=MAX_TOKENS, + system=SYSTEM_PRESERVE, + messages=[{"role": "user", "content": user}], + ) + llm_fluid = _parse_json_response(_text_of(msg)) + mismatches = diff_against_seed(seed, llm_fluid) + assert not mismatches, ( + f"Expected the LLM to preserve ground truth verbatim under the " + f"preserve-prompt; got {len(mismatches)} mismatches: " + f"{[m['path'] for m in mismatches]}" + ) + # The LLM should have added the LLM-owned blocks + assert set(llm_fluid).issuperset({"builds", "executes", "governance"}) + + +def test_guard_catches_mutation_then_repair_reduces_it(anthropic_client, seed_bundle): + """Adversarial path: an "improve the schema" prompt induces mutations, + the guard catches them, and a second-turn repair re-prompt with the + mismatch report measurably reduces them. + + We don't require the repair to fully converge in one extra turn — the + production loop allows 3 attempts. This test pins the property that + the repair turn improves things AT ALL, not that it always wins on the + first try. + """ + seed = load_seed(seed_bundle, allow_remote=False) + seed_view = _seed_view(seed) + sys_adv = ( + "You scaffold FLUID data product contracts. Use the SEED as a " + "starting point but feel free to IMPROVE the schema — add columns, " + "rename for clarity, normalize types. Return a JSON object with " + "`id`, `metadata`, `exposes`, `builds`, `executes`, `governance`." + ) + user_adv = ( + f"SEED:\n```json\n{json.dumps(seed_view, indent=2)}\n```\n\n" + "INTENT: Improve schemas to follow modern naming (snake_case lower) " + "and add a `created_at` timestamp to each table. Return ONLY JSON." + ) + msg1 = anthropic_client.messages.create( + model=MODEL, + max_tokens=MAX_TOKENS, + system=sys_adv, + messages=[{"role": "user", "content": user_adv}], + ) + candidate1 = _parse_json_response(_text_of(msg1)) + mismatches1 = diff_against_seed(seed, candidate1) + if not mismatches1: + pytest.skip( + "LLM ignored the 'improve schema' prompt — happy outcome but " + "this test needs an actual mutation to exercise the repair loop" + ) + + # Repair turn under the production preserve-verbatim system prompt + repair = ( + "Your previous response mutated ground-truth paths. Restore them:\n\n" + + "\n".join(f" • {m['path']}" for m in mismatches1[:8]) + + f"\n\nSeed for reference:\n```json\n{json.dumps(seed_view, indent=2)}\n```\n\n" + "Return the corrected JSON object with the seed's schema and qos " + "preserved exactly." + ) + msg2 = anthropic_client.messages.create( + model=MODEL, + max_tokens=MAX_TOKENS, + system=SYSTEM_PRESERVE, + messages=[ + {"role": "user", "content": user_adv}, + {"role": "assistant", "content": _text_of(msg1)}, + {"role": "user", "content": repair}, + ], + ) + candidate2 = _parse_json_response(_text_of(msg2)) + mismatches2 = diff_against_seed(seed, candidate2) + + assert len(mismatches2) <= len(mismatches1), ( + f"Repair turn should not make things worse: " + f"round 1 had {len(mismatches1)} mismatches, " + f"round 2 had {len(mismatches2)}: {[m['path'] for m in mismatches2[:5]]}" + ) diff --git a/tests/test_forge_seed_runtime_wiring.py b/tests/test_forge_seed_runtime_wiring.py new file mode 100644 index 00000000..7cb3bafc --- /dev/null +++ b/tests/test_forge_seed_runtime_wiring.py @@ -0,0 +1,311 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +"""Mocked-LLM tests for the forge copilot's Phase-7 seed → guard → repair loop. + +These tests verify the runtime wiring — the seed loads once, the system +prompt gains the preservation clause, every attempt's user prompt carries +the ``seed_ground_truth`` block, and the ground-truth diff after each +attempt drives the existing 3-attempt repair loop. + +No real LLM is called. ``call_llm`` is patched to return controlled JSON, +and the normalize/validate seams are patched so the tests focus on the +Phase-7 additions without needing a schema-perfect contract. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any, Dict, List +from unittest.mock import MagicMock, patch + +import pytest +import yaml + +from fluid_build.cli.forge_copilot_runtime import ( + CopilotGenerationError, + generate_copilot_artifacts, +) +from fluid_build.cli.forge_copilot_seed import SeedOptions, load_seed +from fluid_build.providers.odps_standard import BitolOdpsProvider + +FLUID_FIXTURE = Path(__file__).parent / "fixtures" / "fluid" / "contract-multi-expose.fluid.yaml" + + +# --------------------------------------------------------------------------- +# Helpers — construct a bundle + a controlled "LLM" that returns whatever we +# want across the three attempt slots. +# --------------------------------------------------------------------------- + + +@pytest.fixture +def seed_bundle(tmp_path_factory) -> Path: + bundle_dir = tmp_path_factory.mktemp("seed_bundle") + with open(FLUID_FIXTURE) as f: + fluid = yaml.safe_load(f) + prov = BitolOdpsProvider() + prov.strict_validation = False + prov.render(fluid, out_dir=bundle_dir) + return bundle_dir + + +@pytest.fixture +def seed_options(seed_bundle: Path) -> SeedOptions: + return SeedOptions(seed_from=seed_bundle, allow_remote=False) + + +def _preserving_contract_from_seed(seed_bundle: Path) -> Dict[str, Any]: + """Build a FLUID contract that preserves the seed's exposes verbatim — + the shape the LLM is expected to return under the preservation prompt. + """ + seed = load_seed(seed_bundle, allow_remote=False) + return { + "contract": {"id": (seed.fluid.get("contract") or {}).get("id")}, + "metadata": seed.fluid.get("metadata") or {}, + "exposes": [ + { + "id": e["id"], + "contract": {"schema": e["contract"]["schema"]}, + "qos": e.get("qos") or {}, + } + for e in seed.fluid.get("exposes") or [] + ], + } + + +def _mutated_contract(preserving: Dict[str, Any]) -> Dict[str, Any]: + """Drop the first expose's schema — the classic ground-truth violation.""" + mutated = json.loads(json.dumps(preserving)) + if mutated["exposes"]: + mutated["exposes"][0]["contract"]["schema"] = [] + return mutated + + +class _LlmScript: + """Return canned raw-JSON strings across N attempts.""" + + def __init__(self, payloads: List[Dict[str, Any]]): + self._payloads = list(payloads) + self.calls = 0 + + def __call__(self, provider_adapter, llm_config, system_prompt, user_prompt): + payload = self._payloads[min(self.calls, len(self._payloads) - 1)] + self.calls += 1 + # Capture the prompts the runtime built so tests can inspect them + self.last_system_prompt = system_prompt + self.last_user_prompt = user_prompt + return json.dumps(payload) + + +def _patched_seams(script: _LlmScript, preserving_contract: Dict[str, Any]): + """Patch the runtime seams that don't matter for the Phase-7 logic under + test — call_llm, normalize, validate — so the tests exercise only the + seed loading, prompt injection, and ground-truth diff loop. + """ + + def fake_normalize(payload, **_): + return { + "suggestions": payload.get("suggestions") or {}, + "contract": payload.get("contract") or {}, + "readme_markdown": payload.get("readme_markdown") or "", + "additional_files": payload.get("additional_files") or [], + } + + def fake_validate(normalized, **_): + return ([], []) + + return [ + patch("fluid_build.cli.forge_copilot_runtime.call_llm", side_effect=script), + patch( + "fluid_build.cli.forge_copilot_runtime.normalize_generation_payload", + side_effect=fake_normalize, + ), + patch( + "fluid_build.cli.forge_copilot_runtime.validate_generated_result", + side_effect=fake_validate, + ), + # get_llm_provider is called before call_llm; give it a stub so no + # real adapter is required. + patch( + "fluid_build.cli.forge_copilot_runtime.get_llm_provider", + return_value=MagicMock(), + ), + ] + + +def _minimal_llm_config(): + from fluid_build.cli.forge_copilot_llm_providers import LlmConfig + + return LlmConfig( + provider="anthropic", + model="mock-model", + endpoint="https://mock.example.com/v1/messages", + api_key="mock-key", + ) + + +def _minimal_discovery_report(): + from fluid_build.cli.forge_copilot_discovery import DiscoveryReport + + return DiscoveryReport(workspace_roots=["/tmp"]) + + +# --------------------------------------------------------------------------- +# Case 1 — happy path: LLM preserves ground truth on attempt 1 +# --------------------------------------------------------------------------- + + +def test_seed_happy_path_returns_on_first_attempt(seed_options, seed_bundle): + """A seed-aware LLM that preserves everything exits the loop after one + attempt, and never triggers the guard's re-prompt path. + """ + preserving = _preserving_contract_from_seed(seed_bundle) + script = _LlmScript([{"contract": preserving}]) + + patches = _patched_seams(script, preserving) + for p in patches: + p.start() + try: + result = generate_copilot_artifacts( + context={"project_goal": "demo"}, + llm_config=_minimal_llm_config(), + discovery_report=_minimal_discovery_report(), + seed_options=seed_options, + ) + finally: + for p in patches: + p.stop() + + assert script.calls == 1 + assert len(result.attempt_reports) == 1 + # System prompt gained the preservation clause + assert "GROUND TRUTH FROM SEED" in script.last_system_prompt + # User prompt carried the seed block + assert "seed_ground_truth" in script.last_user_prompt + + +# --------------------------------------------------------------------------- +# Case 2 — mutation → guard catches → attempt 2 preserves → success +# --------------------------------------------------------------------------- + + +def test_seed_mutation_triggers_repair_then_succeeds(seed_options, seed_bundle): + preserving = _preserving_contract_from_seed(seed_bundle) + mutated = _mutated_contract(preserving) + + script = _LlmScript([{"contract": mutated}, {"contract": preserving}]) + patches = _patched_seams(script, preserving) + for p in patches: + p.start() + try: + result = generate_copilot_artifacts( + context={"project_goal": "demo"}, + llm_config=_minimal_llm_config(), + discovery_report=_minimal_discovery_report(), + seed_options=seed_options, + ) + finally: + for p in patches: + p.stop() + + assert script.calls == 2 + assert len(result.attempt_reports) == 2 + # Attempt 1's report carries the ground-truth-violation strings + first_report_errors = result.attempt_reports[0].validation_errors + assert first_report_errors, "attempt 1 should record the ground-truth violation" + assert any("ground_truth" in e for e in first_report_errors) + # Attempt 2's user prompt carries the repair feedback derived from the + # attempt-1 mismatch report (via the existing previous_errors slot) + assert "ground_truth" in script.last_user_prompt + + +# --------------------------------------------------------------------------- +# Case 3 — persistent mutation → all 3 attempts fail → CopilotGenerationError +# --------------------------------------------------------------------------- + + +def test_seed_persistent_mutation_exhausts_repair_loop(seed_options, seed_bundle): + preserving = _preserving_contract_from_seed(seed_bundle) + mutated = _mutated_contract(preserving) + + script = _LlmScript([{"contract": mutated}] * 3) + patches = _patched_seams(script, preserving) + for p in patches: + p.start() + try: + with pytest.raises(CopilotGenerationError) as exc_info: + generate_copilot_artifacts( + context={"project_goal": "demo"}, + llm_config=_minimal_llm_config(), + discovery_report=_minimal_discovery_report(), + seed_options=seed_options, + ) + finally: + for p in patches: + p.stop() + + assert script.calls == 3 + err_msg = str(exc_info.value) + assert "3 attempts" in err_msg or "copilot_generation_failed" in err_msg + + +# --------------------------------------------------------------------------- +# Case 4 — no seed_options → prompts do NOT carry the seed block, loop +# behaves exactly as it did before Phase 7 +# --------------------------------------------------------------------------- + + +def test_no_seed_options_leaves_prompts_unchanged(seed_bundle): + preserving = _preserving_contract_from_seed(seed_bundle) + script = _LlmScript([{"contract": preserving}]) + patches = _patched_seams(script, preserving) + for p in patches: + p.start() + try: + generate_copilot_artifacts( + context={"project_goal": "demo"}, + llm_config=_minimal_llm_config(), + discovery_report=_minimal_discovery_report(), + seed_options=None, + ) + finally: + for p in patches: + p.stop() + + # No seed → no preservation clause, no seed_ground_truth block + assert "GROUND TRUTH FROM SEED" not in script.last_system_prompt + assert "seed_ground_truth" not in script.last_user_prompt + + +# --------------------------------------------------------------------------- +# Case 5 — SeedOptions.from_args parses the CLI namespace correctly +# --------------------------------------------------------------------------- + + +def test_seed_options_from_args_none_when_flag_absent(): + from types import SimpleNamespace + + assert SeedOptions.from_args(SimpleNamespace()) is None + assert SeedOptions.from_args(SimpleNamespace(seed_from=None)) is None + + +def test_seed_options_from_args_builds_dataclass(): + from types import SimpleNamespace + + opts = SeedOptions.from_args(SimpleNamespace(seed_from="/tmp/x.odps.yaml", seed_no_remote=True)) + assert opts is not None + assert str(opts.seed_from) == "/tmp/x.odps.yaml" + assert opts.allow_remote is False + + +def test_seed_options_from_args_defaults_allow_remote_true(): + from types import SimpleNamespace + + opts = SeedOptions.from_args(SimpleNamespace(seed_from="/tmp/x.odcs.yaml")) + assert opts is not None + assert opts.allow_remote is True diff --git a/tests/test_odcs_roundtrip.py b/tests/test_odcs_roundtrip.py new file mode 100644 index 00000000..2aba1d59 --- /dev/null +++ b/tests/test_odcs_roundtrip.py @@ -0,0 +1,289 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +"""Phase 1 — ODCS lossless round-trip tests. + +For each fixture: load → import → render → assert zero-diff via +:meth:`OdcsProvider.roundtrip_check`. Section-by-section invariants drill +in on individual lossless guarantees (team object, schema properties, SLA, +quality, relationships, primaryKey, custom properties). +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest +import yaml + +from fluid_build.providers.odcs import OdcsProvider + +FIXTURES = Path(__file__).parent / "fixtures" / "odcs" + + +def _load(name: str) -> dict: + with open(FIXTURES / name) as f: + return yaml.safe_load(f) + + +# --------------------------------------------------------------------------- +# Whole-document round-trip canary +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("fixture", ["contract-full.yaml", "contract-minimal.yaml"]) +def test_roundtrip_zero_diff(fixture: str) -> None: + """ODCS → FLUID → ODCS reproduces the original document byte-for-byte.""" + odcs = _load(fixture) + result = OdcsProvider().roundtrip_check(odcs) + assert result["equal"], ( + f"Round-trip diff for {fixture}:\n" + f" missing: {result['missing']}\n" + f" extra : {result['extra']}\n" + f" changed: {result['changed']}" + ) + + +# --------------------------------------------------------------------------- +# Section-by-section invariants on the full fixture +# --------------------------------------------------------------------------- + + +@pytest.fixture(scope="module") +def full_contract() -> dict: + return _load("contract-full.yaml") + + +@pytest.fixture(scope="module") +def imported(full_contract: dict) -> dict: + return OdcsProvider().import_contract(full_contract) + + +@pytest.fixture(scope="module") +def reexported(imported: dict) -> dict: + return OdcsProvider().render(imported) + + +class TestTeamObjectRoundTrip: + def test_team_object_survives_export(self, reexported: dict) -> None: + assert reexported["team"]["name"] == "commerce-team" + members = reexported["team"]["members"] + assert len(members) == 2 + assert members[0]["username"] == "alice@acme.com" + assert members[0]["role"] == "data-owner" + # dateIn is a non-FLUID-native field — must survive verbatim + assert members[0]["dateIn"] == "2024-01-01" + + def test_team_object_lands_in_fluid_owner(self, imported: dict) -> None: + owner = imported.get("owner") or {} + assert owner["team"] == "commerce-team" + assert owner["email"] == "alice@acme.com" + assert owner["role"] == "data-owner" + + def test_owner_contacts_carry_extra_members(self, imported: dict) -> None: + contacts = (imported.get("owner") or {}).get("contacts") or [] + assert any(c.get("email") == "bob@acme.com" for c in contacts) + + +class TestSchemaPropertyRoundTrip: + def test_required_field_round_trips(self, reexported: dict) -> None: + props = reexported["schema"][0]["properties"] + order_id = next(p for p in props if p["name"] == "order_id") + assert order_id["required"] is True + + def test_primary_key_round_trips_without_tag_pollution( + self, full_contract: dict, reexported: dict + ) -> None: + original_tags = full_contract["schema"][0]["properties"][0]["tags"] + reexported_tags = reexported["schema"][0]["properties"][0]["tags"] + assert ( + original_tags == reexported_tags + ), "primary-key boolean must not bleed into the tags list on round-trip" + assert reexported["schema"][0]["properties"][0].get("primaryKey") is True + + def test_property_level_quality_preserved_verbatim( + self, full_contract: dict, reexported: dict + ) -> None: + original_quality = full_contract["schema"][0]["properties"][0]["quality"] + reexported_quality = reexported["schema"][0]["properties"][0]["quality"] + assert reexported_quality == original_quality + + def test_physical_type_preserved(self, reexported: dict) -> None: + props = reexported["schema"][0]["properties"] + amount = next(p for p in props if p["name"] == "amount") + assert amount["physicalType"] == "NUMERIC" + + +class TestObjectLevelRoundTrip: + def test_relationships_pass_through(self, reexported: dict) -> None: + rels = reexported["schema"][0]["relationships"] + assert len(rels) == 1 + assert rels[0]["from"] == "orders.customer_id" + assert rels[0]["to"] == "customers.id" + + def test_object_quality_pass_through(self, reexported: dict) -> None: + quality = reexported["schema"][0]["quality"] + assert any(q.get("metric") == "rowCount" for q in quality) + + def test_physical_name_pass_through(self, reexported: dict) -> None: + assert reexported["schema"][0]["physicalName"] == "orders_v1" + + +class TestSlaRoundTrip: + def test_sla_properties_verbatim(self, full_contract: dict, reexported: dict) -> None: + # The verbatim pass-through path means the full slaProperties list is + # reproduced byte-for-byte, units and all. + original = full_contract["slaProperties"] + reexported_sla = reexported["slaProperties"] + assert len(reexported_sla) == len(original) + for orig, new in zip(original, reexported_sla): + assert new["property"] == orig["property"] + assert new["value"] == orig["value"] + assert new.get("unit") == orig.get("unit") + + +class TestTopLevelExtrasRoundTrip: + @pytest.mark.parametrize( + "field", + ["description", "tags", "domain", "tenant", "support", "price", "customProperties"], + ) + def test_field_present(self, full_contract: dict, reexported: dict, field: str) -> None: + assert reexported[field] == full_contract[field], f"{field} did not round-trip" + + +# --------------------------------------------------------------------------- +# Regression: the two original Phase-1 import bugs +# --------------------------------------------------------------------------- + + +def test_team_object_import_does_not_crash_on_dict_form() -> None: + """Original bug: _odcs_team_to_fluid_owner was typed for a string but + ODCS v3.1.0 ``team`` is an object — this used to TypeError on import.""" + odcs = _load("contract-full.yaml") + fluid = OdcsProvider().import_contract(odcs) + assert "owner" in fluid + assert fluid["owner"]["team"] == "commerce-team" + + +def test_required_is_read_not_isnullable() -> None: + """Original bug: _odcs_schema_to_field read ``isNullable`` (a field that + doesn't exist in v3.1.0) instead of ``required``.""" + odcs = _load("contract-full.yaml") + fluid = OdcsProvider().import_contract(odcs) + fields = fluid["exposes"][0]["contract"]["schema"] + order_id = next(f for f in fields if f["name"] == "order_id") + assert order_id["required"] is True + + +# --------------------------------------------------------------------------- +# Optional vowl second-pass validation on the way out +# (regression tests for the standards-body-borrowed validator) +# --------------------------------------------------------------------------- + + +class TestVowlSecondPassValidation: + """When ``vowl`` is installed and enabled, ``OdcsProvider.render`` should + run vowl's parser on the emitted ODCS as an independent native sanity + check. When vowl isn't installed the render path silently skips it. + """ + + def test_validate_via_vowl_skips_cleanly_when_not_installed(self, monkeypatch) -> None: + """If vowl can't be imported, ``validate_via_vowl`` returns None and + does NOT raise. Lets us keep it as an opt-in extra.""" + import sys + + from fluid_build.providers.odcs.validation import validate_via_vowl + + # Force the import to fail by hiding the vowl module + monkeypatch.setitem(sys.modules, "vowl", None) + result = validate_via_vowl({"id": "x", "apiVersion": "v3.1.0"}) + assert result is None + + def test_validate_via_vowl_returns_diagnostics_on_valid_odcs(self) -> None: + """Hand-built minimal ODCS goes through vowl cleanly and the + diagnostic dict carries the expected fields.""" + pytest.importorskip("vowl") + + from fluid_build.providers.odcs.validation import validate_via_vowl + + odcs = { + "version": "1.0.0", + "apiVersion": "v3.1.0", + "kind": "DataContract", + "id": "minimal.vowl-ok", + "status": "active", + "schema": [ + { + "name": "events", + "logicalType": "object", + "physicalType": "table", + "properties": [ + {"name": "id", "logicalType": "string", "required": True}, + {"name": "amount", "logicalType": "number"}, + ], + } + ], + } + diag = validate_via_vowl(odcs) + assert diag is not None + assert diag["api_version"] == "v3.1.0" + assert diag["schemas"] == ["events"] + # column-exists + logical-type + required ⇒ at least 3 derived checks + assert diag["total_checks"] >= 3 + + def test_validate_via_vowl_raises_on_missing_required_field(self) -> None: + """vowl surfaces its own validation errors as a ProviderError with a + ``vowl:`` prefix so the source is unambiguous.""" + pytest.importorskip("vowl") + + from fluid_build.providers.base import ProviderError + from fluid_build.providers.odcs.validation import validate_via_vowl + + # Drop the required ``apiVersion`` field — vowl rejects it + odcs = OdcsProvider().render(_load("contract-full.yaml")) + odcs.pop("apiVersion", None) + with pytest.raises(ProviderError, match="vowl"): + validate_via_vowl(odcs) + + def test_odcs_provider_runs_vowl_when_env_flag_set(self, monkeypatch, caplog) -> None: + """``ODCS_VOWL_VALIDATE=true`` flips on the second-pass at render-time + without needing any code-level opt-in.""" + pytest.importorskip("vowl") + + import logging + + monkeypatch.setenv("ODCS_VOWL_VALIDATE", "true") + provider = OdcsProvider() # picks up env at __init__ + with caplog.at_level(logging.INFO): + provider.render(_load("contract-full.yaml")) + # The export path should have logged the vowl diagnostic line + assert any("vowl: ODCS" in r.message for r in caplog.records), ( + f"expected a 'vowl: ODCS …' log line; got: " f"{[r.message for r in caplog.records]}" + ) + + def test_bitol_provider_propagates_strict_mode_to_vowl_on_render(self) -> None: + """``BitolOdpsProvider.render`` propagates its current + ``strict_validation`` to the per-port ``OdcsProvider`` so late toggles + in tests take effect.""" + pytest.importorskip("vowl") + + import yaml + + from fluid_build.providers.odps_standard import BitolOdpsProvider + + prov = BitolOdpsProvider() # ODPS_STRICT defaults to 'true' + assert prov.strict_validation is True + # Toggle off after construction — the per-port OdcsProvider must + # see the new value when render() runs (regression for the + # construction-time-only set bug). + prov.strict_validation = False + with open( + Path(__file__).parent / "fixtures" / "fluid" / "contract-multi-expose.fluid.yaml" + ) as f: + fluid = yaml.safe_load(f) + prov.render(fluid) + assert prov._odcs._vowl_validate_on_export is False diff --git a/tests/test_odps_bitol_export.py b/tests/test_odps_bitol_export.py new file mode 100644 index 00000000..4e8e4242 --- /dev/null +++ b/tests/test_odps_bitol_export.py @@ -0,0 +1,181 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +"""Phase 2 — Bitol ODPS provider export tests. + +Coverage: +- **Validity**: emitted ODPS doc validates against the vendored + odps-product-schema-v1.0.0.json; every per-port ODCS contract validates + against odcs-schema-v3.1.0.json. +- **Linking invariant**: ``port.contractId == contracts[contractId]["id"]`` + exactly, for every output port. The renderer asserts this internally too; + this test catches any future regression. +- **Layout**: ``out_dir`` writes 1 ODPS doc + N ODCS siblings with the + expected file names; ports carry only ``contractId`` references (no + embedded contract bodies). +- **Negative**: duplicate port names raise before any file is written. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest +import yaml + +from fluid_build.providers.base import ProviderError +from fluid_build.providers.odcs.validation import load_schema as load_odcs_schema +from fluid_build.providers.odcs.validation import validate as validate_odcs +from fluid_build.providers.odps_standard import BitolOdpsProvider +from fluid_build.providers.odps_standard.validation import load_schema as load_odps_schema +from fluid_build.providers.odps_standard.validation import validate as validate_odps + +FIXTURE = Path(__file__).parent / "fixtures" / "fluid" / "contract-multi-expose.fluid.yaml" + + +@pytest.fixture(scope="module") +def fluid() -> dict: + with open(FIXTURE) as f: + return yaml.safe_load(f) + + +@pytest.fixture(scope="module") +def bundle(fluid: dict) -> dict: + prov = BitolOdpsProvider() + prov.strict_validation = False # validity is tested explicitly below + return prov.render(fluid) + + +# --------------------------------------------------------------------------- +# Validity +# --------------------------------------------------------------------------- + + +class TestValidity: + def test_odps_doc_validates_against_vendored_schema(self, bundle: dict) -> None: + schema = load_odps_schema() + assert schema is not None, "Vendored ODPS schema must be present" + validate_odps(bundle["product"], schema) # raises on failure + + def test_every_odcs_contract_validates(self, bundle: dict) -> None: + schema = load_odcs_schema() + assert schema is not None + for contract_id, odcs in bundle["contracts"].items(): + validate_odcs(odcs, schema) + + +# --------------------------------------------------------------------------- +# Linking invariant +# --------------------------------------------------------------------------- + + +class TestLinkingInvariant: + def test_every_port_contract_id_matches_emitted_odcs_id(self, bundle: dict) -> None: + product = bundle["product"] + contracts = bundle["contracts"] + for port in product.get("outputPorts", []): + cid = port["contractId"] + assert cid in contracts, f"port {port['name']} references missing contract {cid}" + assert ( + contracts[cid]["id"] == cid + ), f"contract.id {contracts[cid]['id']!r} != port.contractId {cid!r}" + + def test_contract_id_follows_convention(self, bundle: dict) -> None: + product_id = bundle["product"]["id"] + for port in bundle["product"].get("outputPorts", []): + assert port["contractId"].startswith( + f"{product_id}." + ), f"contractId must be `{{productId}}.` — got {port['contractId']!r}" + + +# --------------------------------------------------------------------------- +# File layout (out_dir mode) +# --------------------------------------------------------------------------- + + +class TestOutDirLayout: + def test_writes_one_product_plus_n_contracts(self, fluid: dict, tmp_path: Path) -> None: + prov = BitolOdpsProvider() + prov.strict_validation = False + prov.render(fluid, out_dir=tmp_path) + + odps_files = list(tmp_path.glob("*.odps.yaml")) + odcs_files = list(tmp_path.glob("*.odcs.yaml")) + expected_contracts = len(fluid["exposes"]) + assert len(odps_files) == 1, f"expected 1 ODPS doc, got {len(odps_files)}" + assert ( + len(odcs_files) == expected_contracts + ), f"expected {expected_contracts} ODCS siblings, got {len(odcs_files)}" + + def test_product_doc_carries_only_references_no_embedded_contracts( + self, fluid: dict, tmp_path: Path + ) -> None: + prov = BitolOdpsProvider() + prov.strict_validation = False + prov.render(fluid, out_dir=tmp_path) + + odps_file = next(tmp_path.glob("*.odps.yaml")) + with open(odps_file) as f: + product = yaml.safe_load(f) + for port in product.get("outputPorts", []): + # Bitol fragments mode: only contractId references — no inline body + assert ( + "contract" not in port + ), "fragments mode must not embed contract bodies in the ODPS doc" + + def test_odcs_filenames_match_contract_ids(self, fluid: dict, tmp_path: Path) -> None: + prov = BitolOdpsProvider() + prov.strict_validation = False + bundle = prov.render(fluid, out_dir=tmp_path) + for contract_id in bundle["contracts"]: + expected = tmp_path / f"{contract_id}.odcs.yaml" + assert expected.exists(), f"expected {expected.name}" + + +# --------------------------------------------------------------------------- +# Coverage — FLUID expose fields land in the right ODCS slots +# --------------------------------------------------------------------------- + + +class TestCoverage: + def test_qos_lands_in_odcs_slaProperties(self, bundle: dict) -> None: + contracts = bundle["contracts"] + daily = contracts["commerce.orders-product.daily_orders"] + sla = daily.get("slaProperties", []) + properties = {entry["property"] for entry in sla} + assert "availability" in properties + assert "interval" in properties # from qos.freshnessSLO + + def test_owner_lands_in_odcs_team_object(self, bundle: dict) -> None: + team = bundle["contracts"]["commerce.orders-product.daily_orders"]["team"] + assert team["name"] == "commerce-team" + member = team["members"][0] + assert member["username"] == "alice@acme.com" + + def test_schema_fields_appear_in_properties(self, bundle: dict) -> None: + daily = bundle["contracts"]["commerce.orders-product.daily_orders"] + props = daily["schema"][0]["properties"] + names = {p["name"] for p in props} + assert names == {"order_date", "order_count", "total_amount"} + + +# --------------------------------------------------------------------------- +# Negative cases +# --------------------------------------------------------------------------- + + +class TestNegative: + def test_duplicate_expose_id_raises_before_writing(self, fluid: dict, tmp_path: Path) -> None: + bad = yaml.safe_load(yaml.dump(fluid)) + # Force a duplicate output-port name + bad["exposes"].append(dict(bad["exposes"][0])) + prov = BitolOdpsProvider() + prov.strict_validation = False + with pytest.raises(ProviderError, match="Duplicate output port name"): + prov.render(bad, out_dir=tmp_path) + # No files should have been written + assert list(tmp_path.iterdir()) == [] diff --git a/tests/test_odps_bitol_import.py b/tests/test_odps_bitol_import.py new file mode 100644 index 00000000..99b32c68 --- /dev/null +++ b/tests/test_odps_bitol_import.py @@ -0,0 +1,168 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +"""Phase 3 — three import entry points + round-trip canary. + +Three input shapes converge on one validated FLUID: + 1. Single ODPS file → BitolOdpsProvider.import_contract() + 2. Directory bundle → BitolOdpsProvider.import_directory() + 3. Single ODCS file → OdcsProvider.import_contract() + +Plus the canonical canary: a FLUID exported via Bitol → re-imported → exported +again yields a byte-equal ODPS product doc. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from fluid_build.providers.base import ProviderError +from fluid_build.providers.odcs import OdcsProvider +from fluid_build.providers.odps_standard import BitolOdpsProvider + +FIXTURES = Path(__file__).parent / "fixtures" +BUNDLE_DIR = FIXTURES / "odps" / "product-bitol" +CONTRACTS_ONLY = FIXTURES / "odps" / "contracts-only" +BROKEN = FIXTURES / "odps" / "product-bitol-broken" + + +def _bundle_product_path() -> Path: + return next(BUNDLE_DIR.glob("*.odps.yaml")) + + +def _bundle_first_odcs_path() -> Path: + return next(BUNDLE_DIR.glob("*.odcs.yaml")) + + +@pytest.fixture +def provider() -> BitolOdpsProvider: + prov = BitolOdpsProvider() + prov.strict_validation = False # fixtures aren't tuned for strict ODCS schema + return prov + + +# --------------------------------------------------------------------------- +# Entry 1: single ODPS file +# --------------------------------------------------------------------------- + + +class TestImportContract: + def test_emits_one_expose_per_output_port(self, provider: BitolOdpsProvider) -> None: + fluid = provider.import_contract(_bundle_product_path()) + assert len(fluid["exposes"]) == 2 + + def test_each_expose_carries_a_populated_schema(self, provider: BitolOdpsProvider) -> None: + fluid = provider.import_contract(_bundle_product_path()) + for expose in fluid["exposes"]: + schema = expose.get("contract", {}).get("schema", []) + assert schema, f"expose {expose['id']} has no schema" + assert all("name" in field for field in schema) + + def test_qos_carries_back_from_slaProperties(self, provider: BitolOdpsProvider) -> None: + fluid = provider.import_contract(_bundle_product_path()) + # daily_orders had qos.availability + freshnessSLO; sla preservation + # round-trips into the first expose's qos block. + daily = next(e for e in fluid["exposes"] if e["id"] == "daily_orders") + assert "qos" in daily + + def test_metadata_owner_team_present(self, provider: BitolOdpsProvider) -> None: + fluid = provider.import_contract(_bundle_product_path()) + assert fluid.get("owner", {}).get("team") == "commerce-team" + + def test_passthrough_source_captured(self, provider: BitolOdpsProvider) -> None: + fluid = provider.import_contract(_bundle_product_path()) + pt = (fluid.get("metadata") or {}).get("odps_passthrough") or {} + assert "source" in pt, "import must preserve the original ODPS doc" + + +# --------------------------------------------------------------------------- +# Entry 2: directory +# --------------------------------------------------------------------------- + + +class TestImportDirectory: + def test_directory_import_matches_file_import(self, provider: BitolOdpsProvider) -> None: + from_file = provider.import_contract(_bundle_product_path()) + from_dir = provider.import_directory(BUNDLE_DIR) + assert from_file == from_dir + + +class TestImportOdcsOnlyDirectory: + def test_emits_one_expose_per_odcs_file(self, provider: BitolOdpsProvider) -> None: + fluid = provider.import_directory(CONTRACTS_ONLY) + n_files = len(list(CONTRACTS_ONLY.glob("*.odcs.yaml"))) + assert len(fluid["exposes"]) == n_files + + def test_warning_recorded_in_passthrough(self, provider: BitolOdpsProvider) -> None: + fluid = provider.import_directory(CONTRACTS_ONLY) + pt = (fluid.get("metadata") or {}).get("odps_passthrough") or {} + assert pt.get("odcs_only_directory") is True + assert "odcs_only_warning" in pt + + +# --------------------------------------------------------------------------- +# Entry 3: lone ODCS file (delegates to OdcsProvider) +# --------------------------------------------------------------------------- + + +class TestImportLoneOdcs: + def test_lone_odcs_produces_single_expose(self) -> None: + fluid = OdcsProvider().import_contract(_bundle_first_odcs_path()) + assert len(fluid["exposes"]) == 1 + + def test_lone_odcs_schema_populated(self) -> None: + fluid = OdcsProvider().import_contract(_bundle_first_odcs_path()) + schema = fluid["exposes"][0]["contract"]["schema"] + assert schema, "lone ODCS import must produce a populated schema" + + +# --------------------------------------------------------------------------- +# Round-trip canary: bundle → import → re-export equals original product doc +# --------------------------------------------------------------------------- + + +class TestBundleRoundTrip: + def test_product_doc_round_trips_zero_diff( + self, provider: BitolOdpsProvider, tmp_path: Path + ) -> None: + # Export-then-import-then-re-export must reproduce the product doc. + fluid_in = provider.import_directory(BUNDLE_DIR) + bundle = provider.render(fluid_in, out_dir=tmp_path) + # Compare with the original product doc on disk + import yaml + + with open(_bundle_product_path()) as f: + original_product = yaml.safe_load(f) + assert bundle["product"] == original_product + + +# --------------------------------------------------------------------------- +# Negative cases +# --------------------------------------------------------------------------- + + +class TestNegative: + def test_dangling_contract_id_raises_for_output_port(self, provider: BitolOdpsProvider) -> None: + # Output ports MUST resolve; lenient=False (default) raises + with pytest.raises(ProviderError): + provider.import_contract(BROKEN / "dangling-port.odps.yaml", lenient=False) + + def test_dangling_contract_id_lenient_warns(self, provider: BitolOdpsProvider) -> None: + # With lenient=True, unresolved output ports are warn-and-continue + fluid = provider.import_contract(BROKEN / "dangling-port.odps.yaml", lenient=True) + # The expose stub still gets created — just without a contract body + assert len(fluid["exposes"]) == 1 + + def test_two_odps_docs_in_directory_raises(self, provider: BitolOdpsProvider) -> None: + with pytest.raises(ProviderError, match="found 2 ODPS docs"): + provider.import_directory(BROKEN / "two-odps-docs") + + def test_nonexistent_directory_raises(self, provider: BitolOdpsProvider) -> None: + with pytest.raises(ProviderError, match="is not a directory"): + provider.import_directory(FIXTURES / "does-not-exist") diff --git a/tests/test_odps_resolver.py b/tests/test_odps_resolver.py new file mode 100644 index 00000000..db7e9e7f --- /dev/null +++ b/tests/test_odps_resolver.py @@ -0,0 +1,186 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +"""Phase 3 — ContractResolver unit tests. + +Focused unit tests for the resolver: probe order, candidate filename +generation, http(s) refusal under --no-remote, cache behaviour, validation +gate, and the HTML-with-200 safety check. +""" + +from __future__ import annotations + +from pathlib import Path +from unittest.mock import patch + +import pytest + +from fluid_build.providers.odps_standard.resolver import ( + ContractNotFound, + ContractResolver, + ContractValidationError, + RemoteFetchDisabled, + ResolvedContract, + _looks_like_url, +) + +FIXTURE_DIR = Path(__file__).parent / "fixtures" / "odps" / "product-bitol" + + +def _first_odcs_id() -> str: + """The contractId baked into the fixture's daily_orders ODCS doc.""" + return "commerce.orders-product.daily_orders" + + +# --------------------------------------------------------------------------- +# Local file resolution +# --------------------------------------------------------------------------- + + +class TestLocalResolution: + def test_resolves_from_base_path_with_default_extensions(self) -> None: + resolver = ContractResolver(base_path=FIXTURE_DIR, allow_remote=False) + result = resolver.resolve(_first_odcs_id()) + assert result.source == "local" + assert result.odcs["id"] == _first_odcs_id() + + def test_index_directory_makes_lookups_first_hit(self) -> None: + resolver = ContractResolver(base_path=FIXTURE_DIR, allow_remote=False) + resolver.index_directory(FIXTURE_DIR) + result = resolver.resolve(_first_odcs_id()) + # Confirm the resolver took the index path, not the candidate probes + assert Path(result.origin).parent == FIXTURE_DIR + + def test_cache_hit_avoids_re_read(self) -> None: + resolver = ContractResolver(base_path=FIXTURE_DIR, allow_remote=False) + first = resolver.resolve(_first_odcs_id()) + with patch("fluid_build.providers.odps_standard.resolver.read_input") as mock_read: + second = resolver.resolve(_first_odcs_id()) + mock_read.assert_not_called() + assert first is second + + +class TestCandidateFilenames: + def test_local_candidates_probe_full_id_and_last_segment(self, tmp_path: Path) -> None: + resolver = ContractResolver(base_path=tmp_path, allow_remote=False) + candidates = [str(c) for c in resolver._local_candidates("foo.bar.baz")] + # Both the full id and the last segment must appear + assert any("foo.bar.baz" in c for c in candidates) + assert any("baz" in c and "foo.bar.baz" not in c for c in candidates) + + def test_local_candidates_include_default_subdirs(self, tmp_path: Path) -> None: + resolver = ContractResolver(base_path=tmp_path, allow_remote=False) + candidates = [str(c) for c in resolver._local_candidates("id")] + # Default subdirs: '', contracts, odcs, odcs/contracts + assert any("contracts" in c for c in candidates) + assert any("odcs" in c for c in candidates) + + +# --------------------------------------------------------------------------- +# Error paths +# --------------------------------------------------------------------------- + + +class TestErrors: + def test_contract_not_found_lists_first_5_candidates(self, tmp_path: Path) -> None: + resolver = ContractResolver(base_path=tmp_path, allow_remote=False) + with pytest.raises(ContractNotFound) as exc_info: + resolver.resolve("missing.contract") + # The error message must include candidates tried but cap at 5 + summary + msg = str(exc_info.value) + assert "missing.contract" in msg + assert "Tried:" in msg + + def test_remote_fetch_disabled_raises_on_url_contract_id(self, tmp_path: Path) -> None: + resolver = ContractResolver(base_path=tmp_path, allow_remote=False) + with pytest.raises(RemoteFetchDisabled): + resolver.resolve("https://example.com/contract.odcs.yaml") + + def test_url_hint_with_remote_disabled_raises(self, tmp_path: Path) -> None: + resolver = ContractResolver(base_path=tmp_path, allow_remote=False) + with pytest.raises(RemoteFetchDisabled): + resolver.resolve("anything", hint="https://example.com/c.yaml") + + +# --------------------------------------------------------------------------- +# URL detection +# --------------------------------------------------------------------------- + + +class TestLooksLikeUrl: + @pytest.mark.parametrize( + "value, expected", + [ + ("https://example.com/c.yaml", True), + ("http://example.com/c.yaml", True), + ("ftp://example.com/c.yaml", False), # not http(s) + ("file:///tmp/c.yaml", False), + ("just-an-id", False), + ("product.something", False), + ("https://", False), # no netloc + ], + ) + def test_detection(self, value: str, expected: bool) -> None: + assert _looks_like_url(value) is expected + + +# --------------------------------------------------------------------------- +# Remote fetch — mocked urlopen +# --------------------------------------------------------------------------- + + +class _FakeResponse: + def __init__(self, body: bytes, content_type: str = "application/json") -> None: + self._body = body + self.headers = {"Content-Type": content_type} + + def read(self, amt: int = -1) -> bytes: + # The resolver reads in bounded chunks; serve the body once. + body, self._body = self._body, b"" + return body + + def __enter__(self): + return self + + def __exit__(self, *_): + return False + + +class TestRemoteResolution: + def test_html_with_200_is_refused(self, tmp_path: Path) -> None: + resolver = ContractResolver(base_path=tmp_path, allow_remote=True) + with patch( + "fluid_build.providers.odps_standard.resolver.urlopen", + return_value=_FakeResponse(b"404", "text/html"), + ): + with pytest.raises(ContractNotFound): + resolver.resolve("https://example.com/c.odcs.yaml") + + def test_successful_remote_caches_result(self, tmp_path: Path) -> None: + import json + + body = json.dumps( + { + "version": "1.0.0", + "apiVersion": "v3.1.0", + "kind": "DataContract", + "id": "remote.contract", + "status": "active", + "schema": [], + "servers": [], + } + ).encode() + resolver = ContractResolver(base_path=tmp_path, allow_remote=True) + with patch( + "fluid_build.providers.odps_standard.resolver.urlopen", + return_value=_FakeResponse(body), + ) as mock_open: + first = resolver.resolve("https://example.com/remote.contract") + second = resolver.resolve("https://example.com/remote.contract") + mock_open.assert_called_once() + assert first is second + assert first.source == "remote" diff --git a/tests/test_opds_cli_spec.py b/tests/test_opds_cli_spec.py new file mode 100644 index 00000000..a54f00d5 --- /dev/null +++ b/tests/test_opds_cli_spec.py @@ -0,0 +1,245 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +"""Phase 4 — ``fluid opds`` CLI integration tests. + +Covers the ``--spec`` dispatcher (bitol-1.0.0 vs odpi-4.1), ``--out-dir`` +bundle writes, the import dispatch by input type, the legacy ``--version 4.1`` +deprecation alias, and the ``--no-remote`` flag. +""" + +from __future__ import annotations + +import logging +from argparse import Namespace +from pathlib import Path +from unittest.mock import patch +from urllib.error import URLError + +import pytest +import yaml + +from fluid_build.cli.opds import ( + SPEC_BITOL_1_0_0, + SPEC_ODPI_4_1, + cmd_opds_export, + cmd_opds_import, + resolve_spec, +) + +LOG = logging.getLogger(__name__) + +FIXTURES = Path(__file__).parent / "fixtures" +MULTI_EXPOSE_FLUID = FIXTURES / "fluid" / "contract-multi-expose.fluid.yaml" +BUNDLE_DIR = FIXTURES / "odps" / "product-bitol" + + +# --------------------------------------------------------------------------- +# resolve_spec — flag precedence +# --------------------------------------------------------------------------- + + +class TestResolveSpec: + def test_default_is_bitol(self) -> None: + args = Namespace(spec=None, version=None) + assert resolve_spec(args) == SPEC_BITOL_1_0_0 + + def test_explicit_spec_wins(self) -> None: + args = Namespace(spec=SPEC_ODPI_4_1, version="4.1") + assert resolve_spec(args) == SPEC_ODPI_4_1 + + def test_legacy_version_4_1_maps_to_odpi(self, caplog) -> None: + args = Namespace(spec=None, version="4.1") + with caplog.at_level(logging.WARNING): + assert resolve_spec(args) == SPEC_ODPI_4_1 + assert any("deprecated" in r.message for r in caplog.records) + + +# --------------------------------------------------------------------------- +# Export — Bitol path +# --------------------------------------------------------------------------- + + +class TestExportBitol: + def test_export_bundle_to_out_dir(self, tmp_path: Path) -> None: + args = Namespace( + contract=str(MULTI_EXPOSE_FLUID), + spec=SPEC_BITOL_1_0_0, + version=None, + out="-", + out_dir=str(tmp_path), + format="yaml", + env=None, + validate_strict=False, + pretty=True, + ) + rc = cmd_opds_export(args, LOG) + assert rc == 0 + odps_files = list(tmp_path.glob("*.odps.yaml")) + odcs_files = list(tmp_path.glob("*.odcs.yaml")) + assert len(odps_files) == 1 + assert len(odcs_files) >= 1 + + def test_export_bundle_emits_valid_product(self, tmp_path: Path) -> None: + from fluid_build.providers.odps_standard.validation import ( + load_schema, + validate, + ) + + args = Namespace( + contract=str(MULTI_EXPOSE_FLUID), + spec=SPEC_BITOL_1_0_0, + version=None, + out="-", + out_dir=str(tmp_path), + format="yaml", + env=None, + validate_strict=False, + pretty=True, + ) + cmd_opds_export(args, LOG) + + product_path = next(tmp_path.glob("*.odps.yaml")) + with open(product_path) as f: + product = yaml.safe_load(f) + validate(product, load_schema()) # raises on failure + + +# --------------------------------------------------------------------------- +# Import dispatch — file / directory / lone ODCS +# --------------------------------------------------------------------------- + + +def _import_args(path: Path, *, out: Path | None = None, **overrides) -> Namespace: + base = dict( + path=str(path), + spec=SPEC_BITOL_1_0_0, + version=None, + out=str(out) if out else None, + format="yaml", + no_remote=False, + lenient=False, + ) + base.update(overrides) + return Namespace(**base) + + +class TestImportDispatch: + def test_import_odps_file(self, tmp_path: Path) -> None: + out = tmp_path / "out.fluid.yaml" + product = next(BUNDLE_DIR.glob("*.odps.yaml")) + args = _import_args(product, out=out) + assert cmd_opds_import(args, LOG) == 0 + with open(out) as f: + fluid = yaml.safe_load(f) + assert len(fluid["exposes"]) == 2 + + def test_import_directory(self, tmp_path: Path) -> None: + out = tmp_path / "from-dir.fluid.yaml" + args = _import_args(BUNDLE_DIR, out=out) + assert cmd_opds_import(args, LOG) == 0 + with open(out) as f: + fluid = yaml.safe_load(f) + assert len(fluid["exposes"]) == 2 + + def test_import_lone_odcs_file(self, tmp_path: Path) -> None: + out = tmp_path / "from-odcs.fluid.yaml" + odcs = next(BUNDLE_DIR.glob("*.odcs.yaml")) + args = _import_args(odcs, out=out) + assert cmd_opds_import(args, LOG) == 0 + with open(out) as f: + fluid = yaml.safe_load(f) + assert len(fluid["exposes"]) == 1 + + def test_import_odpi_v4_1_rejected_as_export_only(self, tmp_path: Path) -> None: + product = next(BUNDLE_DIR.glob("*.odps.yaml")) + args = _import_args(product, spec=SPEC_ODPI_4_1) + rc = cmd_opds_import(args, LOG) + assert rc == 2 + + def test_import_nonexistent_path_returns_1(self, tmp_path: Path) -> None: + args = _import_args(tmp_path / "does-not-exist.odps.yaml") + rc = cmd_opds_import(args, LOG) + assert rc == 1 + + def test_directory_with_two_odps_docs_returns_1(self) -> None: + broken = FIXTURES / "odps" / "product-bitol-broken" / "two-odps-docs" + args = _import_args(broken) + rc = cmd_opds_import(args, LOG) + assert rc == 1 + + +class TestImportDispatchYieldsEquivalentFluid: + def test_file_and_directory_imports_are_equivalent(self, tmp_path: Path) -> None: + out_file = tmp_path / "from-file.fluid.yaml" + out_dir = tmp_path / "from-dir.fluid.yaml" + product = next(BUNDLE_DIR.glob("*.odps.yaml")) + cmd_opds_import(_import_args(product, out=out_file), LOG) + cmd_opds_import(_import_args(BUNDLE_DIR, out=out_dir), LOG) + with open(out_file) as f: + file_fluid = yaml.safe_load(f) + with open(out_dir) as f: + dir_fluid = yaml.safe_load(f) + assert file_fluid == dir_fluid + + +# --------------------------------------------------------------------------- +# --no-remote propagation +# --------------------------------------------------------------------------- + + +class TestNoRemoteFlag: + @staticmethod + def _synth_doc(tmp_path: Path) -> Path: + # Synthetic ODPS doc whose port contractId is a URL — the resolver can + # only satisfy it with an http(s) fetch. + odps_doc = tmp_path / "synth.odps.yaml" + odps_doc.write_text( + yaml.dump( + { + "apiVersion": "v1.0.0", + "kind": "DataProduct", + "id": "synth.product", + "name": "synth-product", + "version": "1.0.0", + "status": "draft", + "outputPorts": [ + { + "name": "remote_only", + "version": "1.0.0", + "contractId": "https://example.invalid/c.odcs.yaml", + } + ], + } + ) + ) + return odps_doc + + def test_no_remote_flag_blocks_url_contract_ids(self, tmp_path: Path) -> None: + # With --no-remote the import must fail loud *without* touching the + # network. ``example.invalid`` never resolves, so the return code alone + # cannot tell the two behaviours apart — assert on the fetch itself. + args = _import_args(self._synth_doc(tmp_path), out=tmp_path / "f.yaml", no_remote=True) + with patch( + "fluid_build.providers.odps_standard.resolver.urlopen", + side_effect=URLError("network must not be used"), + ) as fake_urlopen: + rc = cmd_opds_import(args, LOG) + assert rc == 1 # output-port resolution required, no remote allowed + fake_urlopen.assert_not_called() + + def test_without_no_remote_the_fetch_is_attempted(self, tmp_path: Path) -> None: + # Control: the same doc without --no-remote does try the URL, so the + # test above would catch a regression that stops propagating the flag. + args = _import_args(self._synth_doc(tmp_path), out=tmp_path / "f.yaml", no_remote=False) + with patch( + "fluid_build.providers.odps_standard.resolver.urlopen", + side_effect=URLError("blocked in test"), + ) as fake_urlopen: + rc = cmd_opds_import(args, LOG) + assert rc == 1 + fake_urlopen.assert_called_once() diff --git a/tests/test_opds_ext.py b/tests/test_opds_ext.py index 55403dc7..28ead85b 100644 --- a/tests/test_opds_ext.py +++ b/tests/test_opds_ext.py @@ -68,7 +68,7 @@ class TestCmdOpdsExport: @patch("fluid_build.cli.opds.cprint") @patch("fluid_build.cli.bootstrap.build_provider") @patch("fluid_build.cli.bootstrap.load_contract_with_overlay") - def test_export_stdout_pretty(self, mock_load, mock_build, mock_cprint): + def test_export_stdout_pretty(self, mock_load, mock_build, mock_cprint, capsys): mock_load.return_value = {"id": "test", "name": "Test"} mock_provider = MagicMock() mock_provider.render.return_value = {"dataProductId": "test"} @@ -83,7 +83,9 @@ def test_export_stdout_pretty(self, mock_load, mock_build, mock_cprint): result = cmd_opds_export(args, LOG) assert result == 0 - mock_cprint.assert_called() + # The JSON payload is written raw to stdout, never through Rich. + mock_cprint.assert_not_called() + assert json.loads(capsys.readouterr().out) == {"dataProductId": "test"} @patch("fluid_build.cli.opds.cprint") @patch("fluid_build.cli.bootstrap.build_provider") diff --git a/tests/test_publish.py b/tests/test_publish.py index 832033ec..3d30c3d6 100644 --- a/tests/test_publish.py +++ b/tests/test_publish.py @@ -22,7 +22,7 @@ import pytest -from fluid_build.providers.catalogs.base import PublishResult +from fluid_build.providers.catalogs.base import BaseCatalogProvider, PublishResult # --------------------------------------------------------------------------- # Helpers @@ -66,6 +66,33 @@ def _make_args(**kwargs): return argparse.Namespace(**defaults) +class _NewApiProvider(BaseCatalogProvider): + """A catalog provider declaring the current ``publish(fluid, target)`` + API. ``fluid publish`` detects the API from the class's ``publish`` + signature; bare ``MagicMock`` providers take the 0.7.8 CatalogAsset + route (see TestPublishContractLegacyMockProvider).""" + + async def publish(self, fluid, target=None): # pragma: no cover - replaced + raise NotImplementedError + + async def update(self, fluid, target=None): # pragma: no cover - replaced + raise NotImplementedError + + async def verify(self, contract_id=None, *, asset_id=None): # pragma: no cover + raise NotImplementedError + + +def _new_api_provider(): + """A new-API provider whose methods are mocks the tests configure.""" + provider = _NewApiProvider({}) + provider.publish = AsyncMock() + provider.update = AsyncMock() + provider.verify = AsyncMock() + provider.health_check = AsyncMock(return_value=True) + provider.validate_target = MagicMock(return_value=(True, None)) + return provider + + def _run(coro): """Run an async coroutine synchronously.""" loop = asyncio.new_event_loop() @@ -113,7 +140,7 @@ def test_catalog_not_configured_returns_error(self, tmp_path): config = MagicMock() config.get_catalog_config.return_value = None - with patch.object(pub_mod, "load_contract", return_value={"id": "test"}): + with patch.object(pub_mod, "load_contract", return_value={"id": "test", "name": "Test"}): result = _run( pub_mod.publish_contract( contract_path=contract_path, @@ -134,7 +161,7 @@ def test_disabled_catalog_returns_error(self, tmp_path): config = MagicMock() config.get_catalog_config.return_value = {"enabled": False} - with patch.object(pub_mod, "load_contract", return_value={"id": "test"}): + with patch.object(pub_mod, "load_contract", return_value={"id": "test", "name": "Test"}): result = _run( pub_mod.publish_contract( contract_path=contract_path, @@ -156,7 +183,7 @@ def test_provider_creation_failure_returns_error(self, tmp_path): config.get_catalog_config.return_value = {"enabled": True} with ( - patch.object(pub_mod, "load_contract", return_value={"id": "test"}), + patch.object(pub_mod, "load_contract", return_value={"id": "test", "name": "Test"}), patch.object(pub_mod, "get_catalog_provider", side_effect=RuntimeError("no driver")), ): result = _run( @@ -176,17 +203,14 @@ def test_verify_only_asset_exists(self, tmp_path): contract_path = tmp_path / "c.fluid.yaml" contract_path.write_text("id: test") - asset = MagicMock() - asset.id = "asset-123" - provider = MagicMock() - provider.map_contract_to_asset.return_value = asset + provider = _new_api_provider() provider.verify = AsyncMock(return_value=True) config = MagicMock() config.get_catalog_config.return_value = {"enabled": True} with ( - patch.object(pub_mod, "load_contract", return_value={"id": "test"}), + patch.object(pub_mod, "load_contract", return_value={"id": "test", "name": "Test"}), patch.object(pub_mod, "get_catalog_provider", return_value=provider), ): result = _run( @@ -200,6 +224,7 @@ def test_verify_only_asset_exists(self, tmp_path): assert result.success is True assert result.details.get("verified") is True + provider.verify.assert_awaited_once_with("test") def test_verify_only_asset_missing(self, tmp_path): from fluid_build.cli import publish as pub_mod @@ -207,17 +232,14 @@ def test_verify_only_asset_missing(self, tmp_path): contract_path = tmp_path / "c.fluid.yaml" contract_path.write_text("id: test") - asset = MagicMock() - asset.id = "asset-123" - provider = MagicMock() - provider.map_contract_to_asset.return_value = asset + provider = _new_api_provider() provider.verify = AsyncMock(return_value=False) config = MagicMock() config.get_catalog_config.return_value = {"enabled": True} with ( - patch.object(pub_mod, "load_contract", return_value={"id": "test"}), + patch.object(pub_mod, "load_contract", return_value={"id": "test", "name": "Test"}), patch.object(pub_mod, "get_catalog_provider", return_value=provider), ): result = _run( @@ -232,23 +254,20 @@ def test_verify_only_asset_missing(self, tmp_path): assert result.success is False assert "not found" in result.error - def test_dry_run_valid_asset(self, tmp_path): + def test_dry_run_valid_target(self, tmp_path): from fluid_build.cli import publish as pub_mod contract_path = tmp_path / "c.fluid.yaml" contract_path.write_text("id: test") - asset = MagicMock() - asset.id = "asset-123" - provider = MagicMock() - provider.map_contract_to_asset.return_value = asset - provider.validate_asset.return_value = (True, None) + provider = _new_api_provider() + provider.validate_target.return_value = (True, None) config = MagicMock() config.get_catalog_config.return_value = {"enabled": True} with ( - patch.object(pub_mod, "load_contract", return_value={"id": "test"}), + patch.object(pub_mod, "load_contract", return_value={"id": "test", "name": "Test"}), patch.object(pub_mod, "get_catalog_provider", return_value=provider), ): result = _run( @@ -262,6 +281,11 @@ def test_dry_run_valid_asset(self, tmp_path): assert result.success is True assert result.details.get("dry_run") is True + # validate_target is called with (fluid, target); the target's + # contract_id is derived from the FLUID's id field. + (fluid_arg, target_arg), _ = provider.validate_target.call_args + assert fluid_arg == {"id": "test", "name": "Test"} + assert target_arg.contract_id == "test" def test_health_check_failure_returns_error(self, tmp_path): from fluid_build.cli import publish as pub_mod @@ -269,17 +293,14 @@ def test_health_check_failure_returns_error(self, tmp_path): contract_path = tmp_path / "c.fluid.yaml" contract_path.write_text("id: test") - asset = MagicMock() - asset.id = "asset-123" - provider = MagicMock() - provider.map_contract_to_asset.return_value = asset + provider = _new_api_provider() provider.health_check = AsyncMock(return_value=False) config = MagicMock() config.get_catalog_config.return_value = {"enabled": True} with ( - patch.object(pub_mod, "load_contract", return_value={"id": "test"}), + patch.object(pub_mod, "load_contract", return_value={"id": "test", "name": "Test"}), patch.object(pub_mod, "get_catalog_provider", return_value=provider), ): result = _run( @@ -297,13 +318,10 @@ def test_successful_publish(self, tmp_path): from fluid_build.cli import publish as pub_mod contract_path = tmp_path / "c.fluid.yaml" - contract_path.write_text("id: test") + contract_path.write_text("id: test\nname: Test") - asset = MagicMock() - asset.id = "asset-123" - expected = _make_result(success=True, asset_id="asset-123") - provider = MagicMock() - provider.map_contract_to_asset.return_value = asset + expected = _make_result(success=True, asset_id="test") + provider = _new_api_provider() provider.health_check = AsyncMock(return_value=True) provider.publish = AsyncMock(return_value=expected) @@ -311,7 +329,7 @@ def test_successful_publish(self, tmp_path): config.get_catalog_config.return_value = {"enabled": True} with ( - patch.object(pub_mod, "load_contract", return_value={"id": "test"}), + patch.object(pub_mod, "load_contract", return_value={"id": "test", "name": "Test"}), patch.object(pub_mod, "get_catalog_provider", return_value=provider), ): result = _run( @@ -324,24 +342,36 @@ def test_successful_publish(self, tmp_path): ) assert result.success is True - - def test_map_contract_failure_returns_error(self, tmp_path): + # publish() gets the full FLUID dict plus a CatalogTarget carrying + # the raw YAML for byte-preserving catalog storage. + (fluid_arg, target_arg), _ = provider.publish.call_args + assert fluid_arg == {"id": "test", "name": "Test"} + assert target_arg.contract_id == "test" + assert target_arg.contract_name == "Test" + assert target_arg.contract_yaml == "id: test\nname: Test" + + def test_publish_uses_name_fallback_when_id_missing(self, tmp_path): + """A FLUID contract without a top-level ``id`` falls back to + ``name``; the target still gets a usable identifier. + """ from fluid_build.cli import publish as pub_mod contract_path = tmp_path / "c.fluid.yaml" - contract_path.write_text("id: test") + contract_path.write_text("name: SalesProduct") - provider = MagicMock() - provider.map_contract_to_asset.side_effect = KeyError("name") + expected = _make_result(success=True, asset_id="SalesProduct") + provider = _new_api_provider() + provider.health_check = AsyncMock(return_value=True) + provider.publish = AsyncMock(return_value=expected) config = MagicMock() config.get_catalog_config.return_value = {"enabled": True} with ( - patch.object(pub_mod, "load_contract", return_value={"id": "test"}), + patch.object(pub_mod, "load_contract", return_value={"name": "SalesProduct"}), patch.object(pub_mod, "get_catalog_provider", return_value=provider), ): - result = _run( + _run( pub_mod.publish_contract( contract_path=contract_path, catalog_name="my-catalog", @@ -349,9 +379,94 @@ def test_map_contract_failure_returns_error(self, tmp_path): ) ) + (_, target_arg), _ = provider.publish.call_args + assert target_arg.contract_id == "SalesProduct" + assert target_arg.contract_name == "SalesProduct" + + +class TestPublishContractLegacyMockProvider: + """origin/main's (0.7.8) MagicMock-provider tests, restored: a provider + that doesn't declare ``publish(fluid, target)`` takes the CatalogAsset + route (``map_contract_to_asset`` → ``validate_asset`` → ``publish``).""" + + def _publish(self, tmp_path, provider, **kwargs): + from fluid_build.cli import publish as pub_mod + + contract_path = tmp_path / "c.fluid.yaml" + contract_path.write_text("id: test") + config = MagicMock() + config.get_catalog_config.return_value = {"enabled": True} + with patch.object(pub_mod, "load_contract", return_value={"id": "test", "name": "Test"}): + with patch.object(pub_mod, "get_catalog_provider", return_value=provider): + return _run( + pub_mod.publish_contract( + contract_path=contract_path, + catalog_name="my-catalog", + config=config, + **kwargs, + ) + ) + + def test_verify_only_asset_exists(self, tmp_path): + asset = MagicMock() + asset.id = "asset-123" + provider = MagicMock() + provider.map_contract_to_asset.return_value = asset + provider.verify = AsyncMock(return_value=True) + + result = self._publish(tmp_path, provider, verify_only=True) + + assert result.success is True + assert result.details.get("verified") is True + provider.verify.assert_awaited_once_with("asset-123") + + def test_dry_run_valid_asset(self, tmp_path): + asset = MagicMock() + asset.id = "asset-123" + provider = MagicMock() + provider.map_contract_to_asset.return_value = asset + provider.validate_asset.return_value = (True, None) + + result = self._publish(tmp_path, provider, dry_run=True) + + assert result.success is True + assert result.details.get("dry_run") is True + + def test_map_contract_failure_returns_error(self, tmp_path): + provider = MagicMock() + provider.map_contract_to_asset.side_effect = KeyError("name") + + result = self._publish(tmp_path, provider) + assert result.success is False assert "Failed to map contract to asset" in result.error + def test_dry_run_invalid_asset_returns_error(self, tmp_path): + asset = MagicMock() + asset.id = "asset-bad" + provider = MagicMock() + provider.map_contract_to_asset.return_value = asset + provider.validate_asset.return_value = (False, "missing required field") + + result = self._publish(tmp_path, provider, dry_run=True) + + assert result.success is False + assert result.error == "missing required field" + + def test_publish_passes_asset(self, tmp_path): + asset = MagicMock() + asset.id = "asset-123" + provider = MagicMock() + provider.map_contract_to_asset.return_value = asset + provider.health_check = AsyncMock(return_value=True) + provider.publish = AsyncMock(return_value=_make_result(asset_id="asset-123")) + + result = self._publish(tmp_path, provider) + + assert result.success is True + provider.publish.assert_awaited_once_with(asset) + assert asset.contract_yaml == "id: test" + # --------------------------------------------------------------------------- # format_results @@ -788,25 +903,21 @@ def test_show_metrics_plain_text(self, tmp_path): assert "Metrics" in calls or "Total" in calls or "Success" in calls def test_publish_contract_verbose_logging(self, tmp_path): - """Lines 212-213, 268-269: verbose=True triggers extra logger.info calls.""" + """Verbose=True triggers extra logger.info calls in the publish path.""" from fluid_build.cli import publish as pub_mod contract_path = tmp_path / "c.fluid.yaml" contract_path.write_text("id: test") - asset = MagicMock() - asset.id = "asset-xyz" - asset.name = "Test Asset" - provider = MagicMock() - provider.map_contract_to_asset.return_value = asset + provider = _new_api_provider() provider.health_check = AsyncMock(return_value=True) - provider.publish = AsyncMock(return_value=_make_result(success=True, asset_id="asset-xyz")) + provider.publish = AsyncMock(return_value=_make_result(success=True, asset_id="test")) config = MagicMock() config.get_catalog_config.return_value = {"enabled": True} with ( - patch.object(pub_mod, "load_contract", return_value={"id": "test"}), + patch.object(pub_mod, "load_contract", return_value={"id": "test", "name": "Test"}), patch.object(pub_mod, "get_catalog_provider", return_value=provider), ): loop = asyncio.new_event_loop() @@ -825,25 +936,22 @@ def test_publish_contract_verbose_logging(self, tmp_path): assert result.success is True def test_publish_skips_health_check_when_flag_set(self, tmp_path): - """Lines 293-305: health check is skipped when skip_health_check=True.""" + """Health check is skipped when skip_health_check=True.""" from fluid_build.cli import publish as pub_mod contract_path = tmp_path / "c.fluid.yaml" contract_path.write_text("id: test") - asset = MagicMock() - asset.id = "asset-abc" - provider = MagicMock() - provider.map_contract_to_asset.return_value = asset + provider = _new_api_provider() # health_check should NOT be called provider.health_check = AsyncMock(return_value=False) - provider.publish = AsyncMock(return_value=_make_result(success=True, asset_id="asset-abc")) + provider.publish = AsyncMock(return_value=_make_result(success=True, asset_id="test")) config = MagicMock() config.get_catalog_config.return_value = {"enabled": True} with ( - patch.object(pub_mod, "load_contract", return_value={"id": "test"}), + patch.object(pub_mod, "load_contract", return_value={"id": "test", "name": "Test"}), patch.object(pub_mod, "get_catalog_provider", return_value=provider), ): loop = asyncio.new_event_loop() @@ -862,24 +970,21 @@ def test_publish_skips_health_check_when_flag_set(self, tmp_path): provider.health_check.assert_not_called() assert result.success is True - def test_dry_run_invalid_asset_returns_error(self, tmp_path): - """Lines 283-291: dry_run with invalid asset returns error result.""" + def test_dry_run_invalid_target_returns_error(self, tmp_path): + """dry_run with a target that fails validation returns an error.""" from fluid_build.cli import publish as pub_mod contract_path = tmp_path / "c.fluid.yaml" contract_path.write_text("id: test") - asset = MagicMock() - asset.id = "asset-bad" - provider = MagicMock() - provider.map_contract_to_asset.return_value = asset - provider.validate_asset.return_value = (False, "missing required field") + provider = _new_api_provider() + provider.validate_target.return_value = (False, "missing required field") config = MagicMock() config.get_catalog_config.return_value = {"enabled": True} with ( - patch.object(pub_mod, "load_contract", return_value={"id": "test"}), + patch.object(pub_mod, "load_contract", return_value={"id": "test", "name": "Test"}), patch.object(pub_mod, "get_catalog_provider", return_value=provider), ): loop = asyncio.new_event_loop() @@ -897,3 +1002,54 @@ def test_dry_run_invalid_asset_returns_error(self, tmp_path): assert result.success is False assert result.error == "missing required field" + + +# ===================================================================== +# CLI: --target as an alias for --catalog +# (regression test for snowflake-biz-lab Taskfile compat) +# ===================================================================== + + +class TestCatalogTargetAlias: + """``fluid publish ... --target X`` parses as ``--catalog X``. + + The snowflake-biz-lab Taskfile uses ``--target`` in its ``publish:pre`` + task; we accept it so the lab's task definitions work unmodified. + """ + + def test_target_flag_parses_as_catalog(self): + import argparse + + from fluid_build.cli.publish import register + + parser = argparse.ArgumentParser() + subparsers = parser.add_subparsers() + register(subparsers) + + # Use --target (the alias) + ns = parser.parse_args(["publish", "/tmp/c.yaml", "--target", "datamesh-manager"]) + assert ns.catalog == "datamesh-manager" + + def test_target_short_form_dash_c_still_works(self): + import argparse + + from fluid_build.cli.publish import register + + parser = argparse.ArgumentParser() + subparsers = parser.add_subparsers() + register(subparsers) + + ns = parser.parse_args(["publish", "/tmp/c.yaml", "-c", "fluid-command-center"]) + assert ns.catalog == "fluid-command-center" + + def test_default_catalog_unchanged(self): + import argparse + + from fluid_build.cli.publish import register + + parser = argparse.ArgumentParser() + subparsers = parser.add_subparsers() + register(subparsers) + + ns = parser.parse_args(["publish", "/tmp/c.yaml"]) + assert ns.catalog == "fluid-command-center" diff --git a/tests/test_review_fixes2_r2_catalogs_dmm.py b/tests/test_review_fixes2_r2_catalogs_dmm.py new file mode 100644 index 00000000..63a00b2c --- /dev/null +++ b/tests/test_review_fixes2_r2_catalogs_dmm.py @@ -0,0 +1,1022 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Round-2 regression tests for the catalogs / Data Mesh Manager workstream. + +Every expected payload below was captured from origin/main (release 0.7.8) +with the same fake HTTP server, so these tests pin 0.7.8 behaviour for: + +* the deprecated ``CatalogAsset`` API on the built-in catalog providers + (asset fields, keyword forms, ``map_contract_to_asset`` overrides), +* legacy / duck-typed third-party catalog plugins in ``fluid publish``, +* rejecting non-contract files in ``fluid publish``, +* ``fluid publish --catalog datamesh-manager`` status/domain/version, +* the ODPS ``team.name`` without ``--team-id``, +* the ``dps`` spec token. +""" + +from __future__ import annotations + +import asyncio +import copy +import json +import warnings +from pathlib import Path +from types import SimpleNamespace +from typing import Any, Dict, List, Optional, Tuple +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest +import yaml + +from fluid_build.loader import load_contract +from fluid_build.providers.catalogs import ( + CATALOG_PROVIDERS, + BaseCatalogProvider, + CatalogAsset, + CatalogTarget, + PublishResult, +) +from fluid_build.providers.catalogs.datamesh_manager import DataMeshManagerCatalogProvider +from fluid_build.providers.catalogs.fluid_cc import FluidCommandCenterProvider +from fluid_build.providers.datamesh_manager import DataMeshManagerProvider + +REPO_ROOT = Path(__file__).resolve().parents[1] +HELLO = REPO_ROOT / "examples" / "01-hello-world" / "contract.fluid.yaml" +C360 = REPO_ROOT / "examples" / "customer360" / "contract.fluid.yaml" +MARKET_CONFIG = REPO_ROOT / "examples" / "market-config.yaml" + + +def _run(coro): + loop = asyncio.new_event_loop() + try: + return loop.run_until_complete(coro) + finally: + loop.close() + + +@pytest.fixture(autouse=True) +def _no_spec_env(monkeypatch): + monkeypatch.delenv("DMM_DATA_PRODUCT_SPEC", raising=False) + + +def _catalog_deprecations(caught) -> List[str]: + """The catalog API's own deprecation notices among *caught* (unrelated + ones, e.g. Python 3.12's ``datetime.utcnow``, are ignored).""" + markers = ("CatalogAsset", "asset_id", "map_contract_to_asset", "deprecated publish") + return [ + str(w.message) + for w in caught + if issubclass(w.category, DeprecationWarning) + and any(marker in str(w.message) for marker in markers) + ] + + +def _quiet_map(provider, contract): + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + return provider.map_contract_to_asset(contract) + + +def _contract(**overrides: Any) -> Dict[str, Any]: + contract: Dict[str, Any] = { + "fluidVersion": "0.7.2", + "kind": "DataProduct", + "id": "sales.orders", + "name": "Orders", + "description": "Orders data product", + "domain": "sales", + "metadata": {"owner": {"team": "sales-team", "email": "sales@example.com"}}, + "exposes": [], + } + contract.update(overrides) + return contract + + +# --------------------------------------------------------------------------- +# Fake Command Center (httpx) server +# --------------------------------------------------------------------------- + + +class _FakeResp: + def __init__(self, data: Any, status: int = 200) -> None: + self._data = data + self.status_code = status + self.text = json.dumps(data) + + def json(self) -> Any: + return self._data + + def raise_for_status(self) -> None: + return None + + +class _FakeCC: + """Stands in for ``httpx.AsyncClient``; records POST/PATCH bodies.""" + + sent: List[Dict[str, Any]] = [] + + def __init__(self, *a: Any, **k: Any) -> None: + pass + + async def __aenter__(self) -> "_FakeCC": + return self + + async def __aexit__(self, *a: Any) -> bool: + return False + + async def get(self, url, params=None, headers=None, timeout=None): + return _FakeResp({"items": []}) + + async def post(self, url, json=None, headers=None): + _FakeCC.sent.append(json) + return _FakeResp({"id": "new"}) + + async def patch(self, url, json=None, headers=None): + _FakeCC.sent.append(json) + return _FakeResp({"id": "new"}) + + +@pytest.fixture +def fake_cc(): + _FakeCC.sent = [] + with patch("httpx.AsyncClient", _FakeCC): + yield _FakeCC.sent + + +def _without_exposes_summary(payload: Dict[str, Any]) -> Dict[str, Any]: + """``metadata.exposes`` is the additive multi-port summary the branch + adds; everything else must equal 0.7.8's payload.""" + out = copy.deepcopy(payload) + out["metadata"].pop("exposes", None) + return out + + +def _edited_c360_asset(provider) -> CatalogAsset: + asset = _quiet_map(provider, load_contract(str(C360))) + asset.sensitivity = "restricted" + asset.tags = ["custom-tag"] + asset.description = "edited" + asset.name = "Edited Name" + asset.domain = "edited-domain" + asset.owner = "edited-owner" + asset.version = "9.9.9" + asset.layer = "Platinum" + asset.platform = "snowflake" + asset.location = {"db": "X"} + asset.schema = [{"name": "only", "type": "string"}] + asset.contract_yaml = "x: 1\n" + return asset + + +# Captured from origin/main for ``_edited_c360_asset``. +MAIN_CC_EDITED = { + "contract_hash": "d09edadd173a8bbc233d47dcafc30cc876af2fb569812f2bbe0e92a1905bbce8", + "contract_yaml": "x: 1\n", + "description": "edited", + "is_public": False, + "metadata": { + "domain": "edited-domain", + "fluid_contract_id": "gold.customer360_v1", + "layer": "Platinum", + "location": {"db": "X"}, + "owner": "edited-owner", + "owner_email": "cust-platform@example.com", + "platform": "snowflake", + "schema": [{"name": "only", "type": "string"}], + "sensitivity": "restricted", + }, + "name": "Edited Name", + "owner_id": "placeholder", + "tags": ["custom-tag"], + "type": "dataproduct", + "version": "9.9.9", +} + + +def _hand_asset() -> CatalogAsset: + return CatalogAsset( + id="hand.built", + name="Hand", + description="hd", + type="dataproduct", + domain="d", + owner="o", + owner_email="o@x", + layer="Gold", + tags=["t"], + version="2.0.0", + platform="unknown", + location={}, + schema=[{"id": "id", "type": "string"}], + sensitivity="restricted", + ) + + +# Captured from origin/main for ``_hand_asset``. +MAIN_CC_HAND = { + "description": "hd", + "is_public": False, + "metadata": { + "domain": "d", + "fluid_contract_id": "hand.built", + "layer": "Gold", + "location": {}, + "owner": "o", + "owner_email": "o@x", + "platform": "unknown", + "schema": [{"id": "id", "type": "string"}], + "sensitivity": "restricted", + }, + "name": "Hand", + "owner_id": "placeholder", + "tags": ["t"], + "type": "dataproduct", + "version": "2.0.0", +} + + +def _cc() -> FluidCommandCenterProvider: + return FluidCommandCenterProvider({"endpoint": "http://cc"}) + + +# --------------------------------------------------------------------------- +# Defect 0 — publish(CatalogAsset) honours every field set on the asset +# --------------------------------------------------------------------------- + + +class TestLegacyAssetFieldsHonoured: + @pytest.mark.parametrize("method", ["publish", "update", "publish_asset", "update_asset"]) + def test_cc_edited_asset_matches_0_7_8_payload(self, fake_cc, method): + provider = _cc() + asset = _edited_c360_asset(provider) + + with pytest.warns(DeprecationWarning): + result = _run(getattr(provider, method)(asset)) + + assert result.success, result.error + assert _without_exposes_summary(fake_cc[0]) == MAIN_CC_EDITED + + def test_cc_hand_built_asset_matches_0_7_8_payload(self, fake_cc): + with pytest.warns(DeprecationWarning): + result = _run(_cc().publish(_hand_asset())) + + assert result.success, result.error + assert _without_exposes_summary(fake_cc[0]) == MAIN_CC_HAND + assert fake_cc[0]["is_public"] is False + + def test_cc_unedited_asset_matches_new_api_payload(self, fake_cc): + provider = _cc() + contract = load_contract(str(C360)) + asset = _quiet_map(provider, contract) + with pytest.warns(DeprecationWarning): + _run(provider.publish(asset)) + _run(provider.publish(contract)) + assert fake_cc[0] == fake_cc[1] + + def test_minimal_fluid_keeps_schema_and_sensitivity(self): + from fluid_build.providers.catalogs.base import _fluid_target_from_asset + + fluid, target = _fluid_target_from_asset(_hand_asset()) + + assert target.contract_id == "hand.built" + expose = fluid["exposes"][0] + assert expose["contract"]["schema"] == [{"id": "id", "type": "string"}] + assert expose["sensitivity"] == "restricted" + + def test_third_party_new_api_provider_sees_asset_edits(self): + seen: Dict[str, Any] = {} + + class NewApi(BaseCatalogProvider): + async def publish(self, fluid, target=None): + seen["fluid"], seen["target"] = fluid, target + return PublishResult(True, "n", target.contract_id) + + async def update(self, fluid, target=None): + return await self.publish(fluid, target) + + async def verify(self, contract_id): + return True + + provider = NewApi({}) + contract = load_contract(str(C360)) + asset = _quiet_map(provider, contract) + asset.name = "Renamed" + asset.description = "edited" + asset.domain = "edited-domain" + asset.tags = ["custom-tag"] + asset.version = "9.9.9" + asset.sensitivity = "restricted" + asset.owner = "edited-owner" + + with pytest.warns(DeprecationWarning): + _run(provider.publish_asset(asset)) + + fluid = seen["fluid"] + assert seen["target"].contract_name == "Renamed" + assert fluid["name"] == fluid["metadata"]["name"] == "Renamed" + assert fluid["description"] == "edited" + assert fluid["domain"] == fluid["metadata"]["domain"] == "edited-domain" + assert fluid["metadata"]["tags"] == ["custom-tag"] + assert fluid["version"] == "9.9.9" + assert fluid["exposes"][0]["sensitivity"] == "restricted" + assert fluid["metadata"]["owner"]["team"] == "edited-owner" + # unedited parts of the original contract survive + assert fluid["exposes"][0]["binding"] == contract["exposes"][0]["binding"] + assert fluid["builds"] == contract["builds"] + # the caller's asset / raw contract are not mutated + assert contract["name"] == "Customer 360 (Demo)" + assert asset.raw_contract["name"] == "Customer 360 (Demo)" + + +# --------------------------------------------------------------------------- +# Fake Data Mesh Manager +# --------------------------------------------------------------------------- + + +def _dmm(config: Optional[Dict[str, Any]] = None, cls=DataMeshManagerCatalogProvider): + cfg: Dict[str, Any] = {"endpoint": "http://dmm", "auth": {"api_key": "k"}} + cfg.update(config or {}) + catalog = cls(cfg) + calls: List[Tuple[str, str, Any]] = [] + + def fake_request(method, path, json_body=None): + calls.append((method, path, json_body)) + return SimpleNamespace(status_code=200, json=lambda: {}, text="") + + catalog._provider._request = fake_request # type: ignore[assignment] + catalog._provider._session = lambda: SimpleNamespace( # type: ignore[assignment] + get=lambda *a, **k: SimpleNamespace(status_code=404) + ) + return catalog, calls + + +def _product(calls) -> Dict[str, Any]: + puts = [c[2] for c in calls if c[0] == "PUT" and c[1].startswith("/api/dataproducts/")] + assert len(puts) == 1 + return puts[0] + + +def _team_puts(calls) -> List[str]: + return [c[1].rsplit("/", 1)[-1] for c in calls if c[1].startswith("/api/teams/")] + + +# Captured from origin/main for ``_hand_asset`` (main's catalog path is DPS). +MAIN_DMM_HAND = { + "custom": {"domain": "d", "layer": "Gold", "version": "2.0.0"}, + "dataProductSpecification": "0.0.1", + "id": "hand.built", + "info": {"description": "hd", "owner": "o", "status": "active", "title": "Hand"}, + "tags": ["t"], + "teamId": "o", +} + + +class TestDmmLegacyAsset: + def test_hand_built_asset_dps_payload_equals_0_7_8(self): + catalog, calls = _dmm({"data_product_spec": "0.0.1"}) + with pytest.warns(DeprecationWarning): + result = _run(catalog.publish(_hand_asset())) + assert result.success, result.error + assert _product(calls) == MAIN_DMM_HAND + + @pytest.mark.parametrize("method", ["publish", "update", "publish_asset", "update_asset"]) + def test_edited_asset_fields_reach_dps_payload(self, method): + catalog, calls = _dmm({"data_product_spec": "0.0.1"}) + asset = _quiet_map(catalog, load_contract(str(C360))) + asset.name = "Renamed" + asset.domain = "edited-domain" + asset.tags = ["custom-tag"] + asset.description = "edited" + asset.owner = "Edited Owner" + asset.version = "9.9.9" + asset.layer = "Platinum" + + with pytest.warns(DeprecationWarning): + result = _run(getattr(catalog, method)(asset)) + + assert result.success, result.error + product = _product(calls) + # Values origin/main sent for the same asset. + assert product["info"]["title"] == "Renamed" + assert product["info"]["description"] == "edited" + assert product["info"]["status"] == "active" + assert product["info"]["owner"] == "edited-owner" + assert product["teamId"] == "edited-owner" + assert product["tags"] == ["custom-tag"] + assert product["custom"]["domain"] == "edited-domain" + assert product["custom"]["version"] == "9.9.9" + assert product["custom"]["layer"] == "Platinum" + assert _team_puts(calls) == ["edited-owner"] + + def test_edited_asset_fields_reach_odps_payload(self): + catalog, calls = _dmm() + asset = _quiet_map(catalog, load_contract(str(C360))) + asset.name = "Renamed" + asset.domain = "edited-domain" + asset.tags = ["custom-tag"] + asset.version = "9.9.9" + asset.owner = "Edited Owner" + + with pytest.warns(DeprecationWarning): + result = _run(catalog.publish(asset)) + + assert result.success, result.error + product = _product(calls) + assert product["name"] == "Renamed" + assert product["domain"] == "edited-domain" + assert product["tags"] == ["custom-tag"] + assert product["version"] == "9.9.9" + assert product["status"] == "active" + assert product["team"]["name"] == "edited-owner" + + +# --------------------------------------------------------------------------- +# Defect 4 — keyword forms of the 0.7.8 API +# --------------------------------------------------------------------------- + + +class TestLegacyKeywordForms: + @pytest.mark.parametrize("method", ["publish", "update"]) + def test_cc_asset_keyword(self, fake_cc, method): + provider = _cc() + with pytest.warns(DeprecationWarning): + result = _run(getattr(provider, method)(asset=_edited_c360_asset(provider))) + assert result.success, result.error + assert _without_exposes_summary(fake_cc[0]) == MAIN_CC_EDITED + + @pytest.mark.parametrize("method", ["publish", "update"]) + def test_dmm_asset_keyword(self, method): + catalog, calls = _dmm({"data_product_spec": "0.0.1"}) + with pytest.warns(DeprecationWarning): + result = _run(getattr(catalog, method)(asset=_hand_asset())) + assert result.success, result.error + assert _product(calls) == MAIN_DMM_HAND + + def test_cc_verify_asset_id_keyword(self, fake_cc): + provider = _cc() + with pytest.warns(DeprecationWarning, match="asset_id"): + assert _run(provider.verify(asset_id="x")) is False + # positional and the new keyword keep working without a warning + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always", DeprecationWarning) + assert _run(provider.verify("x")) is False + assert _run(provider.verify(contract_id="x")) is False + assert _catalog_deprecations(caught) == [] + + def test_cc_delete_asset_id_keyword(self): + provider = _cc() + with pytest.warns(DeprecationWarning, match="asset_id"): + assert _run(provider.delete(asset_id="x")) is False + + def test_dmm_verify_and_delete_asset_id_keyword(self): + catalog, calls = _dmm() + with pytest.warns(DeprecationWarning, match="asset_id"): + assert _run(catalog.verify(asset_id="p1")) is True + with pytest.warns(DeprecationWarning, match="asset_id"): + assert _run(catalog.delete(asset_id="p1")) is True + assert ("GET", "/api/dataproducts/p1", None) in calls + assert ("DELETE", "/api/dataproducts/p1", None) in calls + + def test_conflicting_or_missing_ids_raise_type_error(self): + provider = _cc() + with pytest.raises(TypeError): + _run(provider.verify()) + with pytest.raises(TypeError): + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + _run(provider.verify("a", asset_id="b")) + with pytest.raises(TypeError): + _run(provider.publish()) + + +# --------------------------------------------------------------------------- +# Defect 6 — non-contract files are rejected +# --------------------------------------------------------------------------- + + +def _cli_publish(path: Path, catalog, catalog_name="x", **kwargs) -> PublishResult: + from fluid_build.cli import publish as pub_mod + + config = MagicMock() + config.get_catalog_config.return_value = {"enabled": True} + with patch.object(pub_mod, "get_catalog_provider", return_value=catalog): + return _run( + pub_mod.publish_contract( + contract_path=path, catalog_name=catalog_name, config=config, **kwargs + ) + ) + + +class TestNonContractRejected: + def test_validate_target_rejects_dict_without_identity(self): + provider = _cc() + fluid = {"fluidVersion": "0.7.2", "builds": []} + ok, err = provider.validate_target(fluid, CatalogTarget.from_fluid(fluid)) + assert ok is False + assert "name" in err and "FLUID contract" in err + + @pytest.mark.parametrize("dry_run", [False, True]) + def test_cc_cli_rejects_market_config(self, fake_cc, dry_run): + result = _cli_publish(MARKET_CONFIG, _cc(), skip_health_check=True, dry_run=dry_run) + assert result.success is False + assert "FLUID contract" in result.error + assert fake_cc == [] + + def test_cc_cli_verify_only_rejects_non_contract(self, tmp_path): + path = tmp_path / "frag.yaml" + path.write_text("sovereignty:\n jurisdiction: EU\n", encoding="utf-8") + provider = _cc() + provider.verify = AsyncMock(return_value=True) # type: ignore[assignment] + result = _cli_publish(path, provider, verify_only=True) + assert result.success is False + assert "FLUID contract" in result.error + provider.verify.assert_not_called() + + def test_dmm_cli_rejects_fragment(self, tmp_path): + path = tmp_path / "frag.yaml" + path.write_text("sovereignty:\n jurisdiction: EU\n", encoding="utf-8") + catalog, calls = _dmm() + result = _cli_publish(path, catalog, skip_health_check=True) + assert result.success is False + assert "FLUID contract" in result.error + assert calls == [] + + def test_dmm_publish_rejects_dict_without_identity(self): + catalog, calls = _dmm() + result = _run(catalog.publish({"kind": "DataProduct", "metadata": {}})) + assert result.success is False + assert "FLUID contract" in result.error + assert calls == [] + + def test_non_mapping_yaml_is_rejected(self, tmp_path): + from fluid_build.cli import publish as pub_mod + + path = tmp_path / "list.yaml" + path.write_text("- a\n- b\n", encoding="utf-8") + # The loader rejects this itself ... + result = _cli_publish(path, _cc(), skip_health_check=True) + assert result.success is False + # ... and publish_contract does not crash if a loader hands it one. + with patch.object(pub_mod, "load_contract", return_value=["a", "b"]): + result = _cli_publish(path, _cc(), skip_health_check=True) + assert result.success is False + assert "FLUID contract" in result.error + + @pytest.mark.parametrize( + "contract", + [ + {"id": "a.b", "name": "A B"}, + {"name": "Only name"}, + {"metadata": {"name": "Only metadata name"}}, + ], + ) + def test_minimal_identity_still_passes(self, contract): + provider = _cc() + assert provider.validate_target(contract, CatalogTarget.from_fluid(contract)) == ( + True, + None, + ) + + def test_fragment_with_only_an_id_is_rejected(self): + # e.g. examples/0.7.1/bitcoin-multifile/fragments/builds/ingestion.yaml: + # 0.7.8 rejected it ("Failed to map contract to asset: 'name'"). + provider = _cc() + fragment = {"id": "bitcoin_price_ingestion", "pattern": "embedded-logic"} + ok, err = provider.validate_target(fragment, CatalogTarget.from_fluid(fragment)) + assert ok is False + assert "'name' is required" in err + + +# --------------------------------------------------------------------------- +# Defects 7 / 21 — legacy-plugin detection is positive for the new API +# --------------------------------------------------------------------------- + + +class _LegacyBase(BaseCatalogProvider): + events: List[Tuple[str, Any]] = [] + + async def update(self, asset): + return await self.publish(asset) + + async def verify(self, asset_id): + self.events.append(("verify", asset_id)) + return True + + +class LegacyExtraOptional(_LegacyBase): + async def publish(self, asset: CatalogAsset, dry_run: bool = False) -> PublishResult: + self.events.append(("publish", asset)) + return PublishResult(True, "legacykw", asset.id) + + +class LegacyVarArgs(_LegacyBase): + async def publish(self, asset, *args, **kw): + self.events.append(("publish", asset)) + return PublishResult(True, "legacyva", asset.id) + + +class NewApiExtraKw(BaseCatalogProvider): + async def publish(self, fluid, target=None, **kw): + return PublishResult(True, "n", target.contract_id if target else "?") + + async def update(self, fluid, target=None): + return await self.publish(fluid, target) + + async def verify(self, contract_id): + return True + + +class NewApiAnnotated(BaseCatalogProvider): + async def publish(self, fluid, tgt: Optional[CatalogTarget] = None): + return PublishResult(True, "n", "x") + + async def update(self, fluid, tgt: Optional[CatalogTarget] = None): + return PublishResult(True, "n", "x") + + async def verify(self, contract_id): + return True + + +class TestLegacyDetection: + @pytest.mark.parametrize("cls", [LegacyExtraOptional, LegacyVarArgs]) + def test_detected_as_legacy(self, cls): + assert cls._uses_legacy_asset_api() is True + + @pytest.mark.parametrize( + "cls", + [ + NewApiExtraKw, + NewApiAnnotated, + FluidCommandCenterProvider, + DataMeshManagerCatalogProvider, + ], + ) + def test_detected_as_new(self, cls): + assert cls._uses_legacy_asset_api() is False + + @pytest.mark.parametrize("cls", [LegacyExtraOptional, LegacyVarArgs]) + def test_cli_publishes_catalog_asset(self, tmp_path, cls): + cls.events = [] + path = tmp_path / "c.fluid.yaml" + path.write_text(yaml.safe_dump(_contract()), encoding="utf-8") + with pytest.warns(DeprecationWarning): + result = _cli_publish(path, cls({})) + assert result.success, result.error + published = [e for k, e in cls.events if k == "publish"] + assert len(published) == 1 and isinstance(published[0], CatalogAsset) + assert published[0].id == "sales.orders" + + @pytest.mark.parametrize("cls", [LegacyExtraOptional, LegacyVarArgs]) + def test_publish_asset_shim_routes_legacy(self, cls): + cls.events = [] + with pytest.warns(DeprecationWarning): + result = _run(cls({}).publish_asset(_hand_asset())) + assert result.success + assert isinstance(cls.events[0][1], CatalogAsset) + + +class DuckLegacy: + """A 0.7.8 catalog plugin that implements the interface without + subclassing BaseCatalogProvider.""" + + def __init__(self) -> None: + self.events: List[Tuple[str, Any]] = [] + + def map_contract_to_asset(self, contract): + self.events.append(("map", contract)) + return CatalogAsset( + id=contract["id"], + name=contract["name"], + description="", + type="dataproduct", + domain=contract.get("domain", "general"), + owner="o", + owner_email="", + layer="Bronze", + tags=[], + version="1.0.0", + platform="unknown", + location={}, + ) + + def validate_asset(self, asset): + self.events.append(("validate", asset)) + return (asset.domain != "blocked", "duck says no") + + async def health_check(self): + return True + + async def publish(self, asset): + self.events.append(("publish", asset)) + return PublishResult(True, "duck", asset.id) + + async def verify(self, asset_id): + self.events.append(("verify", asset_id)) + return True + + +class TestDuckTypedLegacyPlugin: + def _path(self, tmp_path, **overrides) -> Path: + path = tmp_path / "c.fluid.yaml" + path.write_text(yaml.safe_dump(_contract(**overrides)), encoding="utf-8") + return path + + def test_publish(self, tmp_path): + duck = DuckLegacy() + with pytest.warns(DeprecationWarning): + result = _cli_publish(self._path(tmp_path), duck) + assert result.success, result.error + assert [k for k, _ in duck.events] == ["map", "publish"] + + def test_dry_run_uses_validate_asset(self, tmp_path): + duck = DuckLegacy() + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + ok = _cli_publish(self._path(tmp_path), duck, dry_run=True) + blocked = _cli_publish(self._path(tmp_path, domain="blocked"), duck, dry_run=True) + assert ok.success is True + assert (blocked.success, blocked.error) == (False, "duck says no") + + def test_verify_only(self, tmp_path): + duck = DuckLegacy() + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + result = _cli_publish(self._path(tmp_path), duck, verify_only=True) + assert result.success is True + assert ("verify", "sales.orders") in duck.events + + def test_magicmock_provider_takes_asset_route(self, tmp_path): + """origin/main's own tests drive publish_contract with MagicMocks.""" + provider = MagicMock() + asset = MagicMock(id="sales.orders") + provider.map_contract_to_asset.return_value = asset + provider.validate_asset.return_value = (True, None) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + result = _cli_publish(self._path(tmp_path), provider, dry_run=True) + assert result.success is True + provider.validate_asset.assert_called_once_with(asset) + + +# --------------------------------------------------------------------------- +# Defect 8 — map_contract_to_asset overrides on built-in subclasses +# --------------------------------------------------------------------------- + + +class CCSub(FluidCommandCenterProvider): + def map_contract_to_asset(self, contract): + asset = super().map_contract_to_asset(contract) + asset.domain = "plugin-domain" + return asset + + +class DMMSub(DataMeshManagerCatalogProvider): + def map_contract_to_asset(self, contract): + asset = super().map_contract_to_asset(contract) + asset.name = "Renamed By Plugin" + asset.domain = "plugin-domain" + return asset + + +class TestMapOverrideHonoured: + def test_cc_subclass_override_reaches_payload(self, fake_cc, tmp_path): + path = tmp_path / "c.fluid.yaml" + path.write_text(yaml.safe_dump(_contract()), encoding="utf-8") + with pytest.warns(DeprecationWarning, match="map_contract_to_asset"): + result = _cli_publish(path, CCSub({"endpoint": "http://cc"}), skip_health_check=True) + assert result.success, result.error + assert fake_cc[0]["metadata"]["domain"] == "plugin-domain" + assert fake_cc[0]["contract_yaml"] == path.read_text(encoding="utf-8") + + def test_dmm_subclass_override_reaches_dps_payload(self): + catalog, calls = _dmm({"data_product_spec": "0.0.1"}, cls=DMMSub) + with pytest.warns(DeprecationWarning, match="map_contract_to_asset"): + result = _run(catalog.publish(_contract())) + assert result.success, result.error + product = _product(calls) + assert product["info"]["title"] == "Renamed By Plugin" + assert product["custom"]["domain"] == "plugin-domain" + + def test_dmm_subclass_override_reaches_odps_payload(self): + catalog, calls = _dmm(cls=DMMSub) + with pytest.warns(DeprecationWarning, match="map_contract_to_asset"): + result = _run(catalog.publish(_contract())) + assert result.success, result.error + product = _product(calls) + assert product["name"] == "Renamed By Plugin" + assert product["domain"] == "plugin-domain" + + def test_dry_run_validates_overridden_asset(self, tmp_path): + class EmptyDomain(FluidCommandCenterProvider): + def map_contract_to_asset(self, contract): + asset = super().map_contract_to_asset(contract) + asset.domain = "" + return asset + + path = tmp_path / "c.fluid.yaml" + path.write_text(yaml.safe_dump(_contract()), encoding="utf-8") + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + result = _cli_publish(path, EmptyDomain({"endpoint": "http://cc"}), dry_run=True) + assert (result.success, result.error) == (False, "Asset domain is required") + + def test_no_override_no_warning(self, fake_cc): + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always", DeprecationWarning) + result = _run(_cc().publish(_contract())) + assert result.success, result.error + assert _catalog_deprecations(caught) == [] + + +# --------------------------------------------------------------------------- +# Defect 1 — `fluid publish --catalog datamesh-manager` keeps 0.7.8 semantics +# --------------------------------------------------------------------------- + +# custom / info.status / teamId origin/main sent for the shipped examples. +MAIN_DMM_CLI = { + HELLO: { + "custom": {"domain": "example", "layer": "Bronze", "version": "1.0.0"}, + "status": "active", + "team": "learning-team", + }, + C360: { + "custom": {"domain": "Customer", "layer": "Gold", "version": "1.0.0"}, + "status": "active", + "team": "customer-platform", + }, +} + + +class TestDmmCatalogPathCompat: + @pytest.mark.parametrize("example", [HELLO, C360], ids=["hello", "c360"]) + @pytest.mark.parametrize("spec", ["0.0.1", "dps"]) + def test_dps_opt_out_matches_0_7_8(self, example, spec): + catalog, calls = _dmm({"data_product_spec": spec}) + result = _cli_publish(example, catalog, skip_health_check=True) + assert result.success, result.error + product = _product(calls) + expected = MAIN_DMM_CLI[example] + assert product["dataProductSpecification"] == "0.0.1" + for key, value in expected["custom"].items(): + assert product["custom"][key] == value + assert product["info"]["status"] == expected["status"] + assert product["teamId"] == expected["team"] + assert _team_puts(calls) == [expected["team"]] + + @pytest.mark.parametrize("example", [HELLO, C360], ids=["hello", "c360"]) + def test_odps_default_is_active_and_on_created_team(self, example): + catalog, calls = _dmm() + result = _cli_publish(example, catalog, skip_health_check=True) + assert result.success, result.error + product = _product(calls) + expected = MAIN_DMM_CLI[example] + assert product["status"] == "active" + assert product["domain"] == expected["custom"]["domain"] + assert product["version"] == expected["custom"]["version"] + assert product["team"]["name"] == expected["team"] + assert _team_puts(calls) == [expected["team"]] + + def test_declared_status_is_kept(self): + catalog, calls = _dmm({"data_product_spec": "0.0.1"}) + contract = _contract(metadata={"status": "deprecated", "owner": {"team": "t"}}) + result = _run(catalog.publish(contract)) + assert result.success, result.error + assert _product(calls)["info"]["status"] == "deprecated" + + def test_metadata_domain_wins_over_top_level(self): + catalog, calls = _dmm({"data_product_spec": "0.0.1"}) + contract = _contract(metadata={"domain": "meta-domain", "version": "3.0.0"}) + result = _run(catalog.publish(contract)) + assert result.success, result.error + custom = _product(calls)["custom"] + assert custom["domain"] == "meta-domain" + assert custom["version"] == "3.0.0" + + def test_caller_contract_not_mutated(self): + catalog, _ = _dmm() + contract = _contract() + before = copy.deepcopy(contract) + _run(catalog.publish(contract)) + assert contract == before + + +# --------------------------------------------------------------------------- +# Defect 2 — ODPS team.name equals the team _ensure_team creates +# --------------------------------------------------------------------------- + + +class TestOdpsTeamNameIsSlug: + def _provider(self): + provider = DataMeshManagerProvider(api_key="k", api_url="http://dmm") + calls: List[Tuple[str, str, Any]] = [] + + def fake_request(method, path, json_body=None): + calls.append((method, path, json_body)) + return SimpleNamespace(status_code=200, json=lambda: {}, text="") + + provider._request = fake_request # type: ignore[assignment] + provider._session = lambda: SimpleNamespace( # type: ignore[assignment] + get=lambda *a, **k: SimpleNamespace(status_code=404) + ) + return provider, calls + + def test_live_publish_customer360(self): + provider, calls = self._provider() + provider.apply(load_contract(str(C360))) + assert _team_puts(calls) == ["customer-platform"] + assert _product(calls)["team"]["name"] == "customer-platform" + + def test_dry_run_customer360(self): + provider, _ = self._provider() + result = provider.apply(load_contract(str(C360)), dry_run=True) + assert result["payload"]["team"]["name"] == "customer-platform" + + +# --------------------------------------------------------------------------- +# Defect 23 — the `dps` token means DPS 0.0.1 wherever a spec is accepted +# --------------------------------------------------------------------------- + + +class TestDpsToken: + @pytest.mark.parametrize("token", ["dps", "DPS", " Dps "]) + def test_env_var(self, monkeypatch, token): + monkeypatch.setenv("DMM_DATA_PRODUCT_SPEC", token) + provider = DataMeshManagerProvider(api_key="k", api_url="http://dmm") + result = provider.apply(_contract(), dry_run=True) + assert result["payload"]["dataProductSpecification"] == "0.0.1" + + @pytest.mark.parametrize("token", ["dps", "DPS"]) + def test_explicit_value(self, token): + provider = DataMeshManagerProvider(api_key="k", api_url="http://dmm") + result = provider.apply(_contract(), dry_run=True, data_product_specification=token) + assert result["payload"]["dataProductSpecification"] == "0.0.1" + plan = provider.plan(_contract(), data_product_specification=token) + assert plan[0]["payload"]["dataProductSpecification"] == "0.0.1" + + def test_catalog_config(self): + catalog, calls = _dmm({"data_product_spec": "dps"}) + result = _run(catalog.publish(_contract())) + assert result.success, result.error + assert _product(calls)["dataProductSpecification"] == "0.0.1" + + def test_cli_flag(self, tmp_path): + from fluid_build.cli.datamesh_manager import _cmd_publish + + path = tmp_path / "c.fluid.yaml" + path.write_text(yaml.safe_dump(_contract()), encoding="utf-8") + args = SimpleNamespace( + contract=str(path), + overlay=None, + team_id=None, + dry_run=True, + with_contract=False, + no_create_team=False, + contract_format="odcs", + data_product_spec="dps", + validate_generated_contracts=False, + validation_mode="warn", + fail_on_contract_error=False, + api_key="k", + api_url="http://dmm", + ) + with patch("fluid_build.cli.datamesh_manager._print_dry_run") as shown: + assert _cmd_publish(args) == 0 + assert shown.call_args[0][0]["payload"]["dataProductSpecification"] == "0.0.1" + + def test_other_values_pass_through(self): + provider = DataMeshManagerProvider(api_key="k", api_url="http://dmm") + resolve = provider._resolve_data_product_specification + assert resolve("0.0.1") == "0.0.1" + assert resolve("odps") == "odps" + assert resolve(" 0.0.2 ") == "0.0.2" + + +def test_legacy_registry_entry_uses_asset_route(tmp_path, monkeypatch): + """A legacy plugin registered by name in CATALOG_PROVIDERS (as the + verifier's ``legacykw`` plugin was) publishes through the asset flow.""" + from fluid_build.cli import publish as pub_mod + + LegacyExtraOptional.events = [] + monkeypatch.setitem(CATALOG_PROVIDERS, "legacykw", LegacyExtraOptional) + path = tmp_path / "c.fluid.yaml" + path.write_text(yaml.safe_dump(_contract()), encoding="utf-8") + config = MagicMock() + config.get_catalog_config.return_value = {"enabled": True} + with pytest.warns(DeprecationWarning): + result = _run( + pub_mod.publish_contract(contract_path=path, catalog_name="legacykw", config=config) + ) + assert result.success, result.error + assert isinstance(LegacyExtraOptional.events[-1][1], CatalogAsset) diff --git a/tests/test_review_fixes2_r2_cli_seed_docs.py b/tests/test_review_fixes2_r2_cli_seed_docs.py new file mode 100644 index 00000000..8e216ed2 --- /dev/null +++ b/tests/test_review_fixes2_r2_cli_seed_docs.py @@ -0,0 +1,733 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Round-2 regression tests: ``fluid forge --seed-from``, ``fluid opds`` CLI +edge cases, and the Bitol interop docs. + +Each test class names the defect it pins down. +""" + +from __future__ import annotations + +import argparse +import copy +import json +import logging +import os +import re +import shlex +import shutil +import subprocess +import sys +from argparse import Namespace +from pathlib import Path +from typing import Any, Dict, List +from unittest.mock import MagicMock, patch + +import pytest +import yaml + +from fluid_build.cli.forge_copilot_prompts import build_system_prompt +from fluid_build.cli.forge_copilot_runtime import generate_copilot_artifacts +from fluid_build.cli.forge_copilot_seed import ( + SeedOptions, + diff_against_seed, + format_ground_truth_prompt_block, + load_seed, +) +from fluid_build.cli.opds import cmd_opds_export, cmd_opds_import +from fluid_build.providers.base import ProviderError +from fluid_build.schema_manager import FluidSchemaManager + +LOG = logging.getLogger(__name__) + +REPO = Path(__file__).resolve().parent.parent +FIXTURES = Path(__file__).parent / "fixtures" +BUNDLE_DIR = FIXTURES / "odps" / "product-bitol" +HOURLY_ODCS = FIXTURES / "odps" / "contracts-only" / "hourly_orders.odcs.yaml" +MULTI_EXPOSE_FLUID = FIXTURES / "fluid" / "contract-multi-expose.fluid.yaml" +BITOL_DOC = REPO / "docs" / "BITOL_INTEROP.md" +README = REPO / "README.md" + +IDENTIFIER_RE = re.compile(r"^[a-z0-9_][a-z0-9_.-]*[a-z0-9_]$|^[a-z0-9_]$") + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _capability_matrix() -> Dict[str, Any]: + return { + "providers": ["local", "gcp", "aws", "snowflake"], + "templates": {"starter": {}, "analytics": {}}, + "build_engines": ["sql", "python", "dbt"], + } + + +def _contract_from_ground_truth(gt: Dict[str, Any]) -> Dict[str, Any]: + """A minimal FLUID 0.7.2 contract that copies the ground truth verbatim.""" + exposes = [] + for expose in gt["exposes"]: + schema = expose["contract"]["schema"] + first_col = schema[0]["name"] if schema else "x" + built = copy.deepcopy(expose) + built.update( + { + "kind": "table", + "binding": { + "platform": "local", + "format": "csv", + "location": {"path": "runtime/out/data.csv"}, + }, + "semantics": { + "name": "model", + "description": "Semantic model.", + "entities": [{"name": first_col, "type": "primary"}], + "measures": [{"name": "row_count", "agg": "count", "expr": first_col}], + "dimensions": [{"name": first_col, "type": "categorical"}], + "metrics": [{"name": "rows", "type": "simple", "measure": "row_count"}], + }, + } + ) + exposes.append(built) + contract: Dict[str, Any] = { + "fluidVersion": "0.7.2", + "kind": "DataProduct", + "id": gt.get("id") or "seeded.product", + "name": gt.get("name") or "Seeded Product", + "metadata": copy.deepcopy(gt.get("metadata")) + or {"owner": {"team": "data-team", "email": "data-team@example.com"}}, + "builds": [ + { + "id": "main_build", + "pattern": "embedded-logic", + "engine": "sql", + "properties": {"sql": "SELECT 1 AS id"}, + "execution": { + "trigger": {"type": "manual", "iterations": 1}, + "runtime": {"platform": "local", "resources": {"cpu": "1", "memory": "2Gi"}}, + }, + } + ], + "exposes": exposes, + } + for key in ("description", "domain", "tags"): + if key in gt: + contract[key] = copy.deepcopy(gt[key]) + contract["metadata"].setdefault("layer", "Bronze") + return contract + + +class _CopierLlm: + def __init__(self) -> None: + self.calls = 0 + + def __call__(self, provider_adapter, llm_config, system_prompt, user_prompt): + self.calls += 1 + gt = json.loads(user_prompt)["seed_ground_truth"]["ground_truth"] + return json.dumps( + { + "recommended_template": "starter", + "recommended_provider": "local", + "readme_markdown": "# Seeded\n", + "contract": _contract_from_ground_truth(gt), + } + ) + + +def _run_copilot(seed_path: Path, llm: _CopierLlm): + from fluid_build.cli.forge_copilot_discovery import DiscoveryReport + from fluid_build.cli.forge_copilot_llm_providers import LlmConfig + + call_patch = patch("fluid_build.cli.forge_copilot_runtime.call_llm", side_effect=llm) + provider_patch = patch( + "fluid_build.cli.forge_copilot_runtime.get_llm_provider", return_value=MagicMock() + ) + with call_patch, provider_patch: + return generate_copilot_artifacts( + context={"project_goal": "seeded product"}, + llm_config=LlmConfig( + provider="anthropic", + model="mock-model", + endpoint="https://mock.example.com/v1/messages", + api_key="mock-key", + ), + discovery_report=DiscoveryReport(workspace_roots=["/tmp"]), + capability_matrix=_capability_matrix(), + seed_options=SeedOptions(seed_from=seed_path, allow_remote=False), + ) + + +def _write_yaml(path: Path, data: Any) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(yaml.safe_dump(data, sort_keys=False), encoding="utf-8") + return path + + +def _odcs(contract_id: str, objects: List[str]) -> Dict[str, Any]: + return { + "version": "1.0.0", + "apiVersion": "v3.1.0", + "kind": "DataContract", + "id": contract_id, + "status": "active", + "name": "orders", + "schema": [ + { + "name": name, + "logicalType": "object", + "physicalType": "table", + "properties": [{"name": "id", "logicalType": "string"}], + } + for name in objects + ], + } + + +def _camel_odcs(tmp_path: Path) -> Path: + return _write_yaml(tmp_path / "orders.odcs.yaml", _odcs("minimal.contract", ["CustomerOrders"])) + + +def _spaced_port_bundle(tmp_path: Path) -> Path: + bundle = tmp_path / "bundle" + shutil.copytree(BUNDLE_DIR, bundle) + odps_path = next(bundle.glob("*.odps.yaml")) + doc = yaml.safe_load(odps_path.read_text(encoding="utf-8")) + doc["outputPorts"][0]["name"] = "Daily Orders" + _write_yaml(odps_path, doc) + return bundle + + +def _strict_errors(contract: Dict[str, Any]) -> List[Any]: + return FluidSchemaManager().validate_contract(contract, strict=True, offline_only=True).errors + + +# --------------------------------------------------------------------------- +# Defect 9 — exposeId in the ground-truth view must be a FLUID 0.7.2 identifier +# --------------------------------------------------------------------------- + + +class TestSeedExposeIdIsFluidIdentifier: + def test_uppercase_schema_name_is_sanitised_in_prompt(self, tmp_path: Path) -> None: + seed = load_seed(_camel_odcs(tmp_path), allow_remote=False) + gt = format_ground_truth_prompt_block(seed)["ground_truth"] + assert [e["exposeId"] for e in gt["exposes"]] == ["customerorders"] + + def test_spaced_port_name_is_sanitised_in_prompt(self, tmp_path: Path) -> None: + seed = load_seed(_spaced_port_bundle(tmp_path), allow_remote=False) + gt = format_ground_truth_prompt_block(seed)["ground_truth"] + ids = [e["exposeId"] for e in gt["exposes"]] + assert ids == ["daily_orders", "hourly_orders"] + assert all(IDENTIFIER_RE.match(i) for i in ids) + + @pytest.mark.parametrize("builder", [_camel_odcs, _spaced_port_bundle], ids=["camel", "spaced"]) + def test_verbatim_copy_passes_strict_validation_and_guard(self, tmp_path, builder) -> None: + seed = load_seed(builder(tmp_path), allow_remote=False) + block = json.loads(json.dumps(format_ground_truth_prompt_block(seed))) + contract = _contract_from_ground_truth(block["ground_truth"]) + assert _strict_errors(contract) == [] + assert diff_against_seed(seed, contract) == [] + + @pytest.mark.parametrize("builder", [_camel_odcs, _spaced_port_bundle], ids=["camel", "spaced"]) + def test_copilot_succeeds_on_first_attempt(self, tmp_path, builder) -> None: + llm = _CopierLlm() + result = _run_copilot(builder(tmp_path), llm) + assert llm.calls == 1 + assert result.attempt_reports[0].validation_errors == [] + + def test_guard_enforces_the_sanitised_expose_id(self, tmp_path: Path) -> None: + seed = load_seed(_spaced_port_bundle(tmp_path), allow_remote=False) + gt = format_ground_truth_prompt_block(seed)["ground_truth"] + contract = _contract_from_ground_truth(gt) + contract["exposes"][0]["exposeId"] = "renamed_orders" + mismatches = diff_against_seed(seed, contract) + assert [m["path"] for m in mismatches] == ["exposes[0].exposeId"] + assert mismatches[0]["seed"] == "daily_orders" + assert "exposes[0].exposeId" in format_ground_truth_prompt_block(seed)["ground_truth_paths"] + + def test_raw_seed_copy_is_still_clean(self, tmp_path: Path) -> None: + seed = load_seed(_spaced_port_bundle(tmp_path), allow_remote=False) + assert diff_against_seed(seed, seed.fluid) == [] + + def test_colliding_names_get_distinct_deterministic_ids(self, tmp_path: Path) -> None: + path = _write_yaml(tmp_path / "dup.odcs.yaml", _odcs("dup.contract", ["Orders", "orders"])) + seed = load_seed(path, allow_remote=False) + first = format_ground_truth_prompt_block(seed)["ground_truth"] + second = format_ground_truth_prompt_block(load_seed(path, allow_remote=False)) + ids = [e["exposeId"] for e in first["exposes"]] + assert ids == ["orders", "orders_2"] + assert ids == [e["exposeId"] for e in second["ground_truth"]["exposes"]] + contract = _contract_from_ground_truth(first) + assert _strict_errors(contract) == [] + assert diff_against_seed(seed, contract) == [] + + def test_valid_expose_ids_are_unchanged(self) -> None: + seed = load_seed(BUNDLE_DIR, allow_remote=False) + gt = format_ground_truth_prompt_block(seed)["ground_truth"] + assert [e["exposeId"] for e in gt["exposes"]] == ["daily_orders", "hourly_orders"] + + +# --------------------------------------------------------------------------- +# Defect 10 — the --seed-from docs must match what the seed actually enforces +# --------------------------------------------------------------------------- + + +def _forge_seed_help() -> str: + from fluid_build.cli import forge + + parser = argparse.ArgumentParser(prog="fluid") + forge.register(parser.add_subparsers(dest="cmd")) + forge_parser = parser._subparsers._group_actions[0].choices["forge"] + action = next(a for a in forge_parser._actions if a.dest == "seed_from") + return " ".join(action.help.split()) + + +def _doc_seed_section() -> str: + text = BITOL_DOC.read_text(encoding="utf-8") + section = text.split("## `fluid forge --seed-from`", 1)[1].split("\n## ", 1)[0] + return " ".join(section.split()) + + +class TestSeedDocsMatchCode: + def test_help_says_schema_and_qos_only(self) -> None: + help_text = _forge_seed_help() + assert "schema/quality/qos" not in help_text + assert "schema and qos" in help_text + assert "not carried" in help_text + assert "governance" not in help_text + + def test_bitol_doc_seed_section(self) -> None: + section = _doc_seed_section() + assert "schema/quality/qos" not in section + assert "schema and qos" in section + assert "not carried" in section + assert "governance" not in section + + def test_readme_seed_comment(self) -> None: + lines = README.read_text(encoding="utf-8").splitlines() + idx = next(i for i, ln in enumerate(lines) if "--seed-from" in ln) + comment_lines = [] + for ln in reversed(lines[:idx]): + if not ln.startswith("#"): + break + comment_lines.append(ln) + comment = " ".join(reversed(comment_lines)) + assert "governance" not in comment + assert "schema" in comment and "qos" in comment + + def test_system_prompt_does_not_promise_quality(self) -> None: + prompt = build_system_prompt(_capability_matrix(), ["sql"], has_seed_ground_truth=True) + clause = prompt.split("GROUND TRUTH FROM SEED", 1)[1] + assert "governance" not in clause + + +# --------------------------------------------------------------------------- +# Defect 18 — the seed directory scanner must not follow symlinks out +# --------------------------------------------------------------------------- + +SECRET = "SECRET_MARKER_987" + + +def _symlink_or_skip(link: Path, target: Path) -> None: + try: + link.symlink_to(target) + except (OSError, NotImplementedError): # pragma: no cover - platform dependent + pytest.skip("symlinks are not supported here") + + +def _secret_tree(tmp_path: Path) -> Path: + secret = _odcs("shop.orders", ["orders"]) + secret["schema"][0]["properties"] = [ + {"name": "ceo_salary", "logicalType": "integer", "description": SECRET} + ] + return _write_yaml(tmp_path / "private" / "secret.yaml", secret) + + +class TestSeedDirectoryContainment: + def test_bundle_symlink_out_of_root_is_not_read(self, tmp_path: Path) -> None: + secret = _secret_tree(tmp_path) + bundle = tmp_path / "bundle" + _write_yaml( + bundle / "p.odps.yaml", + { + "apiVersion": "v1.0.0", + "kind": "DataProduct", + "id": "shop", + "name": "Shop", + "status": "active", + "version": "1.0.0", + "outputPorts": [ + {"name": "orders", "version": "1.0.0", "contractId": "shop.orders"} + ], + }, + ) + _symlink_or_skip(bundle / "innocent.odcs.yaml", Path("..") / "private" / secret.name) + try: + seed = load_seed(bundle, allow_remote=False) + except ProviderError as exc: + assert SECRET not in str(exc) + else: # pragma: no cover - the port must not resolve to the outside file + assert SECRET not in json.dumps(seed.fluid, default=str) + pytest.fail("output port resolved through a symlink outside the bundle") + + def test_odcs_only_symlink_out_of_root_is_not_read(self, tmp_path: Path) -> None: + secret = _secret_tree(tmp_path) + bundle = tmp_path / "bundle2" + shutil.copy(HOURLY_ODCS, bundle.parent / "hourly.tmp") + bundle.mkdir() + shutil.move(str(bundle.parent / "hourly.tmp"), str(bundle / "hourly_orders.odcs.yaml")) + _symlink_or_skip(bundle / "innocent.odcs.yaml", secret.resolve()) + seed = load_seed(bundle, allow_remote=False) + assert SECRET not in json.dumps(seed.fluid, default=str) + assert SECRET not in json.dumps(format_ground_truth_prompt_block(seed)) + assert all("innocent" not in str(p.get("origin")) for p in seed.provenance) + + def test_symlinked_subdirectory_out_of_root_is_not_read(self, tmp_path: Path) -> None: + _secret_tree(tmp_path) + bundle = tmp_path / "bundle3" + bundle.mkdir() + shutil.copy(HOURLY_ODCS, bundle / "hourly_orders.odcs.yaml") + _symlink_or_skip(bundle / "linked", (tmp_path / "private").resolve()) + seed = load_seed(bundle, allow_remote=False) + assert SECRET not in json.dumps(seed.fluid, default=str) + + def test_only_outside_symlink_means_no_documents(self, tmp_path: Path) -> None: + secret = _secret_tree(tmp_path) + bundle = tmp_path / "bundle4" + bundle.mkdir() + _symlink_or_skip(bundle / "innocent.odcs.yaml", secret.resolve()) + with pytest.raises(ProviderError, match="contains no Bitol ODPS or ODCS documents"): + load_seed(bundle, allow_remote=False) + + def test_symlink_inside_root_is_still_followed(self, tmp_path: Path) -> None: + bundle = tmp_path / "bundle5" + (bundle / "real").mkdir(parents=True) + shutil.copy(HOURLY_ODCS, bundle / "real" / "hourly_orders.odcs.yaml") + _symlink_or_skip(bundle / "alias.odcs.yaml", Path("real") / "hourly_orders.odcs.yaml") + seed = load_seed(bundle, allow_remote=False) + assert seed.fluid.get("exposes") + + +# --------------------------------------------------------------------------- +# Defect 24 — no hint may recommend the deprecated `fluid odps-bitol` +# --------------------------------------------------------------------------- + + +class TestNoDeprecatedCommandHints: + def test_fluid_input_hint_points_at_opds_export(self, tmp_path: Path) -> None: + path = tmp_path / "c.yaml" + path.write_text(MULTI_EXPOSE_FLUID.read_text(encoding="utf-8"), encoding="utf-8") + with pytest.raises(ProviderError) as exc_info: + load_seed(path, allow_remote=False) + message = str(exc_info.value) + assert "odps-bitol" not in message + assert "fluid opds export --spec bitol-1.0.0" in message + + @pytest.mark.parametrize( + "path", + [ + REPO / "fluid_build" / "cli" / "forge_copilot_seed.py", + REPO / "fluid_build" / "cli" / "forge.py", + REPO / "fluid_build" / "cli" / "opds.py", + BITOL_DOC, + README, + ], + ids=lambda p: p.name, + ) + def test_owned_sources_do_not_recommend_odps_bitol(self, path: Path) -> None: + assert "fluid odps-bitol" not in path.read_text(encoding="utf-8") + + +# --------------------------------------------------------------------------- +# Defects 33 / 35 / 36 — fluid opds import/export CLI edge cases +# --------------------------------------------------------------------------- + + +def _parse(argv: List[str]) -> Namespace: + from fluid_build.cli.opds import register + + parser = argparse.ArgumentParser(prog="fluid") + register(parser.add_subparsers(dest="cmd")) + return parser.parse_args(argv) + + +class TestImportFormatFollowsOutSuffix: + def test_json_suffix_writes_json(self, tmp_path: Path) -> None: + out = tmp_path / "c.fluid.json" + args = _parse(["opds", "import", str(BUNDLE_DIR), "--no-remote", "-o", str(out)]) + assert cmd_opds_import(args, LOG) == 0 + data = json.loads(out.read_text(encoding="utf-8")) + assert data["exposes"] + + def test_yaml_suffix_writes_yaml(self, tmp_path: Path) -> None: + out = tmp_path / "c.fluid.yaml" + args = _parse(["opds", "import", str(BUNDLE_DIR), "--no-remote", "-o", str(out)]) + assert cmd_opds_import(args, LOG) == 0 + text = out.read_text(encoding="utf-8") + assert not text.lstrip().startswith("{") + assert yaml.safe_load(text)["exposes"] + + def test_explicit_format_wins_over_suffix(self, tmp_path: Path) -> None: + out = tmp_path / "c.fluid.json" + args = _parse( + ["opds", "import", str(BUNDLE_DIR), "--no-remote", "-o", str(out), "-f", "yaml"] + ) + assert cmd_opds_import(args, LOG) == 0 + assert not out.read_text(encoding="utf-8").lstrip().startswith("{") + + def test_unknown_suffix_defaults_to_yaml(self, tmp_path: Path) -> None: + out = tmp_path / "c.fluid" + args = _parse(["opds", "import", str(BUNDLE_DIR), "--no-remote", "-o", str(out)]) + assert cmd_opds_import(args, LOG) == 0 + assert not out.read_text(encoding="utf-8").lstrip().startswith("{") + + def test_stdout_defaults_to_yaml(self, capsys) -> None: + args = _parse(["opds", "import", str(BUNDLE_DIR), "--no-remote"]) + assert cmd_opds_import(args, LOG) == 0 + out = capsys.readouterr().out + assert not out.lstrip().startswith("{") + assert yaml.safe_load(out)["exposes"] + + def test_json_import_reexports(self, tmp_path: Path) -> None: + out = tmp_path / "c.fluid.json" + args = _parse(["opds", "import", str(BUNDLE_DIR), "--no-remote", "-o", str(out)]) + assert cmd_opds_import(args, LOG) == 0 + export_args = _parse(["opds", "export", str(out), "--out-dir", str(tmp_path / "rt")]) + assert cmd_opds_export(export_args, LOG) == 0 + + +class TestOdpiRejectsBitolOnlyOptions: + def test_out_dir_is_rejected(self, tmp_path: Path, capsys) -> None: + target = tmp_path / "d2" + args = _parse( + [ + "opds", + "export", + str(MULTI_EXPOSE_FLUID), + "--spec", + "odpi-4.1", + "--out-dir", + str(target), + ] + ) + assert cmd_opds_export(args, LOG) == 2 + captured = capsys.readouterr() + assert "--out-dir" in captured.out + captured.err + assert captured.out.strip() == "" or not captured.out.lstrip().startswith("{") + assert not target.exists() + + def test_yaml_format_is_rejected(self, tmp_path: Path, capsys) -> None: + out = tmp_path / "o.json" + args = _parse( + [ + "opds", + "export", + str(MULTI_EXPOSE_FLUID), + "--spec", + "odpi-4.1", + "--out", + str(out), + "-f", + "yaml", + ] + ) + assert cmd_opds_export(args, LOG) == 2 + captured = capsys.readouterr() + assert "--format" in captured.out + captured.err + assert not out.exists() + + def test_json_format_is_accepted(self, tmp_path: Path) -> None: + out = tmp_path / "o.json" + args = _parse( + [ + "opds", + "export", + str(MULTI_EXPOSE_FLUID), + "--spec", + "odpi-4.1", + "--out", + str(out), + "-f", + "json", + ] + ) + assert cmd_opds_export(args, LOG) == 0 + assert out.exists() + + def test_legacy_version_flag_still_exports(self, tmp_path: Path) -> None: + out = tmp_path / "legacy.json" + args = _parse( + ["opds", "export", str(MULTI_EXPOSE_FLUID), "--version", "4.1", "--out", str(out)] + ) + assert cmd_opds_export(args, LOG) == 0 + assert out.exists() + + def test_format_help_says_bitol_only(self) -> None: + from fluid_build.cli.opds import register + + parser = argparse.ArgumentParser(prog="fluid") + register(parser.add_subparsers(dest="cmd")) + odps = parser._subparsers._group_actions[0].choices["odps"] + export = odps._subparsers._group_actions[0].choices["export"] + helps = {a.dest: a.help for a in export._actions} + assert "Bitol specs only" in helps["format"] + assert "odpi-4.1" in helps["out_dir"] + + +class TestSuccessLinesAreNotMangled: + def test_import_success_line_keeps_brackets_and_is_unwrapped(self, tmp_path: Path) -> None: + out = tmp_path / ("[red]" + "x" * 90) / "c.fluid.yaml" + args = _parse(["opds", "import", str(HOURLY_ODCS), "-o", str(out)]) + with patch("fluid_build.cli.opds.cprint") as cprint: + assert cmd_opds_import(args, LOG) == 0 + assert out.exists() + lines = [c for c in cprint.call_args_list if str(out) in " ".join(map(str, c.args))] + assert lines, cprint.call_args_list + for call in lines: + assert call.kwargs.get("markup") is False + assert call.kwargs.get("soft_wrap") is True + + def test_import_success_line_rendered_by_rich(self, tmp_path: Path) -> None: + from rich.console import Console + + out = tmp_path / ("[red]" + "x" * 90) / "c.fluid.yaml" + args = _parse(["opds", "import", str(HOURLY_ODCS), "-o", str(out)]) + rich_console = Console(width=60, record=True, force_terminal=False) + with patch("fluid_build.cli.console.console", rich_console): + assert cmd_opds_import(args, LOG) == 0 + text = rich_console.export_text() + assert str(out) in text + + def test_odpi_success_line_keeps_brackets(self, tmp_path: Path) -> None: + out = tmp_path / ("[red]" + "y" * 90) / "o.json" + out.parent.mkdir(parents=True) + args = _parse( + ["opds", "export", str(MULTI_EXPOSE_FLUID), "--spec", "odpi-4.1", "--out", str(out)] + ) + with patch("fluid_build.cli.opds.cprint") as cprint: + assert cmd_opds_export(args, LOG) == 0 + lines = [c for c in cprint.call_args_list if str(out) in " ".join(map(str, c.args))] + assert lines, cprint.call_args_list + for call in lines: + assert call.kwargs.get("markup") is False + assert call.kwargs.get("soft_wrap") is True + + +# --------------------------------------------------------------------------- +# Defect 34 — the documented command sequences must work when run literally +# --------------------------------------------------------------------------- + +_PLACEHOLDERS = { + "": "commerce.orders-product", + "": "daily_orders", + "": "hourly_orders", + "": "daily_orders", +} + + +def _documented_commands(text: str) -> List[str]: + commands: List[str] = [] + for block in re.findall(r"```bash\n(.*?)```", text, flags=re.S): + for raw in block.splitlines(): + line = raw.split(" #", 1)[0].strip() if not raw.lstrip().startswith("#") else "" + if not line.startswith("fluid ") or line.startswith("fluid forge"): + continue + for placeholder, value in _PLACEHOLDERS.items(): + line = line.replace(placeholder, value) + commands.append(line) + return commands + + +def _run_documented(commands: List[str], cwd: Path) -> None: + contract = yaml.safe_load(MULTI_EXPOSE_FLUID.read_text(encoding="utf-8")) + # The ODPI v4.1 exporter copies metadata.status verbatim and the ODPI + # schema only accepts its own lifecycle values (draft, production, ...), + # so use one both specs accept for the documented ODPI export → validate. + contract["metadata"]["status"] = "draft" + _write_yaml(cwd / "contract.fluid.yaml", contract) + env = dict(os.environ) + env["PYTHONPATH"] = str(REPO) + os.pathsep + env.get("PYTHONPATH", "") + env.pop("ODPS_API_VERSION", None) + env.pop("ODCS_API_VERSION", None) + python = shlex.quote(sys.executable) + for command in commands: + shell_cmd = f"{python} -m fluid_build {command[len('fluid '):]}" + proc = subprocess.run( + shell_cmd, + shell=True, + cwd=str(cwd), + env=env, + capture_output=True, + text=True, + timeout=300, + ) + assert proc.returncode == 0, (command, proc.stdout[-2000:], proc.stderr[-2000:]) + + +class TestDocumentedSequencesRun: + def test_bitol_interop_doc_commands(self, tmp_path: Path) -> None: + text = BITOL_DOC.read_text(encoding="utf-8") + commands = _documented_commands(text.split("## `fluid forge --seed-from`", 1)[0]) + assert any(" import out/ " in f" {c} " for c in commands) + assert any("--spec odpi-4.1" in c for c in commands) + _run_documented(commands, tmp_path) + + def test_readme_bitol_commands(self, tmp_path: Path) -> None: + text = README.read_text(encoding="utf-8") + section = text.split("### Which ODPS? — disambiguation", 1)[1].split("\n---", 1)[0] + commands = _documented_commands(section) + assert commands + _run_documented(commands, tmp_path) + + +# --------------------------------------------------------------------------- +# Defect 22 — features.yaml provider keys must match the provider packages +# --------------------------------------------------------------------------- + + +class TestFeaturesYamlProviderKeys: + @staticmethod + def _features() -> Dict[str, Any]: + return yaml.safe_load((REPO / "fluid_build" / "features.yaml").read_text(encoding="utf-8")) + + def test_every_provider_key_is_a_package(self) -> None: + for key in self._features()["providers"]: + assert (REPO / "fluid_build" / "providers" / key).is_dir(), key + + def test_bitol_provider_is_listed_under_its_package_name(self) -> None: + providers = self._features()["providers"] + assert "odps_standard" in providers + assert "odps_bitol" in providers["odps_standard"]["note"] + + def test_every_manifest_provider_has_a_features_entry(self) -> None: + manifest = yaml.safe_load( + (REPO / "fluid_build" / "build-manifest.yaml").read_text(encoding="utf-8") + ) + providers = self._features()["providers"] + for profile, build in (manifest.get("builds") or {}).items(): + for name in build.get("providers") or []: + if (REPO / "fluid_build" / "providers" / name).is_dir(): + assert name in providers, (profile, name) + + def test_assess_provider_summary_lists_bitol_provider(self) -> None: + proc = subprocess.run( + [sys.executable, str(REPO / "scripts" / "assess_provider.py"), "--summary"], + cwd=str(REPO), + capture_output=True, + text=True, + timeout=300, + ) + assert proc.returncode == 0, proc.stderr + assert "odps_standard" in proc.stdout diff --git a/tests/test_review_fixes2_r2_odcs_mappers.py b/tests/test_review_fixes2_r2_odcs_mappers.py new file mode 100644 index 00000000..9ab7b83f --- /dev/null +++ b/tests/test_review_fixes2_r2_odcs_mappers.py @@ -0,0 +1,647 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +"""Regression tests for the round-2 ODCS mapper defects. + +Each test names the defect it pins. Every one of them failed on the branch +head the round-2 verification ran against (``385811f``). +""" + +from __future__ import annotations + +import contextlib +import copy +import io +import logging +from pathlib import Path +from typing import Any, Dict, List + +import pytest +import yaml + +from fluid_build.providers.odcs import OdcsProvider +from fluid_build.providers.odcs import validation as odcs_validation + +FIXTURES = Path(__file__).parent / "fixtures" +FULL_V320 = FIXTURES / "upstream" / "odcs" / "full-example.v3.2.0.odcs.yaml" +CONTRACT_FULL = FIXTURES / "odcs" / "contract-full.yaml" + + +def _load(path: Path) -> Dict[str, Any]: + with open(path) as f: + return yaml.safe_load(f) + + +def _cli(argv: List[str]) -> Any: + from fluid_build.cli import main as cli_main + + buf = io.StringIO() + try: + with contextlib.redirect_stdout(buf), contextlib.redirect_stderr(buf): + return cli_main(argv) + except SystemExit as exc: + return exc.code + + +def _v320_fixtures() -> List[Path]: + """Every upstream ODCS fixture declaring apiVersion v3.2.0.""" + found = [] + for path in sorted((FIXTURES / "upstream" / "odcs").glob("*.yaml")): + if str(_load(path).get("apiVersion")) == "v3.2.0": + found.append(path) + assert found, "no upstream v3.2.0 ODCS fixtures found" + return found + + +@pytest.fixture(autouse=True) +def _clean_env(monkeypatch): + for var in ( + "ODCS_API_VERSION", + "ODCS_INCLUDE_QUALITY", + "ODCS_INCLUDE_SLA", + "ODCS_VALIDATE", + "ODCS_VOWL_VALIDATE", + ): + monkeypatch.delenv(var, raising=False) + + +# --------------------------------------------------------------------------- +# Defects 11 + 28 — slaProperties survive splitting a multi-SchemaObject ODCS +# into per-expose exports, and qos edits on any expose are honoured. +# --------------------------------------------------------------------------- + + +def _two_tables() -> Dict[str, Any]: + return { + "apiVersion": "v3.1.0", + "kind": "DataContract", + "id": "two-tables", + "version": "1.0.0", + "status": "active", + "schema": [ + {"name": "orders", "properties": [{"name": "id", "logicalType": "string"}]}, + {"name": "customers", "properties": [{"name": "id", "logicalType": "string"}]}, + ], + "slaProperties": [ + {"property": "availability", "value": 0.999}, + {"property": "latency", "value": 4, "unit": "h"}, + {"property": "generalAvailability", "value": "2022-05-12T09:30:10-08:00"}, + ], + } + + +def test_defect11_every_scoped_expose_keeps_document_sla() -> None: + src = _two_tables() + prov = OdcsProvider() + fluid = prov.import_contract(src) + for eid in ("orders", "customers"): + out = prov.render(fluid, expose_id=eid) + assert out["slaProperties"] == src["slaProperties"], eid + odcs_validation.validate(out) + assert prov.render(fluid)["slaProperties"] == src["slaProperties"] + + +def test_defect11_upstream_v320_example_keeps_all_sla_per_expose() -> None: + src = _load(FULL_V320) + prov = OdcsProvider() + fluid = prov.import_contract(src) + assert len(fluid["exposes"]) == 2 + for expose in fluid["exposes"]: + out = prov.render(fluid, expose_id=expose["exposeId"]) + assert out["slaProperties"] == src["slaProperties"], expose["exposeId"] + + +def test_defect11_opds_cli_split_keeps_sla_for_every_port(tmp_path) -> None: + src_path = tmp_path / "two-tables.odcs.yaml" + src_path.write_text(yaml.safe_dump(_two_tables(), sort_keys=False)) + fluid_path = tmp_path / "two.fluid.yaml" + assert _cli(["opds", "import", str(src_path), "-o", str(fluid_path)]) in (0, None) + out_dir = tmp_path / "out" + rc = _cli(["opds", "export", str(fluid_path), "--out-dir", str(out_dir)]) + assert rc in (0, None) + odcs_files = sorted(out_dir.glob("*.odcs.yaml")) + assert len(odcs_files) == 2 + for path in odcs_files: + assert _load(path)["slaProperties"] == _two_tables()["slaProperties"], path.name + + +def test_defect11_dmm_per_expose_payloads_keep_sla() -> None: + from fluid_build.providers.datamesh_manager.datamesh_manager import ( + DataMeshManagerProvider, + ) + + fluid = OdcsProvider().import_contract(_two_tables()) + dmm = DataMeshManagerProvider(api_key="test-key", api_url="https://dmm.invalid") + previews = dmm._preview_odcs_per_expose(fluid, "two-tables") + assert len(previews) == 2 + for preview in previews: + assert preview["payload"]["slaProperties"] == _two_tables()["slaProperties"] + + +def test_defect11_scoped_edit_on_projected_expose_still_applies() -> None: + prov = OdcsProvider() + fluid = prov.import_contract(_two_tables()) + fluid["exposes"][0]["qos"]["availability"] = "95%" + out = prov.render(fluid, expose_id="orders") + assert out["slaProperties"][0] == {"property": "availability", "value": 0.95} + assert out["slaProperties"][1:] == _two_tables()["slaProperties"][1:] + + +def test_defect28_qos_edit_on_second_expose_is_honoured() -> None: + src = _two_tables() + src["slaProperties"] = [{"property": "availability", "value": 0.99}] + prov = OdcsProvider() + fluid = prov.import_contract(src) + assert "qos" not in fluid["exposes"][1] + fluid["exposes"][1]["qos"] = {"availability": "90%"} + assert prov.render(fluid)["slaProperties"] == [{"property": "availability", "value": 0.9}] + scoped = prov.render(fluid, expose_id="customers") + assert scoped["slaProperties"] == [{"property": "availability", "value": 0.9}] + # The untouched expose's own scoped contract keeps the source SLA. + assert prov.render(fluid, expose_id="orders")["slaProperties"] == src["slaProperties"] + + +def test_defect28_freshness_and_label_edits_on_second_expose() -> None: + prov = OdcsProvider() + fluid = prov.import_contract(_two_tables()) + fluid["exposes"][1]["qos"] = {"freshnessSLO": "PT1H", "labels": {"tier": "gold"}} + sla = prov.render(fluid)["slaProperties"] + assert {"property": "availability", "value": 0.999} in sla + latency = [p for p in sla if p["property"] == "latency"] + assert latency and latency[0]["value"] == "PT1H" + assert {"property": "label:tier", "value": "gold"} in sla + + +# --------------------------------------------------------------------------- +# Defects 13 + 26 — exporting an imported v3.2.0 contract at a lower +# apiVersion downgrades the v3.2.0-only content so the output validates. +# --------------------------------------------------------------------------- + + +def _every_v320_construct() -> Dict[str, Any]: + """A valid v3.2.0 contract using every v3.2.0-only construct FLUID stashes.""" + return { + "apiVersion": "v3.2.0", + "kind": "DataContract", + "id": "v320-only", + "version": "1.0.0", + "status": "active", + "context": "Use for revenue analytics only.", + "customProperties": [{"property": "costCenter", "value": "CC-1", "vendor": "acme"}], + "servers": [ + { + "server": "lake", + "type": "iceberg", + "catalog": "main", + "catalogUrl": "https://iceberg.example.com", + "namespace": "sales", + }, + { + "server": "pg", + "type": "postgres", + "host": "db", + "port": "${DB_PORT}", + "database": "d", + "schema": "s", + }, + { + "server": "pg2", + "type": "postgres", + "host": "db", + "port": "5432", + "database": "d", + "schema": "s", + }, + {"server": "files", "type": "s3", "location": "s3://b/p", "encoding": "utf-8"}, + { + "server": "ath", + "type": "athena", + "stagingDir": "s3://b/tmp", + "schema": "s", + "workgroup": "primary", + }, + ], + "schema": [ + { + "name": "orders", + "logicalType": "object", + "context": {"instructions": "One row per order."}, + "synonyms": [{"synonym": "sales_orders"}], + "deprecated": False, + "relationships": [ + {"id": "rel1", "type": "foreignKey", "from": "orders.cid", "to": "customers.id"} + ], + "properties": [ + { + "name": "status", + "logicalType": "string", + "enum": [{"value": "open"}, {"value": "closed"}], + "semanticType": "dimension", + "deprecated": True, + "customProperties": [{"property": "x", "value": 1, "vendor": "soda"}], + }, + { + "name": "attrs", + "logicalType": "map", + "physicalType": "MAP", + "map": { + "key": {"logicalType": "string"}, + "value": {"logicalType": "string"}, + }, + }, + { + "name": "embedding", + "logicalType": "vector", + "logicalTypeOptions": {"dimensions": 3, "elementType": "float32"}, + }, + { + "name": "lines", + "logicalType": "array", + "items": { + "name": "line", + "logicalType": "object", + "properties": [ + {"name": "sku", "logicalType": "string", "semanticType": "column"} + ], + }, + }, + ], + } + ], + "slaProperties": [ + { + "property": "availability", + "value": 0.99, + "customProperties": [{"property": "note", "value": "x"}], + "authoritativeDefinitions": [{"type": "sla", "url": "https://x.example.com"}], + } + ], + } + + +def test_defect26_fixture_is_valid_v320_but_not_v310() -> None: + doc = _every_v320_construct() + odcs_validation.validate(doc) + v310 = dict(doc, apiVersion="v3.1.0") + with pytest.raises(Exception): + odcs_validation.validate(v310) + + +@pytest.mark.parametrize("path", _v320_fixtures(), ids=lambda p: p.name) +def test_defect26_upstream_v320_fixtures_downgrade_to_v310(path, caplog) -> None: + fluid = OdcsProvider().import_contract(_load(path)) + with caplog.at_level(logging.WARNING): + out = OdcsProvider(api_version="v3.1.0").render(fluid) + assert out["apiVersion"] == "v3.1.0" + odcs_validation.validate(out) + assert "v3.2.0" in caplog.text and "v3.1.0" in caplog.text + # Per-port renders downgrade too. + for expose in fluid["exposes"]: + scoped = OdcsProvider(api_version="v3.1.0").render(fluid, expose_id=expose["exposeId"]) + odcs_validation.validate(scoped) + + +def test_defect26_map_becomes_object_with_shape_in_custom_properties() -> None: + fluid = OdcsProvider().import_contract(_load(FULL_V320)) + out = OdcsProvider(api_version="v3.1.0").render(fluid) + props = {p["name"]: p for p in out["schema"][1]["properties"]} + attrs = props["receiver_attributes"] + assert attrs["logicalType"] == "object" + assert "map" not in attrs + custom = {c["property"]: c["value"] for c in attrs["customProperties"]} + assert custom["map"]["key"]["logicalType"] == "string" + code = {p["name"]: p for p in out["schema"][0]["properties"]}["rcvr_cntry_code"] + assert "enum" not in code + assert [c for c in code["customProperties"] if c["property"] == "enum"] + + +def test_defect26_every_v320_construct_downgrades(caplog) -> None: + fluid = OdcsProvider().import_contract(_every_v320_construct()) + with caplog.at_level(logging.WARNING): + out = OdcsProvider(api_version="v3.1.0").render(fluid) + odcs_validation.validate(out) + assert "context" not in out + assert {"property": "context", "value": "Use for revenue analytics only."} in out[ + "customProperties" + ] + assert all("vendor" not in c for c in out["customProperties"]) + servers = {s["server"]: s for s in out["servers"]} + assert servers["lake"]["type"] == "custom" + assert servers["lake"]["catalog"] == "main" + lake_custom = {c["property"]: c["value"] for c in servers["lake"]["customProperties"]} + assert lake_custom["namespace"] == "sales" + assert lake_custom["serverType"] == "iceberg" + assert servers["pg2"]["type"] == "postgres" + assert servers["pg2"]["port"] == 5432 + # A ``${VAR}`` port cannot be a v3.1.0 Postgres port (required integer). + assert servers["pg"]["type"] == "custom" + assert servers["pg"]["host"] == "db" + assert "port" not in servers["pg"] + pg_custom = {c["property"]: c["value"] for c in servers["pg"]["customProperties"]} + assert pg_custom == {"serverType": "postgres", "port": "${DB_PORT}"} + assert "encoding" not in servers["files"] + assert "workgroup" not in servers["ath"] + obj = out["schema"][0] + for key in ("context", "synonyms", "deprecated"): + assert key not in obj + assert "id" not in obj["relationships"][0] + props = {p["name"]: p for p in obj["properties"]} + assert props["embedding"]["logicalType"] == "array" + assert "logicalTypeOptions" not in props["embedding"] + assert props["attrs"]["logicalType"] == "object" + for key in ("enum", "semanticType", "deprecated"): + assert key not in props["status"] + assert "semanticType" not in props["lines"]["items"]["properties"][0] + sla = out["slaProperties"][0] + assert sla == {"property": "availability", "value": 0.99} + assert "downgraded" in caplog.text + + +def test_defect13_env_pin_downgrades_and_validate_env_passes(monkeypatch) -> None: + fluid = OdcsProvider().import_contract(_load(FULL_V320)) + monkeypatch.setenv("ODCS_API_VERSION", "v3.1.0") + monkeypatch.setenv("ODCS_VALIDATE", "true") + out = OdcsProvider().render(fluid) + assert out["apiVersion"] == "v3.1.0" + odcs_validation.validate(out) + + +def test_defect13_cli_export_with_env_pin_validates(tmp_path, monkeypatch) -> None: + fluid_path = tmp_path / "full.fluid.yaml" + assert _cli(["odcs", "import", str(FULL_V320), "-o", str(fluid_path)]) in (0, None) + monkeypatch.setenv("ODCS_API_VERSION", "v3.1.0") + out_path = tmp_path / "out.yaml" + assert _cli(["odcs", "export", str(fluid_path), "-o", str(out_path)]) in (0, None) + assert _load(out_path)["apiVersion"] == "v3.1.0" + assert _cli(["odcs", "validate", str(out_path)]) in (0, None) + + +def test_defect26_default_reexport_of_v320_is_still_verbatim() -> None: + src = _every_v320_construct() + assert OdcsProvider().roundtrip_check(src)["equal"] + assert OdcsProvider().roundtrip_check(_load(FULL_V320))["equal"] + + +def test_defect26_v310_import_rendered_at_v310_is_untouched() -> None: + src = _load(CONTRACT_FULL) + fluid = OdcsProvider().import_contract(src) + out = OdcsProvider(api_version="v3.1.0").render(fluid) + assert odcs_validation.roundtrip_check(src, out)["equal"] + + +# --------------------------------------------------------------------------- +# Defect 14 — owner edits on a team without a name update members. +# --------------------------------------------------------------------------- + + +def _with_team(team: Any) -> Dict[str, Any]: + return { + "apiVersion": "v3.1.0", + "kind": "DataContract", + "id": "t", + "version": "1.0.0", + "status": "active", + "team": team, + } + + +_ARRAY_TEAM = [ + {"username": "ceastwood", "role": "Data Scientist", "dateIn": "2022-08-02"}, + {"username": "mhopper", "role": "Data Steward"}, +] + + +def test_defect14_array_team_email_edit_keeps_members() -> None: + prov = OdcsProvider() + fluid = prov.import_contract(_with_team(copy.deepcopy(_ARRAY_TEAM))) + fluid["owner"]["email"] = "clint@example.com" + out = prov.render(fluid) + assert out["team"] == [ + {"username": "clint@example.com", "role": "Data Scientist", "dateIn": "2022-08-02"}, + {"username": "mhopper", "role": "Data Steward"}, + ] + odcs_validation.validate(out) + + +def test_defect14_array_team_role_edit_and_new_contact() -> None: + prov = OdcsProvider() + fluid = prov.import_contract(_with_team(copy.deepcopy(_ARRAY_TEAM))) + fluid["owner"]["role"] = "Owner" + fluid["owner"]["contacts"].append({"email": "new@example.com"}) + out = prov.render(fluid) + assert out["team"] == [ + {"username": "ceastwood", "role": "Owner", "dateIn": "2022-08-02"}, + {"username": "mhopper", "role": "Data Steward"}, + {"username": "new@example.com"}, + ] + + +def test_defect14_object_team_without_name_edit_keeps_team() -> None: + team = {"description": "no name", "members": copy.deepcopy(_ARRAY_TEAM)} + prov = OdcsProvider() + fluid = prov.import_contract(_with_team(team)) + fluid["owner"]["email"] = "clint@example.com" + out = prov.render(fluid) + assert "name" not in out["team"] + assert out["team"]["description"] == "no name" + assert out["team"]["members"][0] == { + "username": "clint@example.com", + "role": "Data Scientist", + "dateIn": "2022-08-02", + } + odcs_validation.validate(out) + + +def test_defect14_setting_team_name_on_nameless_team() -> None: + prov = OdcsProvider() + fluid = prov.import_contract(_with_team(copy.deepcopy(_ARRAY_TEAM))) + fluid["owner"]["team"] = "data-science" + out = prov.render(fluid) + assert out["team"]["name"] == "data-science" + assert out["team"]["members"][0]["dateIn"] == "2022-08-02" + + +def test_defect14_owner_removed_still_drops_team() -> None: + prov = OdcsProvider() + fluid = prov.import_contract(_with_team(copy.deepcopy(_ARRAY_TEAM))) + del fluid["owner"] + assert "team" not in prov.render(fluid) + + +# --------------------------------------------------------------------------- +# Defect 15 — a stashed server stays while an expose binding selects it. +# --------------------------------------------------------------------------- + + +def test_defect15_removing_expects_keeps_bound_server() -> None: + src = _load(CONTRACT_FULL) + prov = OdcsProvider() + fluid = prov.import_contract(src) + fluid["expects"] = [] + out = prov.render(fluid) + assert out["servers"] == src["servers"] + assert odcs_validation.roundtrip_check(src, out)["equal"] + + +def test_defect15_removing_expects_then_editing_binding_keeps_server_identity() -> None: + prov = OdcsProvider() + fluid = prov.import_contract(_load(CONTRACT_FULL)) + fluid["expects"] = [] + binding = fluid["exposes"][0]["binding"] + binding.pop("physical_name", None) + binding["format"] = "parquet" + binding["location"]["dataset"] = "orders_gold" + out = prov.render(fluid) + assert len(out["servers"]) == 1 + server = out["servers"][0] + assert server["server"] == "bq-prod" + assert server["type"] == "bigquery" + assert server["project"] == "acme-prod" + assert server["dataset"] == "orders_gold" + odcs_validation.validate(out) + + +def test_defect15_cli_round_trip_without_expects(tmp_path) -> None: + fluid_path = tmp_path / "cf.fluid.yaml" + assert _cli(["odcs", "import", str(CONTRACT_FULL), "-o", str(fluid_path)]) in (0, None) + fluid = _load(fluid_path) + del fluid["expects"] + fluid_path.write_text(yaml.safe_dump(fluid, sort_keys=False)) + out_path = tmp_path / "out.yaml" + assert _cli(["odcs", "export", str(fluid_path), "-o", str(out_path)]) in (0, None) + assert _load(out_path)["servers"] == _load(CONTRACT_FULL)["servers"] + + +# --------------------------------------------------------------------------- +# Defect 25 — --no-quality suppresses object-level and nested quality too. +# --------------------------------------------------------------------------- + + +def _quality_paths(node: Any, path: str = "") -> List[str]: + found: List[str] = [] + if isinstance(node, dict): + for key, value in node.items(): + if key == "quality": + found.append(f"{path}.quality") + found.extend(_quality_paths(value, f"{path}.{key}")) + elif isinstance(node, list): + for i, value in enumerate(node): + found.extend(_quality_paths(value, f"{path}[{i}]")) + return found + + +def test_defect25_no_quality_env_suppresses_object_quality(monkeypatch) -> None: + fluid = OdcsProvider().import_contract(_load(CONTRACT_FULL)) + monkeypatch.setenv("ODCS_INCLUDE_QUALITY", "false") + out = OdcsProvider().render(fluid) + assert _quality_paths(out) == [] + odcs_validation.validate(out) + + +def test_defect25_no_quality_on_upstream_v320_and_nested_properties() -> None: + src = _load(FULL_V320) + src["schema"][1]["properties"].append( + { + "name": "nested", + "logicalType": "object", + "properties": [ + { + "name": "inner", + "logicalType": "string", + "quality": [{"type": "library", "metric": "nullValues", "mustBe": 0}], + } + ], + } + ) + fluid = OdcsProvider().import_contract(src) + prov = OdcsProvider() + prov.include_quality_checks = False + out = prov.render(fluid) + assert _quality_paths(out) == [] + # With quality on, every level is replayed. + assert len(_quality_paths(OdcsProvider().render(fluid))) == 3 + + +def test_defect25_cli_no_quality(tmp_path) -> None: + fluid_path = tmp_path / "cf.fluid.yaml" + assert _cli(["odcs", "import", str(CONTRACT_FULL), "-o", str(fluid_path)]) in (0, None) + out_path = tmp_path / "out.yaml" + rc = _cli(["odcs", "export", str(fluid_path), "--no-quality", "-o", str(out_path)]) + assert rc in (0, None) + assert _quality_paths(_load(out_path)) == [] + + +# --------------------------------------------------------------------------- +# Defect 27 — FLUID → ODCS status mapping is the 0.7.8 one (case-sensitive). +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "status, expected", + [ + ("development", "draft"), + ("Development", "active"), + ("draft", "draft"), + ("deprecated", "deprecated"), + ("retired", "retired"), + ("whatever", "active"), + ], +) +def test_defect27_status_mapping_matches_078(status, expected) -> None: + assert OdcsProvider()._map_status_to_odcs(status) == expected + fluid = { + "id": "acme.orders", + "metadata": {"version": "1.0.0", "status": status}, + "exposes": [{"exposeId": "o", "contract": {"schema": [{"name": "a", "type": "string"}]}}], + } + assert OdcsProvider().render(fluid)["status"] == expected + + +def test_defect27_snowflake_smoke_example_exports_active() -> None: + from fluid_build.cli.bootstrap import load_contract_with_overlay + + paths = sorted((Path(__file__).resolve().parent.parent / "examples" / "snowflake").rglob("*")) + smoke = [ + p + for p in paths + if p.is_file() + and p.suffix in (".yaml", ".yml") + and "smoke" in str(p) + and "Development" in p.read_text() + ] + if not smoke: + pytest.skip("snowflake smoke example not present") + fluid = load_contract_with_overlay(str(smoke[0]), None, logging.getLogger(__name__)) + assert OdcsProvider().render(fluid)["status"] == "active" + + +# --------------------------------------------------------------------------- +# Defect 29 — a contract imported without ``schema`` gains no ``schema: []``. +# --------------------------------------------------------------------------- + + +def test_defect29_no_schema_round_trip_is_lossless() -> None: + src = _with_team({"name": "x"}) + result = OdcsProvider().roundtrip_check(src) + assert result["equal"], result + + +def test_defect29_schema_added_after_import_is_emitted() -> None: + prov = OdcsProvider() + fluid = prov.import_contract(_with_team({"name": "x"})) + fluid["exposes"].append( + {"exposeId": "orders", "contract": {"schema": [{"name": "id", "type": "string"}]}} + ) + out = prov.render(fluid) + assert [o["name"] for o in out["schema"]] == ["orders"] + odcs_validation.validate(out) + + +def test_defect29_explicit_empty_schema_still_round_trips() -> None: + src = _with_team({"name": "x"}) + src["schema"] = [] + assert OdcsProvider().roundtrip_check(src)["equal"] diff --git a/tests/test_review_fixes2_r2_odps.py b/tests/test_review_fixes2_r2_odps.py new file mode 100644 index 00000000..b311c35d --- /dev/null +++ b/tests/test_review_fixes2_r2_odps.py @@ -0,0 +1,668 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Regression tests for the round-2 ODPS / ODCS review findings. + +Covers schema-less per-port ODCS documents in a bundle, ODCS-only +directory imports, YAML anchors in imported FLUID, FLUID owner / expects +edits reaching the ODPS team and the per-port ODCS teams / servers, the +nested < v1.1.0 downgrade, the legacy contractId pin, the remote-fetch total +deadline, malformed contractIds and YAML alias bombs. +""" + +from __future__ import annotations + +import contextlib +import copy +import datetime +import http.server +import logging +import shutil +import threading +import time +import tracemalloc +from pathlib import Path +from typing import Any, Dict, Iterator, List + +import pytest +import yaml + +from fluid_build.providers.base import ProviderError +from fluid_build.providers.odcs import OdcsProvider +from fluid_build.providers.odcs.io import load_yaml, read_input +from fluid_build.providers.odcs.validation import json_instance +from fluid_build.providers.odps_standard import BitolOdpsProvider, OdpsStandardProvider +from fluid_build.providers.odps_standard.resolver import ContractNotFound, ContractResolver +from fluid_build.providers.odps_standard.validation import load_schema +from fluid_build.providers.odps_standard.validation import validate as validate_odps + +FIXTURES = Path(__file__).parent / "fixtures" +BUNDLE_DIR = FIXTURES / "odps" / "product-bitol" +CONTRACTS_ONLY = FIXTURES / "odps" / "contracts-only" +UPSTREAM_ODPS = sorted((FIXTURES / "upstream" / "odps").glob("*.odps.yaml")) +PRODUCT_FILE = BUNDLE_DIR / "commerce.orders-product.odps.yaml" +DAILY = "commerce.orders-product.daily_orders" +HOURLY = "commerce.orders-product.hourly_orders" + + +def _load(path: Path) -> Any: + with open(path) as f: + return yaml.safe_load(f) + + +def _write(path: Path, doc: Any) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(yaml.safe_dump(doc, sort_keys=False)) + return path + + +def _via_yaml(fluid: Dict[str, Any]) -> Dict[str, Any]: + """Mimic `fluid opds import -o f.yaml` + `fluid opds export f.yaml`.""" + return yaml.safe_load(yaml.safe_dump(fluid, sort_keys=False)) + + +def _strict_provider(api_version: Any = None) -> BitolOdpsProvider: + prov = BitolOdpsProvider(api_version=api_version) + prov.strict_validation = True + return prov + + +@contextlib.contextmanager +def _captured(logger: logging.Logger) -> Iterator[List[str]]: + """Warnings logged on ``logger`` itself (independent of propagation, + which other tests' logging setup may turn off).""" + messages: List[str] = [] + + class _Handler(logging.Handler): + def emit(self, record: logging.LogRecord) -> None: + messages.append(record.getMessage()) + + handler = _Handler(level=logging.WARNING) + old_level, old_disabled = logger.level, logger.disabled + logger.addHandler(handler) + logger.setLevel(logging.WARNING) + logger.disabled = False + try: + yield messages + finally: + logger.removeHandler(handler) + logger.setLevel(old_level) + logger.disabled = old_disabled + + +def _server(odcs: Dict[str, Any], name: str) -> Dict[str, Any]: + return next(s for s in odcs["servers"] if s.get("server") == name) + + +# --------------------------------------------------------------------------- +# Defect 3: a schema-less per-port ODCS keeps its document on bundle import +# --------------------------------------------------------------------------- + + +def _schema_less_odcs(api_version: str) -> Dict[str, Any]: + return { + "apiVersion": api_version, + "kind": "DataContract", + "id": "c-orders", + "version": "2.0.0", + "status": "active", + "name": "Orders contract", + "description": {"purpose": "Orders"}, + "domain": "sales", + "dataProduct": "p1", + "servers": [ + { + "server": "prod", + "type": "postgres", + "host": "db.example.com", + "port": 5432, + "database": "shop", + "schema": "public", + } + ], + "slaProperties": [{"property": "latency", "value": 4, "unit": "d"}], + } + + +def _schema_less_bundle(root: Path, api_version: str) -> Path: + _write( + root / "p.odps.yaml", + { + "apiVersion": "v1.1.0", + "kind": "DataProduct", + "id": "p1", + "name": "P1", + "status": "active", + "outputPorts": [{"name": "orders", "version": "1.0.0", "contractId": "c-orders"}], + }, + ) + _write(root / "c-orders.odcs.yaml", _schema_less_odcs(api_version)) + return root + + +class TestSchemaLessPortContract: + @pytest.mark.parametrize("api_version", ["v3.1.0", "v3.2.0"]) + def test_document_level_fields_round_trip(self, tmp_path: Path, api_version: str) -> None: + prov = _strict_provider() + bundle = prov.render( + _via_yaml(prov.import_directory(_schema_less_bundle(tmp_path, api_version))) + ) + odcs = bundle["contracts"]["c-orders"] + source = _schema_less_odcs(api_version) + # A contract imported without ``schema`` re-exports without one. + assert "schema" not in odcs + assert odcs == source + + def test_snapshot_is_kept_on_the_expose(self, tmp_path: Path) -> None: + fluid = BitolOdpsProvider().import_directory(_schema_less_bundle(tmp_path, "v3.1.0")) + doc = fluid["exposes"][0]["odcs_passthrough"]["_document"] + assert doc["metadata"]["odcs_passthrough"]["_verbatim"]["version"] == "2.0.0" + + +# --------------------------------------------------------------------------- +# Defect 12: ODCS-only directory import keeps each contract's document +# --------------------------------------------------------------------------- + + +class TestOdcsOnlyDirectory: + def test_single_file_directory_round_trips(self, tmp_path: Path) -> None: + src = FIXTURES / "odcs" / "contract-full.yaml" + root = tmp_path / "contracts" + root.mkdir() + shutil.copy(src, root / "orders.odcs.yaml") + source = _load(src) + + prov = BitolOdpsProvider() + prov.strict_validation = False + fluid = _via_yaml(prov.import_directory(root)) + bundle = prov.render(fluid) + assert list(bundle["contracts"]) == [source["id"]] + odcs = bundle["contracts"][source["id"]] + for key in ("id", "name", "version", "description", "domain", "team", "servers"): + assert odcs.get(key) == source.get(key), key + assert bundle["product"]["outputPorts"][0]["contractId"] == source["id"] + + def test_fixture_directory_round_trips_every_contract(self) -> None: + prov = _strict_provider() + bundle = prov.render(_via_yaml(prov.import_directory(CONTRACTS_ONLY))) + for path in sorted(CONTRACTS_ONLY.glob("*.odcs.yaml")): + source = _load(path) + assert bundle["contracts"][source["id"]] == source, path.name + + def test_same_file_and_directory_import_agree(self, tmp_path: Path) -> None: + src = CONTRACTS_ONLY / "hourly_orders.odcs.yaml" + root = tmp_path / "one" + root.mkdir() + shutil.copy(src, root / src.name) + prov = _strict_provider() + odcs = prov.render(prov.import_directory(root))["contracts"][_load(src)["id"]] + assert odcs == _load(src) + + +# --------------------------------------------------------------------------- +# Defect 16: imported FLUID has no YAML anchors +# --------------------------------------------------------------------------- + + +class TestNoYamlAnchors: + @pytest.mark.parametrize("entry", ["directory", "file"]) + def test_imported_bundle_dumps_without_aliases(self, entry: str) -> None: + prov = BitolOdpsProvider() + if entry == "directory": + fluid = prov.import_directory(BUNDLE_DIR) + else: + fluid = prov.import_contract(PRODUCT_FILE) + text = yaml.safe_dump(fluid, sort_keys=False) + assert "&id" not in text and "*id" not in text + + def test_upstream_products_dump_without_aliases(self) -> None: + prov = BitolOdpsProvider() + for path in UPSTREAM_ODPS: + fluid = prov.import_contract(path, allow_remote=False, lenient=True) + text = yaml.safe_dump(fluid, sort_keys=False) + assert "&id" not in text and "*id" not in text, path.name + + def test_render_shares_nothing_with_the_fluid(self) -> None: + prov = BitolOdpsProvider() + fluid = prov.import_contract(PRODUCT_FILE) + product = prov.render(fluid)["product"] + product["team"]["members"].append({"username": "x"}) + assert len(fluid["metadata"]["odps_passthrough"]["team"]["members"]) == 1 + + +# --------------------------------------------------------------------------- +# Defects 17 / 20: owner and expects edits reach the ODPS team and the +# per-port ODCS teams / servers +# --------------------------------------------------------------------------- + + +def _imported_bundle() -> Dict[str, Any]: + return _via_yaml(_strict_provider().import_directory(BUNDLE_DIR)) + + +class TestProductEditsReachEveryDocument: + def test_unedited_bundle_still_round_trips_byte_identically(self, tmp_path: Path) -> None: + prov = _strict_provider() + out = tmp_path / "out" + prov.render(_imported_bundle(), out_dir=out) + for src in BUNDLE_DIR.iterdir(): + assert (out / src.name).read_bytes() == src.read_bytes(), src.name + + def test_owner_edit_reaches_product_and_port_teams(self) -> None: + fluid = _imported_bundle() + fluid["owner"]["team"] = "NEW-TEAM" + fluid["owner"]["email"] = "carol@acme.com" + bundle = _strict_provider().render(fluid) + + team = bundle["product"]["team"] + assert team["name"] == "NEW-TEAM" + assert team["members"][0]["username"] == "carol@acme.com" + assert team["members"][0]["name"] == "Alice" # unedited keys survive + for cid in (DAILY, HOURLY): + odcs_team = bundle["contracts"][cid]["team"] + assert odcs_team["name"] == "NEW-TEAM", cid + assert odcs_team["members"][0]["username"] == "carol@acme.com", cid + assert odcs_team["members"][0]["role"] == "data-owner", cid + + def test_owner_edit_leaves_everything_else_alone(self) -> None: + fluid = _imported_bundle() + fluid["owner"]["team"] = "NEW-TEAM" + bundle = _strict_provider().render(fluid) + product = dict(bundle["product"]) + product.pop("team") + expected = _load(PRODUCT_FILE) + expected.pop("team") + assert product == expected + for cid in (DAILY, HOURLY): + odcs = dict(bundle["contracts"][cid]) + source = _load(BUNDLE_DIR / f"{cid}.odcs.yaml") + odcs.pop("team"), source.pop("team") + assert odcs == source, cid + + def test_expects_edit_reaches_every_port_server(self) -> None: + fluid = _imported_bundle() + customers = next(e for e in fluid["expects"] if e["id"] == "customers_raw") + customers["location"]["database"] = "RAW_DB2" + bundle = _strict_provider().render(fluid) + assert bundle["product"] == _load(PRODUCT_FILE) + for cid in (DAILY, HOURLY): + odcs = bundle["contracts"][cid] + source = _load(BUNDLE_DIR / f"{cid}.odcs.yaml") + assert _server(odcs, "customers_raw")["database"] == "RAW_DB2", cid + expected = copy.deepcopy(source) + _server(expected, "customers_raw")["database"] = "RAW_DB2" + assert odcs == expected, cid + + def test_removed_expect_drops_its_port_servers(self) -> None: + fluid = _imported_bundle() + fluid["expects"] = [e for e in fluid["expects"] if e["id"] != "customers_raw"] + bundle = _strict_provider().render(fluid) + assert "inputPorts" not in bundle["product"] + for cid in (DAILY, HOURLY): + names = [s["server"] for s in bundle["contracts"][cid]["servers"]] + assert "customers_raw" not in names, cid + assert len(names) == 1, cid + + def test_owner_edit_on_a_single_file_import(self) -> None: + prov = _strict_provider() + fluid = _via_yaml(prov.import_contract(PRODUCT_FILE)) + fluid["owner"]["team"] = "NEW-TEAM" + bundle = prov.render(fluid) + assert bundle["product"]["team"]["name"] == "NEW-TEAM" + assert bundle["contracts"][DAILY]["team"]["name"] == "NEW-TEAM" + + +# --------------------------------------------------------------------------- +# Defect 19: every upstream v1.1.0 product exports valid at v1.0.0 +# --------------------------------------------------------------------------- + + +class TestNestedDowngradeToV100: + @pytest.mark.parametrize("path", UPSTREAM_ODPS, ids=lambda p: p.name) + def test_upstream_v110_product_exports_valid_at_v100(self, path: Path) -> None: + fluid = _via_yaml( + BitolOdpsProvider().import_contract(path, allow_remote=False, lenient=True) + ) + prov = BitolOdpsProvider(api_version="v1.0.0") # what --spec bitol-1.0.0 selects + prov.strict_validation = False # their per-port contracts are not vendored + product = prov.render(fluid)["product"] + assert product["apiVersion"] == "v1.0.0" + validate_odps(product) # judged by the v1.0.0 schema it declares + validate_odps(product, load_schema("v1.0.0")) + + def test_nested_keys_move_to_custom_properties_or_are_dropped(self) -> None: + path = FIXTURES / "upstream" / "odps" / "deprecated-data-product.v1.1.0.odps.yaml" + fluid = BitolOdpsProvider().import_contract(path, allow_remote=False, lenient=True) + prov = BitolOdpsProvider(api_version="v1.0.0") + prov.strict_validation = False + with _captured(prov.logger) as logs: + product = prov.render(fluid)["product"] + source = _load(path) + mgmt_src = source["managementPorts"][0] + mgmt = product["managementPorts"][0] + assert "deprecated" not in mgmt + custom = {c["property"]: c["value"] for c in mgmt.get("customProperties") or []} + assert custom["deprecated"] == mgmt_src["deprecated"] + assert "managementPorts[0]" in "\n".join(logs) + + def test_sbom_extras_are_dropped_with_a_warning(self) -> None: + path = FIXTURES / "upstream" / "odps" / "customer-data-product.v1.1.0.odps.yaml" + fluid = BitolOdpsProvider().import_contract(path, allow_remote=False, lenient=True) + prov = BitolOdpsProvider(api_version="v1.0.0") + prov.strict_validation = False + with _captured(prov.logger) as logs: + product = prov.render(fluid)["product"] + for port in product["outputPorts"]: + for sbom in port.get("sbom") or []: + assert set(sbom) <= {"type", "url"} + assert "sbom[0]" in "\n".join(logs) + + def test_v110_export_of_the_same_product_is_untouched(self) -> None: + path = FIXTURES / "upstream" / "odps" / "customer-data-product.v1.1.0.odps.yaml" + fluid = BitolOdpsProvider().import_contract(path, allow_remote=False, lenient=True) + prov = BitolOdpsProvider() + prov.strict_validation = False + assert prov.render(fluid)["product"] == _load(path) + + def test_cli_strict_export_at_bitol_1_0_0_succeeds(self, tmp_path: Path) -> None: + from fluid_build.cli import main + + src = FIXTURES / "upstream" / "odps" / "customer-data-product.v1.1.0.odps.yaml" + imported = tmp_path / "up.fluid.yaml" + rc = main(["opds", "import", str(src), "--no-remote", "--lenient", "-o", str(imported)]) + assert rc in (0, None) + out = tmp_path / "o.odps.yaml" + rc = main(["opds", "export", str(imported), "--spec", "bitol-1.0.0", "--out", str(out)]) + assert rc in (0, None) + validate_odps(_load(out), load_schema("v1.0.0")) + + +# --------------------------------------------------------------------------- +# Defect 5 (documented): expose.contract_id pins the 0.7.8 legacy id +# --------------------------------------------------------------------------- + + +def test_legacy_contract_id_can_be_pinned() -> None: + fluid = { + "id": "data-product", + "metadata": {"name": "Data product", "version": "1.0.0"}, + "exposes": [{"id": "orders", "contract_id": "orders_contract"}], + } + product = OdpsStandardProvider().render(fluid) + assert product["outputPorts"][0]["contractId"] == "orders_contract" + + +# --------------------------------------------------------------------------- +# Defect 30: the total deadline holds while headers / chunk sizes drip +# --------------------------------------------------------------------------- + + +class _DripHandler(http.server.BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def log_message(self, *args: Any) -> None: # silence test output + pass + + def do_GET(self) -> None: # noqa: N802 + try: + if self.path == "/slowhdr": + self.wfile.write(b"HTTP/1.1 200 OK\r\n") + for i in range(40): + self.wfile.write(f"X-Drip-{i}: 1\r\n".encode()) + self.wfile.flush() + time.sleep(0.1) + self.wfile.write(b"Content-Length: 0\r\n\r\n") + elif self.path == "/slowchunk": + self.wfile.write( + b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n" + b"Transfer-Encoding: chunked\r\n\r\n" + ) + self.wfile.flush() + for _ in range(40): + self.wfile.write(b"0") # a never-ending chunk-size line + self.wfile.flush() + time.sleep(0.1) + elif self.path == "/ok": + body = b'{"apiVersion": "v3.1.0", "kind": "DataContract", "id": "r", ' + body += b'"version": "1.0.0", "status": "active", "schema": []}' + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + elif self.path == "/chunked": + body = b'{"apiVersion": "v3.1.0", "kind": "DataContract", "id": "r", ' + body += b'"version": "1.0.0", "status": "active", "schema": []}' + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Transfer-Encoding", "chunked") + self.end_headers() + for part in (body[:20], body[20:]): + self.wfile.write(f"{len(part):x}\r\n".encode() + part + b"\r\n") + self.wfile.write(b"0\r\n\r\n") + elif self.path == "/redirect": + self.send_response(302) + self.send_header("Location", "/ok") + self.send_header("Content-Length", "0") + self.end_headers() + except OSError: + pass + + +@pytest.fixture +def drip_base(monkeypatch) -> Iterator[str]: + for var in ("http_proxy", "HTTP_PROXY", "https_proxy", "HTTPS_PROXY"): + monkeypatch.delenv(var, raising=False) + server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), _DripHandler) + server.daemon_threads = True + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield f"http://127.0.0.1:{server.server_address[1]}" + finally: + server.shutdown() + server.server_close() + + +class TestRemoteDeadline: + @pytest.mark.parametrize("path", ["/slowhdr", "/slowchunk"]) + def test_deadline_holds_while_headers_or_chunk_sizes_drip( + self, drip_base: str, path: str + ) -> None: + resolver = ContractResolver(allow_remote=True, timeout=2, total_timeout=0.5) + started = time.monotonic() + with pytest.raises(ContractNotFound): + resolver.resolve(f"{drip_base}{path}") + assert time.monotonic() - started < 1.5 + + @pytest.mark.parametrize("path", ["/ok", "/chunked", "/redirect"]) + def test_normal_responses_still_resolve(self, drip_base: str, path: str) -> None: + resolved = ContractResolver(allow_remote=True).resolve(f"{drip_base}{path}") + assert resolved.odcs["id"] == "r" + + +# --------------------------------------------------------------------------- +# Defect 31: malformed contractIds are warnings when lenient +# --------------------------------------------------------------------------- + + +def _nul_bundle(root: Path, *, on: str) -> Path: + product = _load(PRODUCT_FILE) + bad = "commerce.orders-product.input.cust\x00omers" + if on == "input": + product["inputPorts"][0]["contractId"] = bad + else: + product["outputPorts"][1]["contractId"] = bad + _write(root / "commerce.orders-product.odps.yaml", product) + for src in BUNDLE_DIR.glob("*.odcs.yaml"): + shutil.copy(src, root / src.name) + return root + + +class TestMalformedContractIds: + def test_input_port_nul_is_a_warning(self, tmp_path: Path) -> None: + prov = BitolOdpsProvider() + with _captured(prov.logger) as logs: + fluid = prov.import_directory(_nul_bundle(tmp_path, on="input")) + assert [e["id"] for e in fluid["expects"]] == ["customers_raw"] + assert "Skipping unresolved contract" in "\n".join(logs) + + def test_output_port_nul_is_a_warning_when_lenient(self, tmp_path: Path) -> None: + fluid = BitolOdpsProvider().import_directory( + _nul_bundle(tmp_path, on="output"), lenient=True + ) + assert len(fluid["exposes"]) == 2 + + def test_output_port_nul_is_a_provider_error_when_strict(self, tmp_path: Path) -> None: + with pytest.raises(ProviderError): + BitolOdpsProvider().import_directory(_nul_bundle(tmp_path, on="output")) + + @pytest.mark.parametrize("bad", ["a\x00b", 42, None, ["x"], "http://["]) + def test_resolver_never_crashes(self, tmp_path: Path, bad: Any) -> None: + with pytest.raises(ContractNotFound): + ContractResolver(base_path=tmp_path, allow_remote=False).resolve(bad) + + +# --------------------------------------------------------------------------- +# Defect 32: YAML alias bombs are refused, fast and small +# --------------------------------------------------------------------------- + + +def _alias_bomb(depth: int = 8, width: int = 9) -> str: + lines = [ + "apiVersion: v3.1.0", + "kind: DataContract", + "id: shop.orders", + "version: 1.0.0", + "status: active", + "customProperties:", + "- property: bomb", + " value:", + " l0: &l0 [" + ",".join(["lol"] * width) + "]", + ] + for i in range(1, depth + 1): + lines.append(f" l{i}: &l{i} [" + ",".join([f"*l{i - 1}"] * width) + "]") + lines.append("schema: []") + return "\n".join(lines) + "\n" + + +def _fails_fast_and_small(fn: Any) -> None: + tracemalloc.start() + started = time.monotonic() + try: + with pytest.raises(ProviderError): + fn() + elapsed = time.monotonic() - started + _, peak = tracemalloc.get_traced_memory() + finally: + tracemalloc.stop() + assert elapsed < 5 + assert peak < 64 * 1024 * 1024 + + +class TestAliasBombs: + def test_bomb_is_small(self) -> None: + assert len(_alias_bomb().encode()) < 700 + + def test_read_input_refuses_it(self, tmp_path: Path) -> None: + path = tmp_path / "shop.orders.odcs.yaml" + path.write_text(_alias_bomb()) + _fails_fast_and_small(lambda: read_input(path)) + + def test_odcs_import_refuses_it(self, tmp_path: Path) -> None: + path = tmp_path / "shop.orders.odcs.yaml" + path.write_text(_alias_bomb()) + _fails_fast_and_small(lambda: OdcsProvider().import_contract(path, lenient=True)) + + def test_bundle_import_refuses_it(self, tmp_path: Path) -> None: + root = tmp_path / "bombb" + _write( + root / "shop.odps.yaml", + { + "apiVersion": "v1.1.0", + "kind": "DataProduct", + "id": "shop", + "status": "active", + "outputPorts": [{"name": "orders", "contractId": "shop.orders"}], + }, + ) + (root / "shop.orders.odcs.yaml").write_text(_alias_bomb()) + _fails_fast_and_small(lambda: BitolOdpsProvider().import_directory(root)) + # Lenient: the port is skipped with a warning, still fast. + started = time.monotonic() + fluid = BitolOdpsProvider().import_directory(root, lenient=True) + assert time.monotonic() - started < 5 + assert "contract" not in fluid["exposes"][0] + + def test_remote_bomb_is_refused(self, monkeypatch) -> None: + from fluid_build.providers.odps_standard import resolver as resolver_mod + + body = _alias_bomb().encode() + + class _Resp: + headers = {"Content-Type": "application/yaml"} + + def __init__(self) -> None: + self._body = body + + def read(self, amt: int = -1) -> bytes: + out, self._body = self._body, b"" + return out + + def __enter__(self): + return self + + def __exit__(self, *_: Any) -> bool: + return False + + monkeypatch.setattr(resolver_mod, "urlopen", lambda url, **kw: _Resp()) + _fails_fast_and_small( + lambda: ContractResolver(allow_remote=True).resolve("https://example.invalid/c.yaml") + ) + + def test_recursive_alias_is_refused(self) -> None: + with pytest.raises(ProviderError): + load_yaml("a: &a [*a]\n") + + def test_modest_aliases_still_load(self) -> None: + doc = load_yaml("base: &b {x: 1, y: [1, 2]}\nother: *b\nmerged: {<<: *b, z: 3}\n") + assert doc["other"] == {"x": 1, "y": [1, 2]} + assert doc["merged"] == {"x": 1, "y": [1, 2], "z": 3} + + def test_large_alias_free_document_still_loads(self) -> None: + text = "items:\n" + "".join(f"- {{a: {i}, b: [x, y]}}\n" for i in range(50_000)) + assert len(load_yaml(text)["items"]) == 50_000 + + def test_json_instance_preserves_shared_references(self) -> None: + shared = [{"when": datetime.date(2024, 1, 1)}] + doc = {"a": shared, "b": shared, "c": (shared,)} + out = json_instance(doc) + assert out["a"] is out["b"] + assert out["c"][0] is out["a"] + assert out["a"][0]["when"] == "2024-01-01" + assert doc["a"][0]["when"] != "2024-01-01" # input untouched + + def test_json_instance_stays_small_on_a_shared_tree(self) -> None: + node: Any = ["lol"] * 9 + for _ in range(8): + node = [node] * 9 + tracemalloc.start() + started = time.monotonic() + try: + json_instance({"bomb": node}) + _, peak = tracemalloc.get_traced_memory() + finally: + tracemalloc.stop() + assert time.monotonic() - started < 2 + assert peak < 8 * 1024 * 1024 diff --git a/tests/test_review_fixes3_convergence.py b/tests/test_review_fixes3_convergence.py new file mode 100644 index 00000000..503bec0a --- /dev/null +++ b/tests/test_review_fixes3_convergence.py @@ -0,0 +1,144 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Regression tests for the defects the round-2 convergence check found.""" + +from __future__ import annotations + +import time +import warnings +from typing import Any, Dict, List + +import pytest + +from fluid_build.providers.catalogs import CatalogTarget +from fluid_build.providers.catalogs.fluid_cc import FluidCommandCenterProvider +from fluid_build.providers.odcs.provider import _merge_product_owner +from fluid_build.providers.odps_standard import resolver + +# --------------------------------------------------------------------------- +# validate_target must not let a 0.7.8 mapper override abort a --dry-run batch +# --------------------------------------------------------------------------- + + +class _NameIndexingSubclass(FluidCommandCenterProvider): + """0.7.8-style subclass: overrides only map_contract_to_asset and, like + the 0.7.8 base mapper, indexes ``contract['name']``.""" + + def map_contract_to_asset(self, contract: Dict[str, Any]): # type: ignore[override] + asset = super().map_contract_to_asset(contract) + asset.name = contract["name"] + return asset + + +class TestValidateTargetGuardsLegacyMapper: + def _provider(self) -> _NameIndexingSubclass: + return _NameIndexingSubclass({"endpoint": "http://cc"}) + + def test_mapper_error_is_reported_not_raised(self) -> None: + fluid = {"id": "sales.refunds", "metadata": {"name": "Refunds"}} + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + ok, err = self._provider().validate_target(fluid, CatalogTarget.from_fluid(fluid)) + assert ok is False + assert err.startswith("Failed to map contract to asset:") + + def test_mappable_contract_still_validates(self) -> None: + fluid = {"id": "sales.orders", "name": "Orders"} + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + ok, err = self._provider().validate_target(fluid, CatalogTarget.from_fluid(fluid)) + assert (ok, err) == (True, None) + + +# --------------------------------------------------------------------------- +# Product owner edits reach only per-port owners inherited from the product +# --------------------------------------------------------------------------- + + +class TestMergeProductOwner: + PRODUCT_THEN = {"team": "t", "email": "a@x.com"} + PRODUCT_NOW = {"team": "t", "email": "z@z.com"} + + def test_port_with_its_own_team_keeps_its_identity(self) -> None: + doc: Dict[str, Any] = {"owner": {"team": "commerce-team", "email": "alice@acme.com"}} + _merge_product_owner(doc, self.PRODUCT_THEN, self.PRODUCT_NOW) + assert doc["owner"] == {"team": "commerce-team", "email": "alice@acme.com"} + + def test_port_that_inherited_the_product_owner_follows_the_edit(self) -> None: + doc: Dict[str, Any] = {"owner": dict(self.PRODUCT_THEN)} + _merge_product_owner(doc, self.PRODUCT_THEN, self.PRODUCT_NOW) + assert doc["owner"] == self.PRODUCT_NOW + + def test_only_inherited_keys_change(self) -> None: + # Same team as the product, but the port has its own contact email. + doc: Dict[str, Any] = {"owner": {"team": "t", "email": "port@acme.com"}} + _merge_product_owner(doc, self.PRODUCT_THEN, {"team": "t2", "email": "z@z.com"}) + assert doc["owner"] == {"team": "t2", "email": "port@acme.com"} + + def test_removing_the_product_owner_keeps_a_distinct_port_team(self) -> None: + doc: Dict[str, Any] = {"owner": {"team": "commerce-team"}} + _merge_product_owner(doc, self.PRODUCT_THEN, None) + assert doc["owner"] == {"team": "commerce-team"} + + def test_removing_the_product_owner_removes_an_inherited_one(self) -> None: + doc: Dict[str, Any] = {"owner": dict(self.PRODUCT_THEN)} + _merge_product_owner(doc, self.PRODUCT_THEN, None) + assert "owner" not in doc + + def test_does_not_mutate_the_document_owner_in_place(self) -> None: + original = {"team": "t", "email": "a@x.com"} + doc: Dict[str, Any] = {"owner": original} + _merge_product_owner(doc, self.PRODUCT_THEN, self.PRODUCT_NOW) + assert original == {"team": "t", "email": "a@x.com"} + + +# --------------------------------------------------------------------------- +# resolver.urlopen accepts timeout=None (deadline-only) +# --------------------------------------------------------------------------- + + +class _FakeOpener: + def __init__(self, seen: List[Any]) -> None: + self.seen = seen + + def open(self, url: str, timeout: Any = None) -> str: + self.seen.append(timeout) + return "response" + + +class TestUrlopenTimeoutNone: + def test_none_timeout_uses_the_remaining_deadline(self, monkeypatch) -> None: + seen: List[Any] = [] + monkeypatch.setattr( + resolver.urllib.request, "build_opener", lambda *handlers: _FakeOpener(seen) + ) + out = resolver.urlopen( + "http://example.test/c.yaml", timeout=None, deadline=time.monotonic() + 5 + ) + assert out == "response" + assert len(seen) == 1 and 0 < seen[0] <= 5 + + @pytest.mark.parametrize("timeout, expected_max", [(2.0, 2.0), (60.0, 5.0)]) + def test_numeric_timeout_is_capped_by_the_deadline( + self, monkeypatch, timeout: float, expected_max: float + ) -> None: + seen: List[Any] = [] + monkeypatch.setattr( + resolver.urllib.request, "build_opener", lambda *handlers: _FakeOpener(seen) + ) + resolver.urlopen( + "http://example.test/c.yaml", timeout=timeout, deadline=time.monotonic() + 5 + ) + assert 0 < seen[0] <= expected_max diff --git a/tests/test_review_fixes_catalogs_dmm.py b/tests/test_review_fixes_catalogs_dmm.py new file mode 100644 index 00000000..6d194ee3 --- /dev/null +++ b/tests/test_review_fixes_catalogs_dmm.py @@ -0,0 +1,780 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Regression tests for the catalogs / Data Mesh Manager review findings. + +Covers: DMM data-product-spec resolution (env-var opt-out, catalog adapter +opt-outs), ``--team-id`` on the ODPS path, the deprecated CatalogAsset +publish API (built-in providers and third-party plugins written against the +pre-0.7.9 ``BaseCatalogProvider``), lenient publish validation, user-config +layering and ``${VAR}`` handling in ``save_user_config``. +""" + +from __future__ import annotations + +import asyncio +import warnings +from pathlib import Path +from types import SimpleNamespace +from typing import Any, Dict, List, Optional, Tuple +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest +import yaml + +from fluid_build.providers.catalogs import ( + CATALOG_PROVIDERS, + BaseCatalogProvider, + CatalogAsset, + CatalogTarget, + PublishResult, +) +from fluid_build.providers.catalogs.datamesh_manager import DataMeshManagerCatalogProvider +from fluid_build.providers.catalogs.fluid_cc import FluidCommandCenterProvider +from fluid_build.providers.datamesh_manager import DataMeshManagerProvider + +REPO_ROOT = Path(__file__).resolve().parents[1] +BITCOIN_EXAMPLE = ( + REPO_ROOT / "examples" / "bitcoin-price-api-imperative-part-a" / "contract.fluid.yaml" +) + + +def _run(coro): + loop = asyncio.new_event_loop() + try: + return loop.run_until_complete(coro) + finally: + loop.close() + + +def _schema_valid_contract(**overrides: Any) -> Dict[str, Any]: + """A FLUID data product as the schema requires it: ``kind`` is mandatory + and (for data products) always ``DataProduct``.""" + contract: Dict[str, Any] = { + "fluidVersion": "0.7.2", + "kind": "DataProduct", + "id": "sales.orders", + "name": "Orders", + "description": "Orders data product", + "domain": "sales", + "metadata": {"owner": {"team": "sales-team", "email": "sales@example.com"}}, + "exposes": [], + } + contract.update(overrides) + return contract + + +@pytest.fixture +def no_spec_env(monkeypatch): + monkeypatch.delenv("DMM_DATA_PRODUCT_SPEC", raising=False) + + +# --------------------------------------------------------------------------- +# Finding 7 — DMM_DATA_PRODUCT_SPEC must work for kind: DataProduct +# --------------------------------------------------------------------------- + + +class TestDmmSpecEnvVarOptOut: + def test_env_var_selects_dps_for_kind_dataproduct_apply(self, monkeypatch): + monkeypatch.setenv("DMM_DATA_PRODUCT_SPEC", "0.0.1") + provider = DataMeshManagerProvider(api_key="k", api_url="https://dmm.example") + + result = provider.apply(_schema_valid_contract(), dry_run=True) + + assert result["payload"]["dataProductSpecification"] == "0.0.1" + assert "apiVersion" not in result["payload"] + + def test_env_var_selects_dps_for_kind_dataproduct_plan(self, monkeypatch): + monkeypatch.setenv("DMM_DATA_PRODUCT_SPEC", "0.0.1") + provider = DataMeshManagerProvider(api_key="k", api_url="https://dmm.example") + + actions = provider.plan(_schema_valid_contract()) + + assert actions[0]["payload"]["dataProductSpecification"] == "0.0.1" + + def test_resolution_order(self, monkeypatch): + provider = DataMeshManagerProvider(api_key="k", api_url="https://dmm.example") + resolve = provider._resolve_data_product_specification + odps = provider.DATA_PRODUCT_SPEC_ODPS + dps = provider.DATA_PRODUCT_SPEC_DPS + + monkeypatch.delenv("DMM_DATA_PRODUCT_SPEC", raising=False) + assert resolve(None) == odps + assert resolve(None, fluid=_schema_valid_contract()) == odps + + monkeypatch.setenv("DMM_DATA_PRODUCT_SPEC", "0.0.1") + assert resolve(None, fluid=_schema_valid_contract()) == "0.0.1" + # provider hint beats the env var, explicit value beats both + assert resolve(None, provider_hint="odps") == odps + assert resolve(None, provider_hint="dps") == dps + assert resolve("odps", provider_hint="dps") == "odps" + + def test_default_without_env_is_odps(self, no_spec_env): + provider = DataMeshManagerProvider(api_key="k", api_url="https://dmm.example") + + result = provider.apply(_schema_valid_contract(), dry_run=True) + + assert result["payload"]["apiVersion"] + assert "dataProductSpecification" not in result["payload"] + + +# --------------------------------------------------------------------------- +# Finding 8 — --team-id must win on the ODPS path +# --------------------------------------------------------------------------- + + +class TestDmmTeamIdOverrideOdps: + def test_dry_run_payload_uses_team_id_override(self, no_spec_env): + provider = DataMeshManagerProvider(api_key="k", api_url="https://dmm.example") + + result = provider.apply(_schema_valid_contract(), dry_run=True, team_id="platform-team") + + assert result["payload"]["team"]["name"] == "platform-team" + + def test_dry_run_payload_uses_team_id_override_explicit_odps(self, no_spec_env): + provider = DataMeshManagerProvider(api_key="k", api_url="https://dmm.example") + + result = provider.apply( + _schema_valid_contract(), + dry_run=True, + team_id="platform-team", + data_product_specification="odps", + ) + + assert result["payload"]["team"]["name"] == "platform-team" + + def test_without_override_owner_team_is_kept(self, no_spec_env): + provider = DataMeshManagerProvider(api_key="k", api_url="https://dmm.example") + + result = provider.apply(_schema_valid_contract(), dry_run=True) + + assert result["payload"]["team"]["name"] == "sales-team" + + def test_live_publish_puts_product_on_override_team(self, no_spec_env): + provider = DataMeshManagerProvider(api_key="k", api_url="https://dmm.example") + calls: List[Tuple[str, str, Any]] = [] + + def fake_request(method, path, json_body=None): + calls.append((method, path, json_body)) + return SimpleNamespace(status_code=200, json=lambda: {}) + + provider._request = fake_request # type: ignore[assignment] + ensured: List[str] = [] + provider._ensure_team = lambda fluid, tid: ensured.append(tid) # type: ignore + + result = provider.apply(_schema_valid_contract(), team_id="platform-team") + + assert ensured == ["platform-team"] + assert result["team_id"] == "platform-team" + put = [c for c in calls if c[1] == "/api/dataproducts/sales.orders"] + assert put and put[0][2]["team"]["name"] == "platform-team" + + def test_cli_dry_run_shows_team_id_override(self, tmp_path, no_spec_env, capsys): + from fluid_build.cli.datamesh_manager import _cmd_publish + + contract_path = tmp_path / "contract.fluid.yaml" + contract_path.write_text(yaml.safe_dump(_schema_valid_contract()), encoding="utf-8") + args = SimpleNamespace( + contract=str(contract_path), + overlay=None, + team_id="platform-team", + dry_run=True, + with_contract=False, + no_create_team=False, + contract_format="odcs", + data_product_spec=None, + validate_generated_contracts=False, + validation_mode="warn", + fail_on_contract_error=False, + api_key="k", + api_url="https://dmm.example", + ) + + with patch( + "fluid_build.cli.datamesh_manager._print_dry_run" + ) as print_dry_run: # capture the exact result the CLI would render + rc = _cmd_publish(args) + + assert rc == 0 + shown = print_dry_run.call_args[0][0] + assert shown["payload"]["team"]["name"] == "platform-team" + + +# --------------------------------------------------------------------------- +# Finding 42 — catalog adapter goes through the resolver + has opt-outs +# --------------------------------------------------------------------------- + + +def _dmm_catalog(config_extra: Optional[Dict[str, Any]] = None): + config: Dict[str, Any] = {"endpoint": "https://dmm.example", "auth": {"api_key": "k"}} + config.update(config_extra or {}) + catalog = DataMeshManagerCatalogProvider(config) + puts: List[Dict[str, Any]] = [] + + def fake_request(method, path, json_body=None): + if method == "PUT" and path.startswith("/api/dataproducts/"): + puts.append(json_body) + return SimpleNamespace(status_code=200, json=lambda: {}) + + catalog._provider._request = fake_request # type: ignore[assignment] + catalog._provider._ensure_team = lambda fluid, tid: None # type: ignore[assignment] + return catalog, puts + + +class TestDmmCatalogAdapterSpecSelection: + def test_default_is_odps(self, no_spec_env): + catalog, puts = _dmm_catalog() + contract = _schema_valid_contract() + + result = _run(catalog.publish(contract, CatalogTarget.from_fluid(contract))) + + assert result.success, result.error + assert puts[0]["apiVersion"] + assert "dataProductSpecification" not in puts[0] + + def test_env_var_opt_out(self, monkeypatch): + monkeypatch.setenv("DMM_DATA_PRODUCT_SPEC", "0.0.1") + catalog, puts = _dmm_catalog() + contract = _schema_valid_contract() + + result = _run(catalog.publish(contract, CatalogTarget.from_fluid(contract))) + + assert result.success, result.error + assert puts[0]["dataProductSpecification"] == "0.0.1" + + @pytest.mark.parametrize( + "config_extra", + [{"data_product_spec": "0.0.1"}, {"provider_hint": "dps"}], + ) + def test_catalog_config_opt_out(self, no_spec_env, config_extra): + catalog, puts = _dmm_catalog(config_extra) + contract = _schema_valid_contract() + + result = _run(catalog.publish(contract, CatalogTarget.from_fluid(contract))) + + assert result.success, result.error + assert puts[0]["dataProductSpecification"] == "0.0.1" + + def test_catalog_hints_opt_out_and_override_config(self, no_spec_env): + catalog, puts = _dmm_catalog({"data_product_spec": "odps"}) + contract = _schema_valid_contract() + target = CatalogTarget.from_fluid( + contract, catalog_hints={"data_product_specification": "0.0.1"} + ) + + result = _run(catalog.publish(contract, target)) + + assert result.success, result.error + assert puts[0]["dataProductSpecification"] == "0.0.1" + + def test_fluid_publish_cli_path_honours_config_opt_out(self, tmp_path, no_spec_env): + """End to end through ``publish_contract`` with the catalog config + key set, as ``fluid publish --catalog datamesh-manager`` would run.""" + from fluid_build.cli import publish as pub_mod + + contract = _schema_valid_contract() + contract_path = tmp_path / "c.fluid.yaml" + contract_path.write_text(yaml.safe_dump(contract), encoding="utf-8") + catalog, puts = _dmm_catalog({"data_product_spec": "0.0.1"}) + config = MagicMock() + config.get_catalog_config.return_value = {"enabled": True} + + with patch.object(pub_mod, "load_contract", return_value=contract): + with patch.object(pub_mod, "get_catalog_provider", return_value=catalog): + result = _run( + pub_mod.publish_contract( + contract_path=contract_path, + catalog_name="datamesh-manager", + config=config, + skip_health_check=True, + ) + ) + + assert result.success, result.error + assert puts[0]["dataProductSpecification"] == "0.0.1" + + +# --------------------------------------------------------------------------- +# Findings 9 / 41 — deprecated CatalogAsset API keeps working for one release +# --------------------------------------------------------------------------- + + +def _asset() -> CatalogAsset: + return CatalogAsset( + id="sales.orders", + name="Orders", + description="d", + type="dataproduct", + domain="sales", + owner="sales-team", + owner_email="s@example.com", + layer="Gold", + tags=[], + version="1.0.0", + platform="unknown", + location={}, + contract_yaml="id: sales.orders\n", + ) + + +class TestBuiltinProvidersAcceptLegacyAsset: + @pytest.mark.parametrize("method", ["publish", "update"]) + def test_fluid_cc_accepts_catalog_asset(self, method): + provider = FluidCommandCenterProvider({"endpoint": "http://cc.example"}) + seen: Dict[str, Any] = {} + + async def fake_impl(fluid, target): + seen["fluid"], seen["target"] = fluid, target + return PublishResult(True, provider.name, target.contract_id) + + provider.health_check = AsyncMock(return_value=True) # type: ignore[assignment] + provider._publish_impl = fake_impl # type: ignore[assignment] + + with pytest.warns(DeprecationWarning): + result = _run(getattr(provider, method)(_asset())) + + assert isinstance(result, PublishResult) + assert result.success, result.error + assert seen["target"].contract_id == "sales.orders" + assert seen["target"].contract_yaml == "id: sales.orders\n" + assert seen["fluid"]["id"] == "sales.orders" + + @pytest.mark.parametrize("method", ["publish", "update"]) + def test_dmm_catalog_accepts_catalog_asset(self, method): + provider = DataMeshManagerCatalogProvider({"endpoint": "http://dmm.example"}) + seen: Dict[str, Any] = {} + + def fake_apply(fluid, **kwargs): + seen["fluid"] = fluid + return {"url": "http://dmm.example/x"} + + provider._provider.apply = fake_apply # type: ignore[assignment] + + with pytest.warns(DeprecationWarning): + result = _run(getattr(provider, method)(_asset())) + + assert result.success, result.error + assert result.asset_id == "sales.orders" + assert seen["fluid"]["id"] == "sales.orders" + + def test_target_defaults_from_fluid(self): + provider = DataMeshManagerCatalogProvider({"endpoint": "http://dmm.example"}) + provider._provider.apply = lambda fluid, **kw: {} # type: ignore[assignment] + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always", DeprecationWarning) + result = _run(provider.publish(_schema_valid_contract())) + assert result.asset_id == "sales.orders" + # The new (fluid, target) API must not trip the catalog's own + # deprecation shims. Unrelated DeprecationWarnings (e.g. Python + # 3.12's datetime.utcnow) are not this test's concern. + catalog_deprecations = [ + str(w.message) + for w in caught + if issubclass(w.category, DeprecationWarning) + and ("CatalogAsset" in str(w.message) or "deprecated publish" in str(w.message)) + ] + assert catalog_deprecations == [] + + @pytest.mark.parametrize("method", ["publish_asset", "update_asset"]) + def test_publish_asset_shims_forward_full_fluid(self, method): + provider = DataMeshManagerCatalogProvider({"endpoint": "http://dmm.example"}) + seen: Dict[str, Any] = {} + + def fake_apply(fluid, **kwargs): + seen["fluid"] = fluid + return {} + + provider._provider.apply = fake_apply # type: ignore[assignment] + contract = _schema_valid_contract( + exposes=[{"exposeId": "a", "kind": "table"}, {"exposeId": "b", "kind": "table"}] + ) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + asset = provider.map_contract_to_asset(contract) + + with pytest.warns(DeprecationWarning): + result = _run(getattr(provider, method)(asset)) + + assert result.success, result.error + assert len(seen["fluid"]["exposes"]) == 2 + + +class LegacyCatalog(BaseCatalogProvider): + """A third-party plugin written exactly against origin/main's + ``BaseCatalogProvider`` API (``publish(asset)``, ``update(asset)``, + ``verify(asset_id)``, ``validate_asset`` / ``map_contract_to_asset`` + overrides).""" + + name = "legacy" + events: List[Tuple[str, Any]] = [] + + async def publish(self, asset: CatalogAsset) -> PublishResult: + self.events.append(("publish", asset)) + return PublishResult( + success=True, + catalog_id=self.name, + asset_id=asset.id, + catalog_url=f"http://legacy.example/{asset.id}", + ) + + async def update(self, asset: CatalogAsset) -> PublishResult: + self.events.append(("update", asset)) + return await self.publish(asset) + + async def verify(self, asset_id: str) -> bool: + self.events.append(("verify", asset_id)) + return True + + def map_contract_to_asset(self, contract: Dict[str, Any]) -> CatalogAsset: + self.events.append(("map", contract)) + asset = super().map_contract_to_asset(contract) + asset.tags = ["mapped-by-plugin"] + return asset + + def validate_asset(self, asset: CatalogAsset) -> Tuple[bool, Optional[str]]: + self.events.append(("validate", asset)) + if asset.domain == "blocked": + return False, "plugin says no" + return super().validate_asset(asset) + + +@pytest.fixture +def legacy_catalog(monkeypatch): + LegacyCatalog.events = [] + monkeypatch.setitem(CATALOG_PROVIDERS, "legacy-cat", LegacyCatalog) + return LegacyCatalog + + +def _publish_via_cli(tmp_path, contract, **kwargs) -> PublishResult: + from fluid_build.cli import publish as pub_mod + + contract_path = tmp_path / "c.fluid.yaml" + contract_path.write_text(yaml.safe_dump(contract), encoding="utf-8") + config = MagicMock() + config.get_catalog_config.return_value = {"enabled": True} + with patch.object(pub_mod, "load_contract", return_value=contract): + return _run( + pub_mod.publish_contract( + contract_path=contract_path, + catalog_name="legacy-cat", + config=config, + **kwargs, + ) + ) + + +class TestLegacyThirdPartyCatalogPlugin: + def test_detected_as_legacy(self, legacy_catalog): + assert LegacyCatalog._uses_legacy_asset_api() is True + assert FluidCommandCenterProvider._uses_legacy_asset_api() is False + assert DataMeshManagerCatalogProvider._uses_legacy_asset_api() is False + + def test_publish_routes_through_asset_flow(self, tmp_path, legacy_catalog): + with pytest.warns(DeprecationWarning, match="publish\\(fluid, target\\)"): + result = _publish_via_cli(tmp_path, _schema_valid_contract()) + + assert result.success, result.error + assert result.asset_id == "sales.orders" + kinds = [k for k, _ in legacy_catalog.events] + assert kinds == ["map", "publish"] + published = legacy_catalog.events[-1][1] + assert isinstance(published, CatalogAsset) + assert published.tags == ["mapped-by-plugin"] # plugin's mapper honoured + assert published.contract_yaml and "sales.orders" in published.contract_yaml + + def test_dry_run_uses_plugin_validate_asset(self, tmp_path, legacy_catalog): + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + blocked = _publish_via_cli( + tmp_path, _schema_valid_contract(domain="blocked"), dry_run=True + ) + allowed = _publish_via_cli(tmp_path, _schema_valid_contract(), dry_run=True) + + assert blocked.success is False + assert blocked.error == "plugin says no" + assert allowed.success is True + assert "validate" in [k for k, _ in legacy_catalog.events] + assert not [k for k, _ in legacy_catalog.events if k == "publish"] + + def test_verify_only_uses_asset_id(self, tmp_path, legacy_catalog): + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + result = _publish_via_cli(tmp_path, _schema_valid_contract(), verify_only=True) + + assert result.success is True + assert ("verify", "sales.orders") in legacy_catalog.events + + def test_health_check_still_runs(self, tmp_path, legacy_catalog): + unhealthy = AsyncMock(return_value=False) + with patch.object(LegacyCatalog, "health_check", unhealthy): + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + result = _publish_via_cli(tmp_path, _schema_valid_contract()) + + assert result.success is False + assert "health check failed" in result.error + assert not [k for k, _ in legacy_catalog.events if k == "publish"] + + @pytest.mark.parametrize("method", ["publish_asset", "update_asset"]) + def test_publish_asset_shim_on_legacy_plugin(self, legacy_catalog, method): + plugin = LegacyCatalog({}) + with pytest.warns(DeprecationWarning): + result = _run(getattr(plugin, method)(_asset())) + assert result.success + assert legacy_catalog.events[0][0] == method.split("_")[0] + + def test_new_api_validate_target_consults_overridden_validate_asset(self): + class NewApiWithLegacyValidator(BaseCatalogProvider): + async def publish(self, fluid, target=None): + return PublishResult(True, "x", "x") + + async def update(self, fluid, target=None): + return PublishResult(True, "x", "x") + + async def verify(self, contract_id): + return True + + def validate_asset(self, asset): + return (asset.domain != "blocked", "nope") + + p = NewApiWithLegacyValidator({}) + bad = _schema_valid_contract(domain="blocked") + ok, err = p.validate_target(bad, CatalogTarget.from_fluid(bad)) + assert (ok, err) == (False, "nope") + + +# --------------------------------------------------------------------------- +# Findings 10 / 20 — lenient validation (domain/owner optional) +# --------------------------------------------------------------------------- + + +class _NewApiStub(BaseCatalogProvider): + async def publish(self, fluid, target=None): + return PublishResult(True, "stub", "x") + + async def update(self, fluid, target=None): + return PublishResult(True, "stub", "x") + + async def verify(self, contract_id): + return True + + +class TestLenientValidateTarget: + @pytest.mark.parametrize( + "contract", + [ + {"id": "a.b", "name": "No domain", "metadata": {"owner": {"team": "t"}}}, + {"id": "a.b", "name": "Email owner", "metadata": {"owner": {"email": "x@y.z"}}}, + {"id": "a.b", "name": "Nothing else"}, + { + "id": "a.b", + "name": "Business ctx", + "metadata": {"businessContext": {"domain": "sales"}}, + }, + ], + ) + def test_missing_domain_or_owner_is_valid(self, contract): + p = _NewApiStub({}) + assert p.validate_target(contract, CatalogTarget.from_fluid(contract)) == (True, None) + + def test_id_and_name_still_required(self): + p = _NewApiStub({}) + ok, err = p.validate_target({}, CatalogTarget(contract_id="", contract_name="x")) + assert ok is False and "id" in err + ok, err = p.validate_target({}, CatalogTarget(contract_id="x", contract_name="")) + assert ok is False and "name" in err + + def test_fluid_cc_publishes_shipped_example_without_domain(self): + from fluid_build.loader import load_contract + + contract = load_contract(str(BITCOIN_EXAMPLE)) + assert not contract.get("domain") + provider = FluidCommandCenterProvider({"endpoint": "http://cc.example"}) + sent: Dict[str, Any] = {} + + async def fake_impl(fluid, target): + from fluid_build.providers.catalogs.fluid_cc.provider import ( + _build_cc_asset_payload, + ) + + sent.update(_build_cc_asset_payload(fluid, target)) + return PublishResult(True, provider.name, target.contract_id) + + provider.health_check = AsyncMock(return_value=True) # type: ignore[assignment] + provider._publish_impl = fake_impl # type: ignore[assignment] + + result = _run(provider.publish(contract, CatalogTarget.from_fluid(contract))) + + assert result.success, result.error + assert sent["metadata"]["domain"] == "general" + + def test_dry_run_accepts_shipped_example_without_domain(self): + from fluid_build.cli import publish as pub_mod + + config = MagicMock() + config.get_catalog_config.return_value = {"endpoint": "http://cc.example"} + result = _run( + pub_mod.publish_contract( + contract_path=BITCOIN_EXAMPLE, + catalog_name="fluid-command-center", + config=config, + dry_run=True, + ) + ) + assert result.success is True, result.error + + +# --------------------------------------------------------------------------- +# Finding 11 — ~/.fluid/config.yaml is an extra, lowest-priority user layer +# --------------------------------------------------------------------------- + + +@pytest.fixture +def fake_home(tmp_path, monkeypatch): + home = tmp_path / "home" + home.mkdir() + project = tmp_path / "project" + project.mkdir() + monkeypatch.setattr("pathlib.Path.home", lambda: home) + monkeypatch.chdir(project) + for var in ("FLUID_CC_ENDPOINT", "FLUID_API_KEY"): + monkeypatch.delenv(var, raising=False) + return home + + +class TestUserConfigLayering: + def test_dot_fluid_config_does_not_hide_fluidrc(self, fake_home): + from fluid_build.config_manager import FluidConfig + + (fake_home / ".fluid").mkdir() + (fake_home / ".fluid" / "config.yaml").write_text( + "command_center:\n url: http://cc.example\n api_key: cc-key\n" + ) + (fake_home / ".fluidrc.yaml").write_text( + "catalogs:\n datamesh-manager:\n endpoint: http://dmm.example\n" + ) + + cfg = FluidConfig() + + assert cfg.get("command_center.url") == "http://cc.example" + assert cfg.get_catalog_config("datamesh-manager")["endpoint"] == "http://dmm.example" + + def test_legacy_file_wins_on_conflict(self, fake_home): + from fluid_build.config_manager import FluidConfig + + (fake_home / ".fluid").mkdir() + (fake_home / ".fluid" / "config.yaml").write_text( + "logging:\n level: DEBUG\nnetwork:\n timeout: 99\n" + ) + (fake_home / ".config" / "fluid").mkdir(parents=True) + (fake_home / ".config" / "fluid" / "config.yaml").write_text("logging:\n level: ERROR\n") + + cfg = FluidConfig() + + assert cfg.get("logging.level") == "ERROR" + assert cfg.get("network.timeout") == 99 + + def test_legacy_first_found_order_preserved(self, fake_home): + from fluid_build.config_manager import FluidConfig + + (fake_home / ".fluidrc.yaml").write_text("logging:\n level: WARNING\n") + (fake_home / ".config" / "fluid").mkdir(parents=True) + (fake_home / ".config" / "fluid" / "config.yaml").write_text( + "logging:\n level: ERROR\nnetwork:\n timeout: 77\n" + ) + + cfg = FluidConfig() + + # main's behaviour: only the first found legacy file is read + assert cfg.get("logging.level") == "WARNING" + assert cfg.get("network.timeout") != 77 + + def test_file_config_does_not_leak_into_later_instances(self, fake_home): + from fluid_build.config_manager import DEFAULT_CONFIG, FluidConfig + + (fake_home / ".fluidrc.yaml").write_text( + "catalogs:\n leaky:\n endpoint: http://leak.example\n" + ) + FluidConfig() + assert "leaky" not in DEFAULT_CONFIG["catalogs"] + + +# --------------------------------------------------------------------------- +# Finding 43 — save_user_config never writes resolved ${VAR} secrets +# --------------------------------------------------------------------------- + + +class TestConfigEnvPlaceholdersOnSave: + def test_save_keeps_placeholder(self, fake_home, tmp_path, monkeypatch): + from fluid_build.config_manager import FluidConfig + + monkeypatch.setenv("MY_DMM_KEY", "sk-live-SECRET") + (fake_home / ".fluidrc.yaml").write_text( + "catalogs:\n datamesh-manager:\n endpoint: http://dmm.example\n" + " api_key: ${MY_DMM_KEY}\n" + ) + cfg = FluidConfig() + # Reads still see the expanded value + assert cfg.get_catalog_config("datamesh-manager")["api_key"] == "sk-live-SECRET" + assert cfg.get("catalogs.datamesh-manager.api_key") == "sk-live-SECRET" + assert cfg.to_dict()["catalogs"]["datamesh-manager"]["api_key"] == "sk-live-SECRET" + + cfg.set("logging.level", "DEBUG") + out = tmp_path / "saved.yaml" + cfg.save_user_config(path=out) + + text = out.read_text() + assert "sk-live-SECRET" not in text + saved = yaml.safe_load(text) + assert saved["catalogs"]["datamesh-manager"]["api_key"] == "${MY_DMM_KEY}" + assert saved["logging"]["level"] == "DEBUG" + + def test_catalog_env_override_not_persisted(self, fake_home, tmp_path, monkeypatch): + from fluid_build.config_manager import FluidConfig + + monkeypatch.setenv("FLUID_API_KEY", "fluid-SECRET") + cfg = FluidConfig() + assert cfg.get_catalog_config("fluid-command-center")["auth"]["api_key"] == "fluid-SECRET" + + out = tmp_path / "saved.yaml" + cfg.save_user_config(path=out) + assert "fluid-SECRET" not in out.read_text() + + +# --------------------------------------------------------------------------- +# Findings 58 / 72 — live e2e script hygiene (static checks; never runs DMM) +# --------------------------------------------------------------------------- + + +class TestDmmLiveE2EScript: + SCRIPT = REPO_ROOT / "scripts" / "dmm_live_e2e.sh" + + def test_bash_syntax(self): + import shutil + import subprocess + + bash = shutil.which("bash") + if not bash: + pytest.skip("bash not available") + proc = subprocess.run([bash, "-n", str(self.SCRIPT)], capture_output=True, text=True) + assert proc.returncode == 0, proc.stderr + + def test_no_predictable_workdir_no_key_in_argv_no_home_overwrite(self): + text = self.SCRIPT.read_text(encoding="utf-8") + assert "mktemp -d" in text + assert "dmm_live_e2e.$$" not in text + assert '-H "x-api-key:' not in text + assert '-H @"$HDR_FILE"' in text + assert 'FLUID_CFG="$HOME/.fluid/config.yaml"' not in text + assert 'HOME="$FLUID_HOME"' in text diff --git a/tests/test_review_fixes_forge_packaging.py b/tests/test_review_fixes_forge_packaging.py new file mode 100644 index 00000000..bb800b2f --- /dev/null +++ b/tests/test_review_fixes_forge_packaging.py @@ -0,0 +1,543 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +"""Regression tests for the forge-packaging review fixes. + +Covers the ``fluid forge --seed-from`` ground-truth pipeline (the seed view +must be FLUID-0.7.2-valid and shared by the prompt and the guard; missing +exposes, FLUID inputs, unresolved output ports, YAML dates, failure +classification, directory provenance) plus the packaging metadata for the +``odcs-strict`` extra and the DMM ``requests`` dependency. + +The copilot tests drive the real :func:`generate_copilot_artifacts` with the +real ``normalize_generation_payload`` and ``validate_generated_result`` (the +strict FLUID 0.7.2 schema check). Only the network call (``call_llm``) and the +provider adapter lookup are patched. +""" + +from __future__ import annotations + +import copy +import json +import shutil +import sys +from pathlib import Path +from typing import Any, Callable, Dict, List, Optional +from unittest.mock import MagicMock, patch + +import pytest +import yaml + +from fluid_build.cli.forge_copilot_prompts import build_system_prompt, build_user_prompt +from fluid_build.cli.forge_copilot_runtime import ( + CopilotGenerationError, + generate_copilot_artifacts, +) +from fluid_build.cli.forge_copilot_seed import ( + SeedOptions, + diff_against_seed, + format_ground_truth_prompt_block, + load_seed, + resolve_at_path, +) +from fluid_build.providers.base import ProviderError +from fluid_build.schema_manager import FluidSchemaManager + +REPO_ROOT = Path(__file__).resolve().parent.parent +FIXTURES = Path(__file__).parent / "fixtures" +BUNDLE_DIR = FIXTURES / "odps" / "product-bitol" +ODCS_FULL = FIXTURES / "odcs" / "contract-full.yaml" +ODCS_MINIMAL = FIXTURES / "odcs" / "contract-minimal.yaml" +UPSTREAM_FULL = FIXTURES / "upstream" / "odcs" / "full-example.v3.2.0.odcs.yaml" +UPSTREAM_ADVENTUREWORKS = ( + FIXTURES / "upstream" / "odcs" / "postgresql-adventureworks.v3.0.0.odcs.yaml" +) + +SEED_INPUTS = [ + pytest.param(BUNDLE_DIR, id="odps-bundle-dir"), + pytest.param(next(BUNDLE_DIR.glob("*.odps.yaml")), id="odps-file"), + pytest.param(ODCS_FULL, id="odcs-contract-full"), + pytest.param(ODCS_MINIMAL, id="odcs-contract-minimal"), + pytest.param(FIXTURES / "odps" / "contracts-only", id="odcs-only-dir"), + pytest.param(UPSTREAM_FULL, id="upstream-full-example-v3.2.0"), + pytest.param(UPSTREAM_ADVENTUREWORKS, id="upstream-adventureworks-v3.0.0"), +] + + +# --------------------------------------------------------------------------- +# Helpers: a scripted "LLM" that copies the seed ground truth verbatim into a +# minimal FLUID 0.7.2 contract, the way the copilot is instructed to. +# --------------------------------------------------------------------------- + + +def _capability_matrix() -> Dict[str, Any]: + return { + "providers": ["local", "gcp", "aws", "snowflake"], + "templates": {"starter": {}, "analytics": {}}, + "build_engines": ["sql", "python", "dbt"], + } + + +def _llm_config(): + from fluid_build.cli.forge_copilot_llm_providers import LlmConfig + + return LlmConfig( + provider="anthropic", + model="mock-model", + endpoint="https://mock.example.com/v1/messages", + api_key="mock-key", + ) + + +def _discovery_report(): + from fluid_build.cli.forge_copilot_discovery import DiscoveryReport + + return DiscoveryReport(workspace_roots=["/tmp"]) + + +def _ground_truth_from_prompt(user_prompt: str) -> Dict[str, Any]: + return json.loads(user_prompt)["seed_ground_truth"]["ground_truth"] + + +def _contract_from_ground_truth(gt: Dict[str, Any]) -> Dict[str, Any]: + """Wrap the ground truth (copied verbatim) in the parts the LLM fills in.""" + exposes = [] + for expose in gt["exposes"]: + first_col = expose["contract"]["schema"][0]["name"] if expose["contract"]["schema"] else "x" + built = dict(copy.deepcopy(expose)) + built.update( + { + "kind": "table", + "binding": { + "platform": "local", + "format": "csv", + "location": {"path": f"runtime/out/{expose['exposeId']}.csv"}, + }, + "semantics": { + "name": expose["exposeId"], + "description": "Semantic model.", + "entities": [{"name": first_col, "type": "primary"}], + "measures": [{"name": "row_count", "agg": "count", "expr": first_col}], + "dimensions": [{"name": first_col, "type": "categorical"}], + "metrics": [{"name": "rows", "type": "simple", "measure": "row_count"}], + }, + } + ) + exposes.append(built) + contract: Dict[str, Any] = { + "fluidVersion": "0.7.2", + "kind": "DataProduct", + "id": gt.get("id") or "seeded.product", + "name": gt.get("name") or "Seeded Product", + "metadata": copy.deepcopy(gt.get("metadata")) + or {"owner": {"team": "data-team", "email": "data-team@example.com"}}, + "builds": [ + { + "id": "main_build", + "pattern": "embedded-logic", + "engine": "sql", + "properties": {"sql": "SELECT 1 AS id"}, + "execution": { + "trigger": {"type": "manual", "iterations": 1}, + "runtime": {"platform": "local", "resources": {"cpu": "1", "memory": "2Gi"}}, + }, + } + ], + "exposes": exposes, + } + for key in ("description", "domain", "tags"): + if key in gt: + contract[key] = copy.deepcopy(gt[key]) + contract["metadata"].setdefault("layer", "Bronze") + return contract + + +def _payload(contract: Dict[str, Any]) -> Dict[str, Any]: + return { + "recommended_template": "starter", + "recommended_provider": "local", + "readme_markdown": "# Seeded\n", + "contract": contract, + } + + +class _CopierLlm: + """Each call reads the seed block from the user prompt and returns a + contract built from it, optionally transformed per attempt.""" + + def __init__(self, transforms: Optional[List[Optional[Callable]]] = None): + self.transforms = transforms or [None] + self.calls = 0 + self.user_prompts: List[str] = [] + self.system_prompts: List[str] = [] + + def __call__(self, provider_adapter, llm_config, system_prompt, user_prompt): + self.system_prompts.append(system_prompt) + self.user_prompts.append(user_prompt) + contract = _contract_from_ground_truth(_ground_truth_from_prompt(user_prompt)) + transform = self.transforms[min(self.calls, len(self.transforms) - 1)] + self.calls += 1 + if transform is not None: + contract = transform(contract) + return json.dumps(_payload(contract)) + + +def _run_copilot(seed_path: Path, llm: _CopierLlm): + with ( + patch("fluid_build.cli.forge_copilot_runtime.call_llm", side_effect=llm), + patch("fluid_build.cli.forge_copilot_runtime.get_llm_provider", return_value=MagicMock()), + ): + return generate_copilot_artifacts( + context={"project_goal": "seeded product"}, + llm_config=_llm_config(), + discovery_report=_discovery_report(), + capability_matrix=_capability_matrix(), + seed_options=SeedOptions(seed_from=seed_path, allow_remote=False), + ) + + +# --------------------------------------------------------------------------- +# Finding 12 — the seed view must be FLUID 0.7.2-valid and shared by the +# prompt and the guard. +# --------------------------------------------------------------------------- + + +class TestSeedViewIsFluid072Valid: + @pytest.mark.parametrize("seed_path", SEED_INPUTS) + def test_verbatim_copy_passes_strict_validation_and_guard(self, seed_path: Path) -> None: + seed = load_seed(seed_path, allow_remote=False) + block = format_ground_truth_prompt_block(seed) + contract = _contract_from_ground_truth(json.loads(json.dumps(block))["ground_truth"]) + + validation = FluidSchemaManager().validate_contract( + contract, strict=True, offline_only=True + ) + assert validation.errors == [] + assert diff_against_seed(seed, contract) == [] + + @pytest.mark.parametrize("seed_path", SEED_INPUTS) + def test_copilot_succeeds_first_attempt_with_real_validation(self, seed_path: Path) -> None: + llm = _CopierLlm() + result = _run_copilot(seed_path, llm) + assert llm.calls == 1 + assert result.attempt_reports[0].validation_errors == [] + + def test_raw_seed_quality_is_not_required_on_the_candidate(self) -> None: + """The candidate may drop ODCS-only keys (quality/classification) and + the guard still passes, while a raw seed copy also passes.""" + seed = load_seed(ODCS_FULL) + assert diff_against_seed(seed, seed.fluid) == [] + stripped = copy.deepcopy(seed.fluid) + for fld in stripped["exposes"][0]["contract"]["schema"]: + fld.pop("quality", None) + assert diff_against_seed(seed, stripped) == [] + + def test_prompt_block_has_no_non_schema_keys(self) -> None: + seed = load_seed(UPSTREAM_FULL) + gt = format_ground_truth_prompt_block(seed)["ground_truth"] + for expose in gt["exposes"]: + assert "id" not in expose + assert "exposeId" in expose + for col in expose["contract"]["schema"]: + assert "quality" not in col + assert "classification" not in col + qos = expose.get("qos") or {} + # ODCS latency '4' (no unit) is not ISO-8601; it must not be + # presented as ground truth. + assert qos.get("freshnessSLO") != "4" + + def test_classification_maps_to_sensitivity_when_in_enum(self) -> None: + seed = load_seed(UPSTREAM_FULL) + gt = format_ground_truth_prompt_block(seed)["ground_truth"] + cols = {c["name"]: c for c in gt["exposes"][0]["contract"]["schema"]} + assert cols["rcvr_id"]["sensitivity"] == "restricted" + assert "sensitivity" not in cols["rcvr_cntry_code"] # 'public' is not a 0.7.2 level + + def test_schema_mutation_still_caught(self) -> None: + seed = load_seed(ODCS_FULL) + mutated = copy.deepcopy(seed.fluid) + mutated["exposes"][0]["contract"]["schema"][0]["type"] = "integer" + assert diff_against_seed(seed, mutated) + + def test_system_prompt_seed_clause_matches_schema(self) -> None: + prompt = build_system_prompt(_capability_matrix(), ["sql"], has_seed_ground_truth=True) + clause = prompt.split("GROUND TRUTH FROM SEED", 1)[1] + assert "exposeId" in clause + assert "exposes' ``id``" not in clause + # The 0.7.2 root schema forbids a top-level governance key. + assert "governance" not in clause + + +# --------------------------------------------------------------------------- +# Finding 44 — fewer exposes than the seed must be a mismatch, not IndexError +# --------------------------------------------------------------------------- + + +class TestMissingExposeIsAMismatch: + def test_resolve_at_path_out_of_range_returns_none(self) -> None: + assert resolve_at_path({"exposes": [{"id": "a"}]}, "exposes[3].contract.schema") is None + assert resolve_at_path({"exposes": {"0": 1}}, "exposes[0]") is None + assert resolve_at_path({"exposes": "abc"}, "exposes[0].x") is None + + def test_diff_reports_missing_expose(self) -> None: + seed = load_seed(BUNDLE_DIR) + candidate = copy.deepcopy(seed.fluid) + candidate["exposes"] = candidate["exposes"][:1] + mismatches = diff_against_seed(seed, candidate) + assert any(m["path"].startswith("exposes[1]") for m in mismatches) + + def test_copilot_repairs_after_dropped_expose(self) -> None: + def drop_second(contract): + contract["exposes"] = contract["exposes"][:1] + return contract + + llm = _CopierLlm([drop_second, None]) + result = _run_copilot(BUNDLE_DIR, llm) + assert llm.calls == 2 + assert any("exposes[1]" in e for e in result.attempt_reports[0].validation_errors) + + +# --------------------------------------------------------------------------- +# Finding 45 — FLUID inputs and unresolved output ports must not seed silently +# --------------------------------------------------------------------------- + + +def _write_fluid_contract(path: Path) -> Path: + path.write_text( + yaml.safe_dump( + { + "fluidVersion": "0.7.2", + "kind": "DataProduct", + "id": "hello.world", + "name": "Hello", + "metadata": {"owner": {"team": "t"}}, + "exposes": [], + } + ) + ) + return path + + +class TestSeedRejectsSilentEmptyGroundTruth: + def test_fluid_contract_file_is_rejected(self, tmp_path: Path) -> None: + fluid_file = _write_fluid_contract(tmp_path / "contract.fluid.yaml") + with pytest.raises(ProviderError, match="FLUID contract"): + load_seed(fluid_file) + + def test_repo_example_fluid_contract_is_rejected(self) -> None: + example = REPO_ROOT / "examples" / "01-hello-world" / "contract.fluid.yaml" + if not example.exists(): + pytest.skip("example contract not present") + with pytest.raises(ProviderError, match="FLUID contract"): + load_seed(example) + + def test_fluid_contract_in_directory_is_not_taken_as_odps(self, tmp_path: Path) -> None: + bundle = tmp_path / "bundle" + shutil.copytree(BUNDLE_DIR, bundle) + _write_fluid_contract(bundle / "contract.fluid.yaml") + seed = load_seed(bundle, allow_remote=False) + assert len(seed.fluid["exposes"]) == 2 + assert len(seed.ground_truth_paths) >= 2 + + def test_directory_with_only_a_fluid_contract_is_rejected(self, tmp_path: Path) -> None: + _write_fluid_contract(tmp_path / "contract.fluid.yaml") + with pytest.raises(ProviderError): + load_seed(tmp_path) + + def test_unresolved_output_port_in_directory_raises(self, tmp_path: Path) -> None: + bundle = tmp_path / "bundle" + shutil.copytree(BUNDLE_DIR, bundle) + (bundle / "commerce.orders-product.daily_orders.odcs.yaml").unlink() + with pytest.raises(ProviderError, match="daily_orders"): + load_seed(bundle, allow_remote=False) + + def test_unresolved_output_port_in_odps_file_raises(self, tmp_path: Path) -> None: + bundle = tmp_path / "bundle" + shutil.copytree(BUNDLE_DIR, bundle) + (bundle / "commerce.orders-product.daily_orders.odcs.yaml").unlink() + with pytest.raises(ProviderError, match="daily_orders"): + load_seed(next(bundle.glob("*.odps.yaml")), allow_remote=False) + + def test_odps_without_output_ports_is_rejected(self, tmp_path: Path) -> None: + odps = yaml.safe_load((next(BUNDLE_DIR.glob("*.odps.yaml"))).read_text()) + odps["outputPorts"] = [] + odps["inputPorts"] = [] + target = tmp_path / "empty.odps.yaml" + target.write_text(yaml.safe_dump(odps)) + with pytest.raises(ProviderError, match="no ground truth"): + load_seed(target, allow_remote=False) + + +# --------------------------------------------------------------------------- +# Finding 60 — YAML dates in a seed must not crash the prompt or the guard +# --------------------------------------------------------------------------- + + +_DATE_ODCS = """\ +version: "1.0.0" +apiVersion: v3.1.0 +kind: DataContract +id: dated.orders +status: active +name: dated +slaProperties: + - property: generalAvailability + value: 2024-01-01 +schema: + - name: orders + logicalType: object + physicalType: table + properties: + - name: order_id + logicalType: string + required: true + quality: + - type: library + metric: nullValues + mustBe: 0 + customProperties: + - property: effectiveFrom + value: 2024-01-01 +""" + + +class TestSeedWithDates: + @pytest.fixture + def dated_seed(self, tmp_path: Path) -> Path: + target = tmp_path / "orders.odcs.yaml" + target.write_text(_DATE_ODCS) + return target + + def test_prompt_block_is_json_serialisable(self, dated_seed: Path) -> None: + seed = load_seed(dated_seed) + block = format_ground_truth_prompt_block(seed) + json.dumps(block, sort_keys=True) + prompt = build_user_prompt( + context={"project_goal": "x"}, + discovery_report=_discovery_report(), + capability_matrix=_capability_matrix(), + seed_contract={}, + seed_template="starter", + seed_provider="local", + attempt_index=1, + previous_errors=[], + previous_payload=None, + seed_ground_truth=block, + ) + assert "seed_ground_truth" in prompt + + def test_json_round_tripped_seed_is_not_a_mismatch(self, dated_seed: Path) -> None: + seed = load_seed(dated_seed) + candidate = json.loads(json.dumps(seed.fluid, default=str)) + assert diff_against_seed(seed, candidate) == [] + + def test_copilot_runs_with_dated_seed(self, dated_seed: Path) -> None: + llm = _CopierLlm() + _run_copilot(dated_seed, llm) + assert llm.calls == 1 + + +# --------------------------------------------------------------------------- +# Finding 61 — ground-truth-only failures are classified as seed mismatches +# --------------------------------------------------------------------------- + + +class TestSeedMismatchFailureClass: + def test_ground_truth_only_failure_is_not_ambiguous_intent(self) -> None: + def drop_column(contract): + contract["exposes"][0]["contract"]["schema"] = contract["exposes"][0]["contract"][ + "schema" + ][:1] + return contract + + llm = _CopierLlm([drop_column]) + with pytest.raises(CopilotGenerationError) as exc_info: + _run_copilot(ODCS_FULL, llm) + assert llm.calls == 3 + assert exc_info.value.context["failure_class"] == "seed_mismatch" + + def test_non_seed_errors_still_classified_normally(self) -> None: + def bad_provider(contract): + contract["exposes"][0]["binding"]["platform"] = "nope" + return contract + + llm = _CopierLlm([bad_provider]) + with pytest.raises(CopilotGenerationError) as exc_info: + _run_copilot(ODCS_FULL, llm) + assert exc_info.value.context["failure_class"] != "seed_mismatch" + + +# --------------------------------------------------------------------------- +# Finding 62 — directory provenance lists the contracts actually consumed +# --------------------------------------------------------------------------- + + +class TestDirectoryProvenance: + def test_subdir_and_json_contracts_are_recorded(self, tmp_path: Path) -> None: + bundle = tmp_path / "subdir" + (bundle / "contracts").mkdir(parents=True) + shutil.copy(next(BUNDLE_DIR.glob("*.odps.yaml")), bundle / "product.odps.yaml") + daily = BUNDLE_DIR / "commerce.orders-product.daily_orders.odcs.yaml" + hourly = BUNDLE_DIR / "commerce.orders-product.hourly_orders.odcs.yaml" + shutil.copy(daily, bundle / "contracts" / "daily.odcs.yaml") + (bundle / "contracts" / "hourly.odcs.json").write_text( + json.dumps(yaml.safe_load(hourly.read_text())) + ) + + seed = load_seed(bundle, allow_remote=False) + origins = [entry["origin"] for entry in seed.provenance] + assert str(bundle / "contracts" / "daily.odcs.yaml") in origins + assert str(bundle / "contracts" / "hourly.odcs.json") in origins + assert seed.provenance[0]["origin"] == str(bundle) + + def test_odcs_only_directory_lists_every_contract(self, tmp_path: Path) -> None: + root = tmp_path / "only" + (root / "nested").mkdir(parents=True) + shutil.copy(ODCS_FULL, root / "nested" / "orders.odcs.yml") + shutil.copy(ODCS_MINIMAL, root / "minimal.yaml") + seed = load_seed(root, allow_remote=False) + origins = [entry["origin"] for entry in seed.provenance] + assert str(root / "nested" / "orders.odcs.yml") in origins + assert str(root / "minimal.yaml") in origins + + +# --------------------------------------------------------------------------- +# Findings 47 / 70 — packaging metadata +# --------------------------------------------------------------------------- + + +def _pyproject() -> Dict[str, Any]: + if sys.version_info >= (3, 11): + import tomllib + else: # pragma: no cover - exercised on 3.9/3.10 CI + tomllib = pytest.importorskip("tomli") + with open(REPO_ROOT / "pyproject.toml", "rb") as fh: + return tomllib.load(fh) + + +def _requirements(specs: List[str]): + from packaging.requirements import Requirement + + return {Requirement(spec).name: Requirement(spec) for spec in specs} + + +class TestPackagingMetadata: + def test_odcs_strict_extra_resolves_on_python_39(self) -> None: + reqs = _requirements(_pyproject()["project"]["optional-dependencies"]["odcs-strict"]) + for name in ("vowl", "open-data-contract-standard"): + marker = reqs[name].marker + assert marker is not None, f"{name} needs a python_version marker" + assert not marker.evaluate({"python_version": "3.9"}) + assert marker.evaluate({"python_version": "3.10"}) + assert marker.evaluate({"python_version": "3.12"}) + # deepdiff supports 3.9 and stays unconditional + assert reqs["deepdiff"].marker is None + + def test_requests_is_a_core_dependency(self) -> None: + reqs = _requirements(_pyproject()["project"]["dependencies"]) + assert "requests" in reqs + assert reqs["requests"].marker is None diff --git a/tests/test_review_fixes_odcs_mappers.py b/tests/test_review_fixes_odcs_mappers.py new file mode 100644 index 00000000..968be5e7 --- /dev/null +++ b/tests/test_review_fixes_odcs_mappers.py @@ -0,0 +1,833 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +"""Regression tests for the ODCS mapper review findings. + +Each test names the finding it pins. Every one of them failed on the branch +head the review ran against (``fa13e55``). +""" + +from __future__ import annotations + +import argparse +import contextlib +import copy +import io +import logging +from pathlib import Path +from typing import Any, Dict, List + +import pytest +import yaml + +from fluid_build.cli.bootstrap import load_contract_with_overlay +from fluid_build.providers.odcs import OdcsProvider +from fluid_build.providers.odcs import validation as odcs_validation +from fluid_build.providers.odcs.mappers import types as odcs_types + +REPO = Path(__file__).resolve().parent.parent +FIXTURES = Path(__file__).parent / "fixtures" +LOG = logging.getLogger(__name__) + + +def _load(path: Path) -> Dict[str, Any]: + with open(path) as f: + return yaml.safe_load(f) + + +def _props(odcs: Dict[str, Any]) -> List[Dict[str, Any]]: + return [p for obj in odcs.get("schema") or [] for p in obj.get("properties") or []] + + +def _fresh(fields: List[Dict[str, Any]], binding: Any = None, **top: Any) -> Dict[str, Any]: + expose: Dict[str, Any] = {"exposeId": "orders", "contract": {"schema": fields}} + if binding is not None: + expose["binding"] = binding + fluid = { + "fluidVersion": "0.7.2", + "kind": "DataProduct", + "id": "acme.orders", + "metadata": {"version": "1.0.0", "status": "active"}, + "exposes": [expose], + } + fluid.update(top) + return fluid + + +def _cli(argv: List[str]) -> Any: + from fluid_build.cli import main as cli_main + + buf = io.StringIO() + try: + with contextlib.redirect_stdout(buf), contextlib.redirect_stderr(buf): + return cli_main(argv) + except SystemExit as exc: + return exc.code + + +# --------------------------------------------------------------------------- +# Findings 6, 13, 19, 21 — binding-level format reaches the ODCS server, so +# every shipped example / template exports valid ODCS. +# --------------------------------------------------------------------------- + + +def _shipped_contracts() -> List[Path]: + found = sorted((REPO / "examples").rglob("*.yaml")) + sorted( + (REPO / "fluid_build" / "templates").glob("*/contract.fluid.yaml") + ) + return [p for p in found if "fluidVersion" in p.read_text()] + + +_SHIPPED = _shipped_contracts() + + +def _assert_valid_odcs_exports(path: Path) -> None: + contract = load_contract_with_overlay(str(path), None, LOG) + for prov in (OdcsProvider(), OdcsProvider(api_version="v3.2.0")): + whole = prov.render(contract) + odcs_validation.validate(whole) # schema picked from its apiVersion + for _eid, doc in prov.render_all_ports(contract): + odcs_validation.validate(doc) + + +@pytest.mark.parametrize("path", _SHIPPED, ids=lambda p: str(p.relative_to(REPO))) +def test_every_shipped_contract_that_validates_exports_valid_odcs(path: Path) -> None: + if _cli(["validate", str(path)]) != 0: + pytest.skip("not a valid FLUID contract (fluid validate fails)") + _assert_valid_odcs_exports(path) + + +@pytest.mark.parametrize( + "path", + sorted((REPO / "fluid_build" / "templates").glob("*/contract.fluid.yaml")), + ids=lambda p: p.parent.name, +) +def test_every_template_exports_valid_odcs(path: Path) -> None: + # Templates are what `fluid init` scaffolds; several fail `fluid + # validate` for reasons unrelated to ODCS (e.g. builds[].pattern), but + # their ODCS export must still be valid. + _assert_valid_odcs_exports(path) + + +def test_local_binding_format_is_emitted_on_the_server() -> None: + fluid = _fresh( + [{"name": "a", "type": "string"}], + binding={"platform": "local", "format": "csv", "location": {"path": "out/a.csv"}}, + ) + server = OdcsProvider().render(fluid)["servers"][0] + assert server["format"] == "csv" + assert server["path"] == "out/a.csv" + + +def test_location_format_wins_over_binding_format() -> None: + fluid = _fresh( + [{"name": "a", "type": "string"}], + binding={ + "platform": "aws", + "format": "parquet", + "delimiter": "new_line", + "location": {"bucket": "b", "path": "p", "format": "json"}, + }, + ) + server = OdcsProvider().render(fluid)["servers"][0] + assert server["format"] == "json" + assert server["delimiter"] == "new_line" + + +@pytest.mark.parametrize( + "contract", + [ + "examples/01-hello-world/contract.fluid.yaml", + "fluid_build/templates/hello-world/contract.fluid.yaml", + ], +) +def test_default_odps_export_succeeds_on_local_contracts(contract: str, tmp_path: Path) -> None: + # Default spec (Bitol) + default strict validation. + rc = _cli(["odps", "export", str(REPO / contract), "--out", str(tmp_path / "p.yaml")]) + assert rc == 0 + + +def test_bitol_strict_render_of_local_contract() -> None: + from fluid_build.providers.odps_standard import BitolOdpsProvider + + contract = load_contract_with_overlay( + str(REPO / "examples/01-hello-world/contract.fluid.yaml"), None, LOG + ) + prov = BitolOdpsProvider() + prov.strict_validation = True + bundle = prov.render(contract) + servers = [s for doc in bundle["contracts"].values() for s in doc.get("servers", [])] + assert servers and all(s.get("format") == "csv" for s in servers) + + +# --------------------------------------------------------------------------- +# Finding 1 — --no-quality / ODCS_INCLUDE_QUALITY=false suppress property quality +# --------------------------------------------------------------------------- + + +def test_no_quality_suppresses_fresh_property_quality() -> None: + fluid = _fresh( + [ + {"name": "id", "type": "string", "required": True, "tags": ["primary-key"]}, + { + "name": "email", + "type": "string", + "validations": [{"type": "pattern", "value": "^.+@.+$"}], + "quality": [{"type": "library", "metric": "rowCount", "mustBeGreaterThan": 0}], + }, + ] + ) + prov = OdcsProvider() + assert any("quality" in p for p in _props(prov.render(fluid))) + prov.include_quality_checks = False + assert not any("quality" in p for p in _props(prov.render(fluid))) + + +def test_no_quality_suppresses_imported_property_quality() -> None: + prov = OdcsProvider() + fluid = prov.import_contract(_load(FIXTURES / "odcs" / "contract-full.yaml")) + prov.include_quality_checks = False + assert not any("quality" in p for p in _props(prov.render(fluid))) + + +def test_no_quality_env_var(monkeypatch) -> None: + monkeypatch.setenv("ODCS_INCLUDE_QUALITY", "false") + fluid = _fresh([{"name": "id", "type": "string", "required": True}]) + assert not any("quality" in p for p in _props(OdcsProvider().render(fluid))) + + +def test_no_quality_cli_flag(tmp_path: Path) -> None: + from fluid_build.cli.odcs import _run_odcs_export + + out = tmp_path / "out.yaml" + args = argparse.Namespace( + contract=str(REPO / "examples/customer360/contract.fluid.yaml"), + output=str(out), + format="yaml", + no_quality=True, + no_sla=False, + ) + assert _run_odcs_export(args) == 0 + assert not any("quality" in p for p in _props(_load(out))) + + +def test_legacy_field_property_shim_respects_no_quality() -> None: + prov = OdcsProvider() + prov.include_quality_checks = False + prop = prov._fluid_field_to_odcs_property({"name": "id", "required": True}, {}) + assert "quality" not in prop + + +# --------------------------------------------------------------------------- +# Finding 2 — no binding.platform invented from SchemaObject.physicalType +# --------------------------------------------------------------------------- + + +def _postgres_odcs() -> Dict[str, Any]: + return { + "version": "1.0.0", + "apiVersion": "v3.1.0", + "kind": "DataContract", + "id": "pg.customers", + "status": "active", + "servers": [ + { + "server": "customers", + "type": "postgres", + "host": "db", + "port": 5432, + "database": "crm", + "schema": "public", + } + ], + "schema": [ + { + "name": "customers", + "physicalType": "table", + "properties": [{"name": "id", "logicalType": "integer", "physicalType": "int4"}], + } + ], + } + + +def test_import_derives_binding_from_the_server_not_physical_type() -> None: + fluid = OdcsProvider().import_contract(_postgres_odcs()) + binding = fluid["exposes"][0]["binding"] + assert binding["platform"] == "postgres" + assert binding["location"] == { + "host": "db", + "port": 5432, + "database": "crm", + "schema": "public", + } + + +def test_import_without_servers_has_no_binding() -> None: + odcs = _postgres_odcs() + del odcs["servers"] + fluid = OdcsProvider().import_contract(odcs) + assert "binding" not in fluid["exposes"][0] + + +def test_import_with_ambiguous_servers_has_no_binding() -> None: + odcs = _postgres_odcs() + # Two schema-valid servers, neither named after the SchemaObject: which + # one backs it is ambiguous, so no binding may be guessed. + odcs["servers"] = [ + { + "server": "a", + "type": "postgres", + "host": "h", + "port": 5432, + "database": "crm", + "schema": "public", + }, + {"server": "b", "type": "snowflake", "account": "x", "database": "d", "schema": "s"}, + ] + fluid = OdcsProvider().import_contract(odcs) + assert "binding" not in fluid["exposes"][0] + + +def test_field_added_to_imported_postgres_contract_gets_no_bigquery_type() -> None: + prov = OdcsProvider() + fluid = prov.import_contract(_postgres_odcs()) + fluid["exposes"][0]["contract"]["schema"].append({"name": "new_col", "type": "integer"}) + props = {p["name"]: p for p in _props(prov.render(fluid))} + assert props["id"]["physicalType"] == "int4" + assert props["new_col"].get("physicalType") != "INT64" + + +def test_odps_bundle_import_then_strict_export_succeeds(tmp_path: Path) -> None: + from fluid_build.providers.odps_standard import BitolOdpsProvider + + prov = BitolOdpsProvider() + fluid = prov.import_directory(FIXTURES / "odps" / "product-bitol") + platforms = {e["exposeId"]: (e.get("binding") or {}).get("platform") for e in fluid["exposes"]} + assert platforms == {"daily_orders": "gcp", "hourly_orders": "kafka"} + prov.strict_validation = True + bundle = prov.render(fluid) # raised "'project' is a required property" before + for doc in bundle["contracts"].values(): + odcs_validation.validate(doc) + + +# --------------------------------------------------------------------------- +# Finding 3 — array-form team and explicit empty slaProperties round-trip +# --------------------------------------------------------------------------- + + +def _minimal(api: str = "v3.1.0", **extra: Any) -> Dict[str, Any]: + doc = { + "version": "1.0.0", + "apiVersion": api, + "kind": "DataContract", + "id": "t", + "status": "active", + "schema": [{"name": "t", "properties": [{"name": "a", "logicalType": "string"}]}], + } + doc.update(extra) + return doc + + +_ARRAY_TEAM = [ + {"username": "ceastwood", "role": "Data Scientist", "dateIn": "2022-08-02"}, + {"username": "mhopper", "role": "Data Steward"}, +] + + +@pytest.mark.parametrize("api", ["v3.0.2", "v3.1.0"]) +def test_array_team_round_trips_and_projects_owner(api: str) -> None: + odcs = _minimal(api, team=copy.deepcopy(_ARRAY_TEAM)) + prov = OdcsProvider() + fluid = prov.import_contract(odcs) + assert fluid["owner"]["email"] == "ceastwood" + assert fluid["owner"]["contacts"] == [{"email": "mhopper", "role": "Data Steward"}] + assert prov.roundtrip_check(odcs)["equal"] + + +@pytest.mark.parametrize("team", [[], {}]) +def test_empty_team_round_trips(team: Any) -> None: + odcs = _minimal(team=team) + assert OdcsProvider().roundtrip_check(odcs)["equal"] + + +def test_empty_sla_properties_round_trip() -> None: + odcs = _minimal(slaProperties=[]) + assert OdcsProvider().roundtrip_check(odcs)["equal"] + + +# --------------------------------------------------------------------------- +# Finding 22 — `fluid odcs validate` selects the schema from apiVersion +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "fixture", + [ + FIXTURES / "upstream" / "odcs" / "full-example.v3.2.0.odcs.yaml", + FIXTURES / "odcs" / "contract-full.yaml", + ], + ids=["v3.2.0", "v3.1.0"], +) +def test_argparse_odcs_validate_is_version_selected(fixture: Path, monkeypatch) -> None: + from fluid_build.cli.odcs import _run_odcs_validate + + monkeypatch.delenv("ODCS_API_VERSION", raising=False) + assert _run_odcs_validate(argparse.Namespace(odcs_file=str(fixture))) == 0 + assert _cli(["odcs", "validate", str(fixture)]) == 0 + + +def test_argparse_odcs_validate_still_rejects_invalid(tmp_path: Path) -> None: + from fluid_build.cli.odcs import _run_odcs_validate + + bad = tmp_path / "bad.yaml" + bad.write_text(yaml.safe_dump({"apiVersion": "v3.2.0", "kind": "DataContract"})) + assert _run_odcs_validate(argparse.Namespace(odcs_file=str(bad))) == 1 + + +def test_legacy_validate_shim_is_version_selected() -> None: + doc = _load(FIXTURES / "upstream" / "odcs" / "full-example.v3.2.0.odcs.yaml") + OdcsProvider()._validate_odcs(doc) # raised "'v3.2.0' is not one of [...]" + + +# --------------------------------------------------------------------------- +# Findings 26 / 51 — v3.2.0-only logical types and server engines +# --------------------------------------------------------------------------- + + +def _map_vector_fluid() -> Dict[str, Any]: + return _fresh([{"name": "attrs", "type": "map"}, {"name": "emb", "type": "vector"}]) + + +def test_map_vector_fall_back_below_v320() -> None: + out = OdcsProvider().render(_map_vector_fluid()) + assert out["apiVersion"] == "v3.1.0" + types = {p["name"]: p["logicalType"] for p in _props(out)} + assert types == {"attrs": "object", "emb": "array"} + odcs_validation.validate(out) + + +def test_map_vector_emitted_at_v320() -> None: + out = OdcsProvider(api_version="v3.2.0").render(_map_vector_fluid()) + props = {p["name"]: p for p in _props(out)} + assert props["attrs"]["logicalType"] == "map" + assert "map" in props["attrs"] + assert props["emb"]["logicalType"] == "vector" + odcs_validation.validate(out) + + +def test_logical_type_helper_is_version_aware() -> None: + assert odcs_types.fluid_to_logical("map") == "object" + assert odcs_types.fluid_to_logical("vector", "v3.1.0") == "array" + assert odcs_types.fluid_to_logical("map", "v3.2.0") == "map" + + +@pytest.mark.parametrize("platform", ["iceberg", "teradata", "hana", "exasol", "actian"]) +def test_v320_engines_map_to_custom_below_v320(platform: str) -> None: + fluid = _fresh( + [{"name": "a", "type": "string"}], + binding={ + "platform": platform, + "location": { + "catalog": "c", + "catalogUrl": "https://c/v1", + "namespace": "n", + "host": "h", + "database": "d", + }, + }, + ) + out = OdcsProvider().render(fluid) + assert out["servers"][0]["type"] == "custom" + odcs_validation.validate(out) + out32 = OdcsProvider(api_version="v3.2.0").render(fluid) + assert out32["servers"][0]["type"] != "custom" + odcs_validation.validate(out32) + + +# --------------------------------------------------------------------------- +# Finding 28 — no aliasing between stash, FLUID, input and output +# --------------------------------------------------------------------------- + + +_UPSTREAM_V320 = FIXTURES / "upstream" / "odcs" / "full-example.v3.2.0.odcs.yaml" + + +def test_imported_fluid_dumps_without_yaml_anchors() -> None: + fluid = OdcsProvider().import_contract(_load(_UPSTREAM_V320)) + dumped = yaml.dump(fluid, default_flow_style=False, sort_keys=False) + assert "&id" not in dumped and "*id" not in dumped + + +def test_cli_import_edit_export_after_removing_a_quality_rule(tmp_path: Path) -> None: + from fluid_build.cli.odcs import _run_odcs_import + + imported = tmp_path / "imported.fluid.yaml" + args = argparse.Namespace(odcs_file=str(_UPSTREAM_V320), output=str(imported), format="yaml") + assert _run_odcs_import(args) == 0 + text = imported.read_text() + # An anchor on a field's quality item aliased from the stash meant that + # deleting the rule in an editor left a dangling ``*id001``. + assert "&id" not in text and "*id" not in text + fluid = yaml.safe_load(text) + for expose in fluid["exposes"]: + for fld in expose["contract"]["schema"]: + fld["quality"] = [] + edited = tmp_path / "edited.fluid.yaml" + edited.write_text(yaml.dump(fluid, sort_keys=False)) + out = tmp_path / "out.yaml" + from fluid_build.cli.odcs import _run_odcs_export + + export_args = argparse.Namespace( + contract=str(edited), output=str(out), format="yaml", no_quality=False, no_sla=False + ) + assert _run_odcs_export(export_args) == 0 + assert not any(p.get("quality") for p in _props(_load(out))) + + +def test_mutating_an_export_does_not_leak_into_later_exports_or_input() -> None: + source = _load(_UPSTREAM_V320) + pristine = copy.deepcopy(source) + prov = OdcsProvider() + fluid = prov.import_contract(source) + fluid_before = copy.deepcopy(fluid) + + out1 = prov.render(fluid) + out1["description"]["usage"] = "MUTATED" + out1["schema"][0]["properties"][0].setdefault("quality", []).append({"type": "text"}) + out1["team"]["members"][0]["role"] = "MUTATED" + out1["servers"][0]["type"] = "MUTATED" + out1["slaProperties"][0]["value"] = "MUTATED" + + out2 = prov.render(fluid) + assert out2["description"]["usage"] != "MUTATED" + assert prov.roundtrip_check(pristine)["equal"] + assert odcs_validation.roundtrip_check(pristine, out2)["equal"] + assert fluid == fluid_before + assert source == pristine + + +def test_import_does_not_alias_the_callers_input() -> None: + source = _load(_UPSTREAM_V320) + pristine = copy.deepcopy(source) + fluid = OdcsProvider().import_contract(source) + for expose in fluid["exposes"]: + for fld in expose["contract"]["schema"]: + for item in fld.get("quality") or []: + item["MUTATED"] = True + assert source == pristine + + +# --------------------------------------------------------------------------- +# Finding 29 — FLUID edits to owner / expects / qos win over the stash +# --------------------------------------------------------------------------- + + +@pytest.fixture +def full_import(): + prov = OdcsProvider() + return prov, prov.import_contract(_load(FIXTURES / "odcs" / "contract-full.yaml")) + + +def test_owner_edit_reaches_team(full_import) -> None: + prov, fluid = full_import + fluid["owner"]["team"] = "new-team" + fluid["owner"]["email"] = "carol@acme.com" + out = prov.render(fluid) + assert out["team"]["name"] == "new-team" + assert out["team"]["members"][0]["username"] == "carol@acme.com" + assert out["team"]["members"][0]["dateIn"] == "2024-01-01" # same member, new email + # unmodelled stashed keys survive + assert out["team"]["description"] == "Owners of the commerce domain data" + assert out["team"]["members"][1] == { + "username": "bob@acme.com", + "name": "Bob", + "role": "steward", + } + odcs_validation.validate(out) + + +def test_contact_edit_keeps_member_extras(full_import) -> None: + prov, fluid = full_import + fluid["owner"]["contacts"][0]["role"] = "owner" + out = prov.render(fluid) + assert out["team"]["members"][0]["dateIn"] == "2024-01-01" + assert out["team"]["members"][1]["role"] == "owner" + + +def test_owner_removal_drops_team(full_import) -> None: + prov, fluid = full_import + del fluid["owner"] + assert "team" not in prov.render(fluid) + + +def test_expects_edits_reach_servers(full_import) -> None: + prov, fluid = full_import + fluid["expects"][0]["location"]["dataset"] = "orders_v2" + fluid["expects"].append( + { + "id": "sf_raw", + "provider": "snowflake", + "location": {"account": "acme", "database": "RAW", "schema": "S"}, + } + ) + out = prov.render(fluid) + servers = {s["server"]: s for s in out["servers"]} + assert servers["bq-prod"] == { + "server": "bq-prod", + "type": "bigquery", + "project": "acme-prod", + "dataset": "orders_v2", + } + assert servers["sf_raw"]["type"] == "snowflake" + assert servers["sf_raw"]["account"] == "acme" + odcs_validation.validate(out) + + +def test_expect_removal_drops_server(full_import) -> None: + # The server is dropped only when neither its expect nor an expose + # binding owns it any more: exposes[0].binding is derived from it, and a + # binding that still selects it keeps it (round-2 defect 15). + prov, fluid = full_import + fluid["expects"] = [] + del fluid["exposes"][0]["binding"] + assert prov.render(fluid)["servers"] == [] + + +def test_expose_binding_edit_reaches_its_server(full_import) -> None: + prov, fluid = full_import + assert fluid["exposes"][0]["binding"]["location"]["dataset"] == "orders" + fluid["exposes"][0]["binding"]["location"]["dataset"] = "orders_gold" + out = prov.render(fluid) + assert [s["dataset"] for s in out["servers"]] == ["orders_gold"] + + +def test_qos_edits_reach_sla_properties(full_import) -> None: + prov, fluid = full_import + qos = fluid["exposes"][0]["qos"] + assert qos["availability"] == "99%" + qos["availability"] = "50%" + qos["freshnessSLO"] = "PT1H" + qos["labels"] = {"tier": "gold"} + out = prov.render(fluid) + sla = {p["property"]: p for p in out["slaProperties"]} + assert sla["availability"]["value"] == 0.5 + assert sla["interval"]["value"] == "PT1H" + assert sla["retention"] == {"property": "retention", "value": "365", "unit": "d"} + assert sla["label:tier"]["value"] == "gold" + odcs_validation.validate(out) + + +def test_untouched_import_still_round_trips(full_import) -> None: + prov, fluid = full_import + original = _load(FIXTURES / "odcs" / "contract-full.yaml") + assert odcs_validation.roundtrip_check(original, prov.render(fluid))["equal"] + + +# --------------------------------------------------------------------------- +# Finding 30 — SchemaObjects without properties survive round-trip +# --------------------------------------------------------------------------- + + +def test_schema_objects_without_properties_round_trip() -> None: + odcs = _minimal() + odcs["schema"] = [ + {"name": "documented_only", "physicalType": "table", "description": "doc"}, + {"name": "empty_props", "properties": []}, + {"name": "t", "properties": [{"name": "a", "logicalType": "string"}]}, + ] + odcs_validation.validate(odcs) + assert OdcsProvider().roundtrip_check(odcs)["equal"] + + +def test_fresh_expose_without_fields_is_still_skipped() -> None: + fluid = _fresh([]) + assert OdcsProvider().render(fluid)["schema"] == [] + + +# --------------------------------------------------------------------------- +# Finding 31 — common FLUID numeric/timestamp types +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "fluid_type, logical, bigquery, snowflake", + [ + ("number", "number", "FLOAT64", "NUMBER"), + ("FLOAT64", "number", "FLOAT64", "FLOAT"), + ("INT64", "integer", "INT64", "NUMBER"), + ("smallint", "integer", "INT64", "NUMBER"), + ("TIMESTAMP_NTZ", "timestamp", "DATETIME", "TIMESTAMP_NTZ"), + ], +) +def test_numeric_and_timestamp_types(fluid_type, logical, bigquery, snowflake) -> None: + assert odcs_types.fluid_to_logical(fluid_type) == logical + assert odcs_types.fluid_to_physical(fluid_type, "gcp") == bigquery + assert odcs_types.fluid_to_physical(fluid_type, "snowflake") == snowflake + fluid = _fresh([{"name": "amt", "type": fluid_type}], binding={"platform": "gcp"}) + prop = _props(OdcsProvider().render(fluid))[0] + assert (prop["logicalType"], prop["physicalType"]) == (logical, bigquery) + + +# --------------------------------------------------------------------------- +# Finding 32 — top-level FLUID name/description/tags/domain are exported +# --------------------------------------------------------------------------- + + +def test_top_level_fluid_fields_are_exported() -> None: + fluid = _fresh( + [{"name": "a", "type": "string"}], + name="Customer 360", + description="All about customers", + domain="Customer", + tags=["crm"], + lifecycle={"state": "deprecated"}, + ) + out = OdcsProvider().render(fluid) + assert out["name"] == "Customer 360" + assert out["description"] == {"purpose": "All about customers"} + assert out["domain"] == "Customer" + assert out["tags"] == ["crm"] + odcs_validation.validate(out) + + +def test_metadata_fields_still_win_over_top_level() -> None: + fluid = _fresh([{"name": "a", "type": "string"}], name="top") + fluid["metadata"]["name"] = "meta" + assert OdcsProvider().render(fluid)["name"] == "meta" + + +def test_customer360_example_exports_name_and_domain() -> None: + contract = load_contract_with_overlay( + str(REPO / "examples/customer360/contract.fluid.yaml"), None, LOG + ) + out = OdcsProvider().render(contract) + assert out["name"] == contract["name"] + assert out["domain"] == contract["domain"] + + +# --------------------------------------------------------------------------- +# Finding 33 — fresh field quality keeps auto-checks and normalises items +# --------------------------------------------------------------------------- + + +def test_fresh_custom_quality_is_merged_with_auto_checks() -> None: + fld = { + "name": "id", + "required": True, + "tags": ["primary-key"], + "quality": [{"description": "IDs must match CRM"}], + } + quality = OdcsProvider()._extract_field_quality(fld) + assert [q.get("metric") for q in quality[:2]] == ["nullValues", "duplicateValues"] + assert quality[2] == {"type": "text", "description": "IDs must match CRM"} + + +def test_fresh_non_odcs_quality_items_are_dropped() -> None: + fluid = _fresh( + [ + { + "name": "amt", + "type": "number", + "required": True, + "quality": [{"rule": "positive", "severity": "high"}], + } + ] + ) + out = OdcsProvider().render(fluid) + assert [q["metric"] for q in _props(out)[0]["quality"]] == ["nullValues"] + odcs_validation.validate(out) + + +# --------------------------------------------------------------------------- +# Finding 52 — removing the FLUID description keeps the rest of the object +# --------------------------------------------------------------------------- + + +def test_removing_description_purpose_keeps_usage_and_limitations() -> None: + prov = OdcsProvider() + fluid = prov.import_contract(_load(_UPSTREAM_V320)) + del fluid["metadata"]["description"] + desc = prov.render(fluid)["description"] + assert "purpose" not in desc + assert desc["usage"] == "Predict sales over time" + assert desc["authoritativeDefinitions"] + + +# --------------------------------------------------------------------------- +# Finding 53 — a type change drops stale physicalType / logicalTypeOptions +# --------------------------------------------------------------------------- + + +def test_type_change_drops_stale_type_details() -> None: + odcs = _minimal() + odcs["schema"][0]["properties"] = [ + { + "name": "qty", + "logicalType": "integer", + "physicalType": "INT64", + "logicalTypeOptions": {"minimum": 0}, + } + ] + prov = OdcsProvider() + fluid = prov.import_contract(odcs) + fluid["exposes"][0]["contract"]["schema"][0]["type"] = "string" + out = prov.render(fluid) + prop = _props(out)[0] + assert prop["logicalType"] == "string" + assert "logicalTypeOptions" not in prop + assert prop.get("physicalType") != "INT64" + odcs_validation.validate(out) + + +def test_unchanged_type_keeps_type_details() -> None: + odcs = _minimal() + odcs["schema"][0]["properties"] = [ + { + "name": "qty", + "logicalType": "integer", + "physicalType": "INT64", + "logicalTypeOptions": {"minimum": 0}, + } + ] + assert OdcsProvider().roundtrip_check(odcs)["equal"] + + +# --------------------------------------------------------------------------- +# Finding 65 — fresh servers keep a StableId ``id`` +# --------------------------------------------------------------------------- + + +def test_fresh_servers_carry_a_stable_id() -> None: + fluid = _fresh( + [{"name": "a", "type": "string"}], + binding={"platform": "gcp", "location": {"project": "p", "dataset": "d"}}, + ) + fluid["exposes"][0]["exposeId"] = "orders.v1 daily" + fluid["expects"] = [{"id": "raw_events", "provider": "kafka", "location": {"host": "k"}}] + for api in ("v3.1.0", "v3.2.0"): + out = OdcsProvider(api_version=api).render(fluid) + assert [(s["id"], s["server"]) for s in out["servers"]] == [ + ("orders_v1_daily", "orders.v1 daily"), + ("raw_events", "raw_events"), + ] + odcs_validation.validate(out) + + +def test_validation_accepts_yaml_dates_as_iso_strings() -> None: + import datetime + + from fluid_build.providers.odps_standard import validation as odps_validation + + doc = yaml.safe_load( + (Path(__file__).parent / "fixtures" / "odcs" / "contract-minimal.yaml").read_text() + ) + doc["slaProperties"] = [{"property": "effectiveFrom", "value": datetime.date(2024, 1, 1)}] + before = copy.deepcopy(doc) + odcs_validation.validate(doc) # does not raise + assert doc == before # never mutates the caller's document + assert odcs_validation.json_instance(doc)["slaProperties"][0]["value"] == "2024-01-01" + assert odps_validation.json_instance is odcs_validation.json_instance diff --git a/tests/test_review_fixes_odps_roundtrip.py b/tests/test_review_fixes_odps_roundtrip.py new file mode 100644 index 00000000..eb8d64d9 --- /dev/null +++ b/tests/test_review_fixes_odps_roundtrip.py @@ -0,0 +1,941 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Regression tests for the ODPS / ODCS round-trip review findings. + +Covers the Bitol bundle import → export path (per-port ODCS state, the +contractId linking invariant, same-name ports), the ODPS v1.0.0 downgrade, +the legacy output-port custom properties, strict ODCS import, resolver +safety (id match, path traversal, remote limits) and bundle-export paths. +""" + +from __future__ import annotations + +import copy +import http.server +import json +import os +import threading +import time +from pathlib import Path +from typing import Any, Dict, Iterator + +import pytest +import yaml + +from fluid_build.providers.base import ProviderError +from fluid_build.providers.odcs import OdcsProvider +from fluid_build.providers.odcs.validation import roundtrip_check +from fluid_build.providers.odps_standard import BitolOdpsProvider, OdpsStandardProvider +from fluid_build.providers.odps_standard.resolver import ( + ContractNotFound, + ContractResolver, +) +from fluid_build.providers.odps_standard.validation import validate as validate_odps + +FIXTURES = Path(__file__).parent / "fixtures" +BUNDLE_DIR = FIXTURES / "odps" / "product-bitol" +UPSTREAM_ODPS = sorted((FIXTURES / "upstream" / "odps").glob("*.odps.yaml")) +MULTI_EXPOSE_FLUID = FIXTURES / "fluid" / "contract-multi-expose.fluid.yaml" + +ORDERS_ID = "3f2b1c4e-0000-4000-8000-000000000001" +EVENTS_ID = "shop-events-contract" + + +def _load(path: Path) -> Any: + with open(path) as f: + return yaml.safe_load(f) + + +def _write(path: Path, doc: Any) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(yaml.safe_dump(doc, sort_keys=False)) + return path + + +def _odps_doc() -> Dict[str, Any]: + return { + "apiVersion": "v1.1.0", + "kind": "DataProduct", + "id": "shop", + "name": "Shop", + "version": "3.0.0", + "status": "active", + "type": "sourceAligned", + "outputPorts": [ + # No description: the ODCS SchemaObject's must not leak onto it. + {"name": "orders", "version": "2.0.0", "contractId": ORDERS_ID, "type": "postgres"}, + { + "name": "events", + "version": "1.0.0", + "contractId": EVENTS_ID, + "description": "Event stream", + }, + ], + } + + +def _orders_odcs() -> Dict[str, Any]: + """ODCS v3.2.0 with a postgres server, document-level fields and v3.2.0 enum.""" + return { + "apiVersion": "v3.2.0", + "kind": "DataContract", + "id": ORDERS_ID, + "version": "2.3.0", + "status": "draft", + "name": "Orders contract", + "description": {"purpose": "Orders", "limitations": "None known", "usage": "Reporting"}, + "customProperties": [{"property": "ownerTeam", "value": "sales"}], + "authoritativeDefinitions": [ + {"type": "businessDefinition", "url": "https://example.com/orders"} + ], + "schema": [ + { + "name": "tbl_orders", + "businessName": "Orders table", + "physicalType": "table", + "description": "The orders table (from ODCS)", + "logicalType": "object", + "properties": [ + { + "name": "order_id", + "logicalType": "string", + "physicalType": "varchar(36)", + "required": True, + }, + { + "name": "state", + "logicalType": "string", + "enum": [{"value": "open"}, {"value": "closed"}], + }, + ], + } + ], + "servers": [ + { + "server": "prod", + "type": "postgres", + "host": "db.example.com", + "port": 5432, + "database": "shop", + "schema": "public", + } + ], + "slaProperties": [{"property": "latency", "value": 4, "unit": "d"}], + } + + +def _events_odcs() -> Dict[str, Any]: + """ODCS v3.1.0 with a team, a kafka server and two SchemaObjects.""" + return { + "apiVersion": "v3.1.0", + "kind": "DataContract", + "id": EVENTS_ID, + "version": "1.0.0", + "status": "active", + "name": "Events", + "team": { + "name": "streaming", + "members": [{"username": "a@example.com", "role": "owner"}], + }, + "schema": [ + { + "name": "clicks", + "physicalType": "topic", + "logicalType": "object", + "properties": [{"name": "click_id", "logicalType": "string"}], + }, + { + "name": "views", + "physicalType": "topic", + "logicalType": "object", + "properties": [{"name": "view_id", "logicalType": "string"}], + }, + ], + "servers": [{"server": "events", "type": "kafka", "host": "kafka:9092"}], + } + + +@pytest.fixture +def hand_bundle(tmp_path: Path) -> Path: + bundle = tmp_path / "bundle" + _write(bundle / "shop.odps.yaml", _odps_doc()) + _write(bundle / f"{ORDERS_ID}.odcs.yaml", _orders_odcs()) + _write(bundle / f"{EVENTS_ID}.odcs.yaml", _events_odcs()) + return bundle + + +def _strict_provider() -> BitolOdpsProvider: + prov = BitolOdpsProvider() + prov.strict_validation = True + return prov + + +def _via_yaml(fluid: Dict[str, Any]) -> Dict[str, Any]: + """Mimic `fluid opds import -o f.yaml` + `fluid opds export f.yaml`.""" + return yaml.safe_load(yaml.safe_dump(fluid, sort_keys=False)) + + +def _expose(fluid: Dict[str, Any], expose_id: str) -> Dict[str, Any]: + return next(e for e in fluid["exposes"] if (e.get("exposeId") or e.get("id")) == expose_id) + + +# --------------------------------------------------------------------------- +# Findings 0 / 17 / 23 / 24 (+54): per-port ODCS state survives the bundle +# --------------------------------------------------------------------------- + + +class TestBundleRoundTripIsLosslessPerPort: + @pytest.mark.parametrize("entry", ["directory", "file"]) + def test_repo_fixture_bundle_round_trips_every_document_strictly(self, entry: str) -> None: + prov = _strict_provider() + if entry == "directory": + fluid = prov.import_directory(BUNDLE_DIR) + else: + fluid = prov.import_contract(next(BUNDLE_DIR.glob("*.odps.yaml"))) + bundle = prov.render(_via_yaml(fluid)) # strict validation runs inside render + + assert bundle["product"] == _load(BUNDLE_DIR / "commerce.orders-product.odps.yaml") + on_disk = sorted(p.name for p in BUNDLE_DIR.glob("*.odcs.yaml")) + assert sorted(f"{cid}.odcs.yaml" for cid in bundle["contracts"]) == on_disk + for cid, odcs in bundle["contracts"].items(): + assert odcs == _load(BUNDLE_DIR / f"{cid}.odcs.yaml"), cid + + def test_hand_built_bundle_round_trips_every_document_strictly(self, hand_bundle: Path) -> None: + prov = _strict_provider() + bundle = prov.render(_via_yaml(prov.import_directory(hand_bundle))) + assert roundtrip_check(_odps_doc(), bundle["product"])["equal"] + assert bundle["product"] == _odps_doc() + assert bundle["contracts"][ORDERS_ID] == _orders_odcs() + assert bundle["contracts"][EVENTS_ID] == _events_odcs() + # apiVersion is per-document and preserved (v3.2.0 stays v3.2.0). + assert bundle["contracts"][ORDERS_ID]["apiVersion"] == "v3.2.0" + assert bundle["contracts"][EVENTS_ID]["apiVersion"] == "v3.1.0" + + def test_written_bundle_matches_sources(self, hand_bundle: Path, tmp_path: Path) -> None: + prov = _strict_provider() + out = tmp_path / "out" + prov.render(prov.import_directory(hand_bundle), out_dir=out) + for src in hand_bundle.iterdir(): + assert _load(out / src.name) == _load(src), src.name + + def test_fluid_origin_bundle_round_trips_zero_diff(self, tmp_path: Path) -> None: + prov = _strict_provider() + first = prov.render(_load(MULTI_EXPOSE_FLUID), out_dir=tmp_path / "a") + again = prov.render(_via_yaml(prov.import_directory(tmp_path / "a"))) + assert again["product"] == first["product"] + assert again["contracts"] == first["contracts"] + + def test_import_keeps_port_state_on_the_expose(self) -> None: + fluid = BitolOdpsProvider().import_directory(BUNDLE_DIR) + daily = _expose(fluid, "daily_orders") + doc = daily["odcs_passthrough"]["_document"] + doc_pt = doc["metadata"]["odcs_passthrough"] + assert doc_pt["_verbatim"]["apiVersion"] == "v3.1.0" + assert doc_pt["servers"][0]["project"] == "acme-prod" + assert daily["odcs_passthrough"]["_verbatim"]["name"] == "daily_orders" + # FLUID-native binding comes from the port's own ODCS server. + assert daily["binding"] == { + "platform": "gcp", + "location": {"project": "acme-prod", "dataset": "orders"}, + } + # The input port gets back the upstream server the contract describes. + customers = next(e for e in fluid["expects"] if e["id"] == "customers_raw") + assert customers["location"]["account"] == "company-prod" + + def test_port_description_is_not_invented_from_odcs(self, hand_bundle: Path) -> None: + # Finding 54: the port had no description; the SchemaObject had one. + prov = _strict_provider() + fluid = prov.import_directory(hand_bundle) + assert "description" not in _expose(fluid, "orders") + bundle = prov.render(fluid) + assert "description" not in bundle["product"]["outputPorts"][0] + schema_object = bundle["contracts"][ORDERS_ID]["schema"][0] + assert schema_object["description"] == "The orders table (from ODCS)" + + def test_expose_edits_still_reach_the_per_port_odcs(self, hand_bundle: Path) -> None: + prov = _strict_provider() + fluid = prov.import_directory(hand_bundle) + orders = _expose(fluid, "orders") + orders["contract"]["schema"][0]["description"] = "Primary key" + orders["description"] = "All orders" + bundle = prov.render(fluid) + odcs = bundle["contracts"][ORDERS_ID] + assert odcs["schema"][0]["properties"][0]["description"] == "Primary key" + assert odcs["schema"][0]["description"] == "All orders" + assert bundle["product"]["outputPorts"][0]["description"] == "All orders" + # Everything else is still the source document. + assert odcs["servers"] == _orders_odcs()["servers"] + assert odcs["version"] == "2.3.0" + + +# --------------------------------------------------------------------------- +# Findings 4 / 16 / 25: per-port ODCS id follows the port's real contractId +# --------------------------------------------------------------------------- + + +class TestLinkingInvariant: + def test_odcs_render_accepts_contract_id(self) -> None: + fluid = _load(MULTI_EXPOSE_FLUID) + odcs = OdcsProvider().render(fluid, expose_id="daily_orders", contract_id="custom-id") + assert odcs["id"] == "custom-id" + default = OdcsProvider().render(fluid, expose_id="daily_orders") + assert default["id"] == "commerce.orders-product.daily_orders" + + @pytest.mark.parametrize("path", UPSTREAM_ODPS, ids=lambda p: p.name) + def test_upstream_examples_re_export(self, path: Path) -> None: + prov = BitolOdpsProvider() + prov.strict_validation = False # their contracts are not vendored + fluid = prov.import_contract(path, allow_remote=False, lenient=True) + bundle = prov.render(_via_yaml(fluid)) + source = _load(path) + assert bundle["product"] == source + expected = [p["contractId"] for p in source["outputPorts"] if p.get("contractId")] + assert list(bundle["contracts"]) == expected + for cid, odcs in bundle["contracts"].items(): + assert odcs["id"] == cid + + def test_uuid_contract_ids_survive_renames(self, hand_bundle: Path) -> None: + prov = _strict_provider() + fluid = prov.import_directory(hand_bundle) + renamed = _expose(fluid, "orders") + renamed["id"] = renamed["exposeId"] = "orders_v2" + fluid["contract"]["id"] = "shop2" + bundle = prov.render(fluid) + port = bundle["product"]["outputPorts"][0] + assert port["name"] == "orders_v2" + assert port["contractId"] == ORDERS_ID + assert bundle["contracts"][ORDERS_ID]["id"] == ORDERS_ID + + def test_port_without_contract_id_does_not_crash(self) -> None: + odps = { + "apiVersion": "v1.1.0", + "kind": "DataProduct", + "id": "p2", + "name": "P2", + "outputPorts": [{"name": "orders"}, {"name": "other", "contractId": "p2.other"}], + } + prov = BitolOdpsProvider() + prov.strict_validation = False + fluid = prov.import_contract(odps, allow_remote=False, lenient=True) + bundle = prov.render(fluid) + assert bundle["product"] == odps # contractId stays absent + assert list(bundle["contracts"]) == ["p2.other"] + + +# --------------------------------------------------------------------------- +# Finding 36: same-name ports that differ only in version +# --------------------------------------------------------------------------- + + +class TestSameNamePorts: + def _bundle(self, root: Path) -> Path: + odps = { + "apiVersion": "v1.1.0", + "kind": "DataProduct", + "id": "shop", + "name": "Shop", + "status": "active", + "outputPorts": [ + {"name": "orders", "version": "1.0.0", "contractId": "shop.orders.v1"}, + {"name": "orders", "version": "2.0.0", "contractId": "shop.orders.v2"}, + ], + } + _write(root / "shop.odps.yaml", odps) + for cid, col in (("shop.orders.v1", "id_v1"), ("shop.orders.v2", "id_v2")): + _write( + root / f"{cid}.odcs.yaml", + { + "apiVersion": "v3.1.0", + "kind": "DataContract", + "id": cid, + "version": "1.0.0", + "status": "active", + "schema": [ + { + "name": "orders", + "logicalType": "object", + "properties": [{"name": col, "logicalType": "string"}], + } + ], + }, + ) + return root + + def test_each_port_gets_its_own_contract_and_re_exports(self, tmp_path: Path) -> None: + root = self._bundle(tmp_path / "b") + prov = _strict_provider() + fluid = prov.import_directory(root) + ids = [e["id"] for e in fluid["exposes"]] + assert ids == ["orders@1.0.0", "orders@2.0.0"] + cols = [e["contract"]["schema"][0]["name"] for e in fluid["exposes"]] + assert cols == ["id_v1", "id_v2"] + bundle = prov.render(_via_yaml(fluid)) + assert bundle["product"] == _load(root / "shop.odps.yaml") + for cid in ("shop.orders.v1", "shop.orders.v2"): + assert bundle["contracts"][cid] == _load(root / f"{cid}.odcs.yaml") + + +# --------------------------------------------------------------------------- +# Finding 5 (provider part): a v1.0.0 target never carries v1.1.0-only keys +# --------------------------------------------------------------------------- + + +class TestDowngradeToV100: + def test_imported_v110_product_rendered_at_v100_is_valid(self) -> None: + odps = { + "apiVersion": "v1.1.0", + "kind": "DataProduct", + "id": "p11", + "name": "P11", + "status": "active", + "type": "sourceAligned", + "deprecated": False, + "synonyms": [{"synonym": "Product eleven"}], + "outputPorts": [ + { + "id": "port-1", + "name": "orders", + "contractId": "p11.orders", + "deprecated": False, + "customProperties": [{"id": "cp1", "property": "tier", "value": "gold"}], + } + ], + "inputPorts": [{"name": "raw", "version": "1.0.0"}], + } + fluid = BitolOdpsProvider().import_contract(odps, allow_remote=False, lenient=True) + prov = BitolOdpsProvider(api_version="v1.0.0") + prov.strict_validation = False # the per-port ODCS is an unresolved stub + product = prov.render(fluid)["product"] + assert product["apiVersion"] == "v1.0.0" + for key in ("type", "deprecated", "context", "synonyms"): + assert key not in product + custom = {c["property"]: c["value"] for c in product["customProperties"]} + assert custom == { + "type": "sourceAligned", + "deprecated": False, + "synonyms": [{"synonym": "Product eleven"}], + } + port = product["outputPorts"][0] + assert not {"id", "deprecated"} & set(port) + assert port["version"] == "1" # required by v1.0.0 + assert {"property": "tier", "value": "gold"} in port["customProperties"] + assert product["inputPorts"][0]["contractId"] == "p11.input.raw" + validate_odps(product) # v1.0.0 schema + # Without an explicit version the source's v1.1.0 is kept untouched. + assert BitolOdpsProvider().render(fluid)["product"] == odps + + def test_fresh_v100_product_moves_deprecated_to_custom_properties(self) -> None: + fluid = _load(MULTI_EXPOSE_FLUID) + fluid.setdefault("metadata", {})["deprecated"] = True + prov = BitolOdpsProvider(api_version="v1.0.0") + product = prov.render(fluid)["product"] + assert "deprecated" not in product + assert {"property": "deprecated", "value": True} in product["customProperties"] + validate_odps(product) + + +# --------------------------------------------------------------------------- +# Finding 18: fresh output ports keep main's custom properties / overrides +# --------------------------------------------------------------------------- + + +def _pii_fluid() -> Dict[str, Any]: + return { + "fluidVersion": "0.7.1", + "kind": "DataProduct", + "id": "sales.orders_v1", + "name": "Orders", + "metadata": {"environment": "production", "status": "active"}, + "access_control": {"auto_approve": True}, + "exposes": [ + { + "id": "orders", + "contract_id": "orders-contract-v2", + "classification": "pii", + "location": {"account": "acme", "database": "SALES"}, + "binding": {"platform": "snowflake"}, + "contract": {"schema": [{"name": "email", "type": "string"}]}, + }, + { + "id": "totals", + "contract": {"schema": [{"name": "total", "type": "number"}]}, + }, + ], + } + + +class TestFreshOutputPortSurface: + def test_custom_properties_contract_id_and_default_version(self, monkeypatch) -> None: + monkeypatch.setenv("ODPS_DEFAULT_PORT_VERSION", "2") + prov = BitolOdpsProvider() + prov.strict_validation = False + bundle = prov.render(_pii_fluid()) + orders, totals = bundle["product"]["outputPorts"] + assert orders["version"] == "2" + assert orders["contractId"] == "orders-contract-v2" + custom = {c["property"]: c["value"] for c in orders["customProperties"]} + assert custom == { + "server": {"account": "acme", "database": "SALES"}, + "environment": "production", + "containsPii": True, + "autoApprove": False, + "status": "active", + } + totals_custom = {c["property"]: c["value"] for c in totals["customProperties"]} + assert totals_custom["containsPii"] is False + assert totals_custom["autoApprove"] is True + # The sibling ODCS follows the overridden contractId. + assert bundle["contracts"]["orders-contract-v2"]["id"] == "orders-contract-v2" + validate_odps(bundle["product"]) + + def test_legacy_alias_emits_them_too(self, monkeypatch) -> None: + monkeypatch.delenv("ODPS_DEFAULT_PORT_VERSION", raising=False) + product = OdpsStandardProvider().render(_pii_fluid()) + assert product["outputPorts"][0]["version"] == "1" + names = [c["property"] for c in product["outputPorts"][0]["customProperties"]] + assert names == ["server", "environment", "containsPii", "autoApprove", "status"] + + def test_pii_in_field_classification_disables_auto_approve(self) -> None: + fluid = _pii_fluid() + fluid["exposes"][1]["contract"]["schema"][0]["classification"] = "Sensitive" + prov = BitolOdpsProvider() + prov.strict_validation = False + port = prov.render(fluid)["product"]["outputPorts"][1] + custom = {c["property"]: c["value"] for c in port["customProperties"]} + assert custom["containsPii"] is True and custom["autoApprove"] is False + + +# --------------------------------------------------------------------------- +# Finding 27: ODCS import rejects schema-invalid input unless lenient +# --------------------------------------------------------------------------- + + +def _invalid_odcs() -> Dict[str, Any]: + doc = _events_odcs() + doc["schema"][0]["properties"][0]["logicalType"] = "not-a-type" + return doc + + +class TestStrictOdcsImport: + def test_invalid_odcs_raises_by_default(self) -> None: + with pytest.raises(ProviderError, match="ODCS validation failed"): + OdcsProvider().import_contract(_invalid_odcs()) + + def test_lenient_opt_in_warns_and_converts(self) -> None: + fluid = OdcsProvider().import_contract(_invalid_odcs(), lenient=True) + assert fluid["exposes"] + + def test_odcs_only_directory_honours_lenient(self, tmp_path: Path) -> None: + _write(tmp_path / "bad.odcs.yaml", _invalid_odcs()) + with pytest.raises(ProviderError): + BitolOdpsProvider().import_directory(tmp_path) + fluid = BitolOdpsProvider().import_directory(tmp_path, lenient=True) + assert len(fluid["exposes"]) == 2 + + +# --------------------------------------------------------------------------- +# Finding 34: resolution problems are skipped when lenient +# --------------------------------------------------------------------------- + + +class TestLenientResolution: + def test_url_input_port_with_remote_disabled_is_skipped(self, hand_bundle: Path) -> None: + odps = _odps_doc() + odps["inputPorts"] = [ + { + "name": "crm", + "version": "1.0.0", + "contractId": "https://contracts.example.invalid/crm.odcs.yaml", + } + ] + _write(hand_bundle / "shop.odps.yaml", odps) + fluid = BitolOdpsProvider().import_directory(hand_bundle) # allow_remote=False + assert [e["id"] for e in fluid["expects"]] == ["crm"] + assert "contract" not in fluid["expects"][0] + + def test_invalid_local_input_contract_is_skipped(self, hand_bundle: Path) -> None: + odps = _odps_doc() + odps["inputPorts"] = [{"name": "up", "version": "1.0.0", "contractId": "up.contract"}] + _write(hand_bundle / "shop.odps.yaml", odps) + bad = _invalid_odcs() + bad["id"] = "up.contract" + _write(hand_bundle / "upstream" / "up.contract.odcs.yaml", bad) + fluid = BitolOdpsProvider().import_directory(hand_bundle) + assert "contract" not in fluid["expects"][0] + + def test_output_port_remote_disabled_is_skipped_when_lenient(self, tmp_path: Path) -> None: + odps = { + "apiVersion": "v1.0.0", + "kind": "DataProduct", + "id": "p", + "status": "active", + "outputPorts": [ + {"name": "o", "version": "1", "contractId": "https://example.invalid/o.yaml"} + ], + } + prov = BitolOdpsProvider() + with pytest.raises(ProviderError): + prov.import_contract(odps, base_path=tmp_path, allow_remote=False) + fluid = prov.import_contract(odps, base_path=tmp_path, allow_remote=False, lenient=True) + assert len(fluid["exposes"]) == 1 + + +# --------------------------------------------------------------------------- +# Finding 35: a local probe must find the requested contract, not a namesake +# --------------------------------------------------------------------------- + + +class TestResolverIdMatch: + def _contract(self, cid: str) -> Dict[str, Any]: + return { + "apiVersion": "v3.1.0", + "kind": "DataContract", + "id": cid, + "version": "1.0.0", + "status": "active", + "schema": [], + } + + def test_last_segment_namesake_is_not_returned(self, tmp_path: Path) -> None: + _write(tmp_path / "contracts" / "orders.odcs.yaml", self._contract("shop.orders")) + with pytest.raises(ContractNotFound): + ContractResolver(base_path=tmp_path).resolve("warehouse.orders") + + def test_last_segment_file_with_matching_id_still_resolves(self, tmp_path: Path) -> None: + _write(tmp_path / "orders.odcs.yaml", self._contract("warehouse.orders")) + resolved = ContractResolver(base_path=tmp_path).resolve("warehouse.orders") + assert resolved.odcs["id"] == "warehouse.orders" + + def test_namesake_does_not_leak_into_an_input_expect(self, tmp_path: Path) -> None: + own = self._contract("shop.orders") + own["schema"] = [ + { + "name": "orders", + "logicalType": "object", + "properties": [{"name": "shop_order_id", "logicalType": "string"}], + } + ] + _write(tmp_path / "contracts" / "orders.odcs.yaml", own) + odps = { + "apiVersion": "v1.0.0", + "kind": "DataProduct", + "id": "shop", + "status": "active", + "outputPorts": [{"name": "orders", "version": "1", "contractId": "shop.orders"}], + "inputPorts": [{"name": "orders", "version": "1", "contractId": "warehouse.orders"}], + } + _write(tmp_path / "shop.odps.yaml", odps) + fluid = BitolOdpsProvider().import_directory(tmp_path) + assert fluid["exposes"][0]["contract"]["schema"][0]["name"] == "shop_order_id" + assert "contract" not in fluid["expects"][0] + + +# --------------------------------------------------------------------------- +# Findings 49 / 50 / 56: ids never escape the bundle / output directory +# --------------------------------------------------------------------------- + + +class TestPathTraversal: + def _secret(self, root: Path) -> Path: + return _write( + root / "secret" / "private.odcs.yaml", + { + "apiVersion": "v3.1.0", + "kind": "DataContract", + "id": str(root / "secret" / "private"), + "version": "1.0.0", + "status": "active", + "schema": [], + }, + ) + + def test_absolute_contract_id_is_not_read(self, tmp_path: Path) -> None: + self._secret(tmp_path) + bundle = tmp_path / "bundle" + bundle.mkdir() + resolver = ContractResolver(base_path=bundle, allow_remote=False) + with pytest.raises(ContractNotFound): + resolver.resolve(str(tmp_path / "secret" / "private")) + + def test_dot_dot_contract_id_is_not_read(self, tmp_path: Path) -> None: + secret = self._secret(tmp_path) + doc = _load(secret) + doc["id"] = "../secret/private" + _write(secret, doc) + bundle = tmp_path / "bundle" + bundle.mkdir() + resolver = ContractResolver(base_path=bundle, allow_remote=False) + with pytest.raises(ContractNotFound): + resolver.resolve("../secret/private") + + def test_symlink_out_of_the_bundle_is_ignored(self, tmp_path: Path) -> None: + secret = self._secret(tmp_path) + doc = _load(secret) + doc["id"] = "linked" + _write(secret, doc) + bundle = tmp_path / "bundle" + bundle.mkdir() + try: + (bundle / "linked.odcs.yaml").symlink_to(secret) + except OSError: + pytest.skip("symlinks not supported") + resolver = ContractResolver(base_path=bundle, allow_remote=False) + resolver.index_directory(bundle) + with pytest.raises(ContractNotFound): + resolver.resolve("linked") + with pytest.raises(ProviderError, match="no ODPS doc and no ODCS files"): + BitolOdpsProvider().import_directory(bundle) + + @pytest.mark.parametrize("bad_id", ["../escaped/orders", "/abs/orders", "a\\b", ".."]) + def test_export_refuses_ids_that_escape_out_dir(self, tmp_path: Path, bad_id: str) -> None: + fluid = _load(MULTI_EXPOSE_FLUID) + fluid["id"] = bad_id + fluid.get("contract", {}).pop("id", None) + out = tmp_path / "trav" / "out" + prov = BitolOdpsProvider() + prov.strict_validation = False + with pytest.raises(ProviderError, match="Refusing to write bundle file"): + prov.render(fluid, out_dir=out) + assert not (tmp_path / "trav").exists() + + def test_export_refuses_contract_ids_that_escape_out_dir(self, tmp_path: Path) -> None: + fluid = _pii_fluid() + fluid["exposes"][0]["contract_id"] = "../../x/p.daily" + prov = BitolOdpsProvider() + prov.strict_validation = False + with pytest.raises(ProviderError, match="Refusing to write bundle file"): + prov.render(fluid, out_dir=tmp_path / "out") + assert not (tmp_path / "out").exists() + assert not list(tmp_path.rglob("*.odcs.yaml")) + + +# --------------------------------------------------------------------------- +# Finding 37: FLUID contracts are not mistaken for ODPS docs +# --------------------------------------------------------------------------- + + +class TestFluidContractsAreNotOdps: + @pytest.mark.parametrize("kind", ["DataProduct", "DataContract"]) + def test_project_dir_with_fluid_contract_and_bundle(self, tmp_path: Path, kind: str) -> None: + fluid = _load(MULTI_EXPOSE_FLUID) + fluid["kind"] = kind # FLUID's kind enum overlaps both Bitol kinds + _write(tmp_path / "contract.fluid.yaml", fluid) + BitolOdpsProvider().render(fluid, out_dir=tmp_path) + imported = BitolOdpsProvider().import_directory(tmp_path) + assert len(imported["exposes"]) == 2 + assert imported["contract"]["id"] == "commerce.orders-product" + + @pytest.mark.parametrize("kind", ["DataProduct", "DataContract"]) + def test_directory_with_only_a_fluid_contract_raises(self, tmp_path: Path, kind: str) -> None: + fluid = _load(MULTI_EXPOSE_FLUID) + fluid["kind"] = kind + _write(tmp_path / "contract.fluid.yaml", fluid) + with pytest.raises(ProviderError, match="no ODPS doc and no ODCS files"): + BitolOdpsProvider().import_directory(tmp_path) + + +# --------------------------------------------------------------------------- +# Finding 57: `fluid opds import .` names the product after the directory +# --------------------------------------------------------------------------- + + +def test_odcs_only_import_of_dot_uses_the_real_directory_name(monkeypatch) -> None: + monkeypatch.chdir(FIXTURES / "odps" / "contracts-only") + fluid = BitolOdpsProvider().import_directory(Path(".")) + assert fluid["metadata"]["name"] == "contracts-only" + assert fluid["contract"]["id"] == "contracts-only" + + +# --------------------------------------------------------------------------- +# Finding 71: remote fetch limits +# --------------------------------------------------------------------------- + + +def _valid_contract_json() -> bytes: + return json.dumps( + { + "apiVersion": "v3.1.0", + "kind": "DataContract", + "id": "remote.contract", + "version": "1.0.0", + "status": "active", + "schema": [], + } + ).encode() + + +class _Handler(http.server.BaseHTTPRequestHandler): + def log_message(self, *args: Any) -> None: # silence test output + pass + + def do_GET(self) -> None: # noqa: N802 + if self.path == "/ok": + body = _valid_contract_json() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + elif self.path == "/big": + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.end_headers() # no Content-Length: the cap must hold while reading + for _ in range(64): + self.wfile.write(b" " * 1024) + elif self.path == "/ftp": + self.send_response(302) + self.send_header("Location", "ftp://127.0.0.1/x.odcs.yaml") + self.end_headers() + elif self.path == "/slow": + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.end_headers() + try: + for _ in range(40): + self.wfile.write(b" ") + self.wfile.flush() + time.sleep(0.1) + except OSError: + pass + + +@pytest.fixture +def http_base(monkeypatch) -> Iterator[str]: + for var in ("http_proxy", "HTTP_PROXY", "https_proxy", "HTTPS_PROXY"): + monkeypatch.delenv(var, raising=False) + server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), _Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield f"http://127.0.0.1:{server.server_address[1]}" + finally: + server.shutdown() + server.server_close() + + +class TestRemoteFetchLimits: + def test_small_body_is_fetched(self, http_base: str) -> None: + resolved = ContractResolver(allow_remote=True).resolve(f"{http_base}/ok") + assert resolved.odcs["id"] == "remote.contract" + + def test_body_over_the_cap_is_refused(self, http_base: str) -> None: + resolver = ContractResolver(allow_remote=True, max_bytes=4096) + with pytest.raises(ContractNotFound): + resolver.resolve(f"{http_base}/big") + + def test_redirect_to_ftp_is_refused(self, http_base: str, monkeypatch) -> None: + import urllib.request + + ftp_calls = [] + + def _no_ftp(self: Any, req: Any) -> Any: + ftp_calls.append(req.full_url) + raise urllib.error.URLError("ftp disabled in tests") + + monkeypatch.setattr(urllib.request.FTPHandler, "ftp_open", _no_ftp) + with pytest.raises(ContractNotFound): + ContractResolver(allow_remote=True).resolve(f"{http_base}/ftp") + assert ftp_calls == [] # the redirect was refused, not followed + + def test_total_deadline_is_enforced(self, http_base: str) -> None: + resolver = ContractResolver(allow_remote=True, timeout=2, total_timeout=0.5) + started = time.monotonic() + with pytest.raises(ContractNotFound): + resolver.resolve(f"{http_base}/slow") + assert time.monotonic() - started < 3 + + def test_undecodable_body_is_contract_not_found(self, http_base: str, monkeypatch) -> None: + from fluid_build.providers.odps_standard import resolver as resolver_mod + + class _Resp: + headers = {"Content-Type": "application/json"} + + def __init__(self) -> None: + self._body = b"\xff\xfe\x00bad" + + def read(self, amt: int = -1) -> bytes: + body, self._body = self._body, b"" + return body + + def __enter__(self): + return self + + def __exit__(self, *_: Any) -> bool: + return False + + # ``urlopen`` also receives the fetch deadline (``deadline=``). + monkeypatch.setattr(resolver_mod, "urlopen", lambda url, timeout, **kw: _Resp()) + with pytest.raises(ContractNotFound): + ContractResolver(allow_remote=True).resolve("https://example.invalid/c.yaml") + + +def test_to_model_message_mentions_python_floor(monkeypatch) -> None: + import builtins + + real_import = builtins.__import__ + + def fake_import(name: str, *args: Any, **kwargs: Any) -> Any: + if name.startswith("open_data_contract_standard"): + raise ImportError(name) + return real_import(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", fake_import) + with pytest.raises(ProviderError, match=r"Python >= 3\.10"): + OdcsProvider().to_model(_events_odcs()) + + +def test_render_does_not_mutate_the_source_fluid(hand_bundle: Path) -> None: + prov = _strict_provider() + fluid = prov.import_directory(hand_bundle) + before = copy.deepcopy(fluid) + prov.render(fluid) + assert fluid == before + assert os.path.isdir(hand_bundle) + + +class TestFreshProductCustomPropertiesMatchMain: + """origin/main's OdpsStandardProvider emitted product-level + customProperties ``type`` / ``domain`` / ``fluidVersion``; consumers such + as Data Mesh Manager read them, so fresh v1.0.0 exports keep them.""" + + def _fluid(self) -> Dict[str, Any]: + return { + "id": "sales.orders", + "metadata": { + "name": "Orders", + "version": "2.1.0", + "domain": "sales", + "product_type": "sourceAligned", + }, + "exposes": [], + } + + def test_v100_fresh_product_carries_mains_custom_properties(self) -> None: + prov = BitolOdpsProvider() + prov.strict_validation = False + product = prov.render(self._fluid())["product"] + assert product["customProperties"] == [ + {"property": "type", "value": "sourceAligned"}, + {"property": "domain", "value": "sales"}, + {"property": "fluidVersion", "value": "2.1.0"}, + ] + + def test_v110_fresh_product_has_native_type_and_keeps_the_rest(self) -> None: + prov = BitolOdpsProvider(api_version="v1.1.0") + prov.strict_validation = False + product = prov.render(self._fluid())["product"] + assert product["type"] == "sourceAligned" + assert {"property": "type", "value": "sourceAligned"} not in product["customProperties"] + assert {"property": "domain", "value": "sales"} in product["customProperties"] diff --git a/tests/test_review_fixes_opds_cli.py b/tests/test_review_fixes_opds_cli.py new file mode 100644 index 00000000..8d653280 --- /dev/null +++ b/tests/test_review_fixes_opds_cli.py @@ -0,0 +1,628 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Regression tests for the ``fluid opds`` / ``fluid odps-bitol`` CLI review fixes. + +Each test class names the review finding it pins down. +""" + +from __future__ import annotations + +import argparse +import datetime +import json +import logging +import warnings +from argparse import Namespace +from pathlib import Path +from typing import Any, Dict +from unittest.mock import MagicMock, patch + +import pytest +import yaml + +from fluid_build.cli.opds import ( + ODPS_VERSIONS, + SPEC_BITOL_1_0_0, + SPEC_BITOL_1_1_0, + SPEC_ODPI_4_1, + cmd_opds_export, + cmd_opds_import, + cmd_opds_info, + cmd_opds_validate, + register, +) +from fluid_build.providers.base import ProviderError + +LOG = logging.getLogger(__name__) + +REPO = Path(__file__).resolve().parent.parent +FIXTURES = Path(__file__).parent / "fixtures" +MULTI_EXPOSE_FLUID = FIXTURES / "fluid" / "contract-multi-expose.fluid.yaml" +BUNDLE_DIR = FIXTURES / "odps" / "product-bitol" +HOURLY_ODCS = FIXTURES / "odps" / "contracts-only" / "hourly_orders.odcs.yaml" +UPSTREAM_V110 = FIXTURES / "upstream" / "odps" / "simple-data-product.v1.1.0.odps.yaml" + +LONG_TEXT = ( + "Orders [beta] data product sourced from the OLTP replica and refreshed every hour " + "by the commerce team; the pattern ^[a-z0-9_]+$ must survive verbatim" +) +PATTERN = "^[a-z0-9_]+$" + + +def _parse(argv): + parser = argparse.ArgumentParser(prog="fluid") + sub = parser.add_subparsers(dest="cmd") + register(sub) + return parser.parse_args(argv) + + +def _export_args(**overrides: Any) -> Namespace: + base: Dict[str, Any] = dict( + contract=str(MULTI_EXPOSE_FLUID), + spec=None, + version=None, + out="-", + out_dir=None, + format=None, + env=None, + validate_strict=True, + pretty=True, + ) + base.update(overrides) + return Namespace(**base) + + +def _import_args(path: Path, **overrides: Any) -> Namespace: + base: Dict[str, Any] = dict( + path=str(path), + spec=None, + version=None, + out=None, + format="yaml", + no_remote=True, + lenient=False, + ) + base.update(overrides) + return Namespace(**base) + + +def _validate_args(path: Path, **overrides: Any) -> Namespace: + base: Dict[str, Any] = dict(file=str(path), spec=None, version=None, full_schema=True) + base.update(overrides) + return Namespace(**base) + + +def _product_file(directory: Path) -> Path: + return next(p for p in directory.iterdir() if ".odps." in p.name) + + +# --------------------------------------------------------------------------- +# Finding 5 — no explicit --spec ⇒ provider's own version chain applies +# --------------------------------------------------------------------------- + + +class TestExportVersionChain: + def test_env_var_selects_version_when_spec_omitted(self, tmp_path, monkeypatch, capsys): + monkeypatch.setenv("ODPS_API_VERSION", "v1.1.0") + rc = cmd_opds_export(_export_args(out_dir=str(tmp_path)), LOG) + assert rc == 0 + product = yaml.safe_load(_product_file(tmp_path).read_text()) + assert product["apiVersion"] == "v1.1.0" + assert "v1.1.0" in capsys.readouterr().out # message reports what was written + + def test_explicit_spec_pins_version_over_env(self, tmp_path, monkeypatch, capsys): + monkeypatch.setenv("ODPS_API_VERSION", "v1.1.0") + rc = cmd_opds_export(_export_args(out_dir=str(tmp_path), spec=SPEC_BITOL_1_0_0), LOG) + assert rc == 0 + product = yaml.safe_load(_product_file(tmp_path).read_text()) + assert product["apiVersion"] == "v1.0.0" + assert "v1.0.0" in capsys.readouterr().out + + def test_default_without_env_stays_v1_0_0(self, tmp_path, monkeypatch): + monkeypatch.delenv("ODPS_API_VERSION", raising=False) + assert cmd_opds_export(_export_args(out_dir=str(tmp_path)), LOG) == 0 + product = yaml.safe_load(_product_file(tmp_path).read_text()) + assert product["apiVersion"] == "v1.0.0" + + def test_provider_built_without_explicit_version_when_spec_omitted(self, tmp_path): + seen = {} + + def _fake_init(self, api_version=None): + seen["api_version"] = api_version + raise RuntimeError("stop") + + with patch("fluid_build.providers.odps_standard.BitolOdpsProvider.__init__", _fake_init): + with patch("fluid_build.cli.opds.console_error"): + cmd_opds_export(_export_args(out_dir=str(tmp_path)), LOG) + assert seen["api_version"] is None + + with patch("fluid_build.providers.odps_standard.BitolOdpsProvider.__init__", _fake_init): + with patch("fluid_build.cli.opds.console_error"): + cmd_opds_export(_export_args(out_dir=str(tmp_path), spec=SPEC_BITOL_1_1_0), LOG) + assert seen["api_version"] == "v1.1.0" + + def test_reexport_of_imported_v110_bundle_preserves_version(self, tmp_path, monkeypatch): + monkeypatch.delenv("ODPS_API_VERSION", raising=False) + first = tmp_path / "v11" + assert ( + cmd_opds_export( + _export_args(out_dir=str(first), spec=SPEC_BITOL_1_1_0, validate_strict=False), + LOG, + ) + == 0 + ) + fluid_path = tmp_path / "imported.fluid.yaml" + assert cmd_opds_import(_import_args(first, out=str(fluid_path)), LOG) == 0 + second = tmp_path / "re" + assert ( + cmd_opds_export( + _export_args(contract=str(fluid_path), out_dir=str(second), validate_strict=False), + LOG, + ) + == 0 + ) + product = yaml.safe_load(_product_file(second).read_text()) + assert product["apiVersion"] == "v1.1.0" + + +# --------------------------------------------------------------------------- +# Finding 14 — machine-readable stdout must bypass Rich +# --------------------------------------------------------------------------- + + +def _odcs_with_pattern(tmp_path: Path) -> Path: + doc = yaml.safe_load(HOURLY_ODCS.read_text()) + doc["description"]["purpose"] = LONG_TEXT + prop = doc["schema"][0]["properties"][0] + prop["logicalTypeOptions"] = {"pattern": PATTERN} + path = tmp_path / "sku.odcs.yaml" + path.write_text(yaml.safe_dump(doc, sort_keys=False)) + return path + + +class TestStdoutPayloadsAreRaw: + def test_import_yaml_to_stdout_round_trips(self, tmp_path, capsys): + src = _odcs_with_pattern(tmp_path) + out_file = tmp_path / "file.fluid.yaml" + assert cmd_opds_import(_import_args(src, out=str(out_file)), LOG) == 0 + capsys.readouterr() + assert cmd_opds_import(_import_args(src), LOG) == 0 + stdout = capsys.readouterr().out + assert PATTERN in stdout + assert yaml.safe_load(stdout) == yaml.safe_load(out_file.read_text()) + + def test_import_json_to_stdout_round_trips(self, tmp_path, capsys): + src = _odcs_with_pattern(tmp_path) + out_file = tmp_path / "file.fluid.json" + assert cmd_opds_import(_import_args(src, out=str(out_file), format="json"), LOG) == 0 + capsys.readouterr() + assert cmd_opds_import(_import_args(src, format="json"), LOG) == 0 + stdout = capsys.readouterr().out + assert json.loads(stdout) == json.loads(out_file.read_text()) + + def test_bitol_export_to_stdout_is_valid_json(self, tmp_path, capsys): + contract = yaml.safe_load(MULTI_EXPOSE_FLUID.read_text()) + contract["metadata"]["description"] = LONG_TEXT + src = tmp_path / "c.fluid.yaml" + src.write_text(yaml.safe_dump(contract, sort_keys=False)) + assert cmd_opds_export(_export_args(contract=str(src)), LOG) == 0 + product = json.loads(capsys.readouterr().out) + assert LONG_TEXT in json.dumps(product, ensure_ascii=False) + + @pytest.mark.parametrize("pretty", [True, False]) + def test_odpi_export_to_stdout_is_valid_json(self, pretty, capsys): + payload = {"dataProductId": "x", "dataProductDescription": LONG_TEXT, "n": 1} + provider = MagicMock() + provider.render.return_value = payload + with patch("fluid_build.cli.bootstrap.build_provider", return_value=provider): + with patch("fluid_build.cli.bootstrap.load_contract_with_overlay", return_value={}): + rc = cmd_opds_export(_export_args(spec=SPEC_ODPI_4_1, pretty=pretty), LOG) + assert rc == 0 + assert json.loads(capsys.readouterr().out) == payload + + def test_info_json_to_stdout_is_valid_json(self, capsys): + assert cmd_opds_info(Namespace(version=None, spec=None, json=True), LOG) == 0 + assert json.loads(capsys.readouterr().out) == ODPS_VERSIONS + + +# --------------------------------------------------------------------------- +# Findings 15 / 55 — --out FILE writes the full bundle, format from suffix +# --------------------------------------------------------------------------- + + +class TestExportOutFile: + def test_parser_format_default_is_unset(self): + ns = _parse(["opds", "export", "c.yaml", "--out", "p.json"]) + assert ns.format is None + + def test_out_json_infers_json_and_writes_sibling_contracts(self, tmp_path, capsys): + out = tmp_path / "product.json" + assert cmd_opds_export(_export_args(out=str(out)), LOG) == 0 + product = json.loads(out.read_text()) + siblings = sorted(p.name for p in tmp_path.glob("*.odcs.json")) + expected = sorted(f"{p['contractId']}.odcs.json" for p in product["outputPorts"]) + assert siblings == expected and len(siblings) == 2 + for name in siblings: + json.loads((tmp_path / name).read_text()) + message = capsys.readouterr().out + assert str(out) in message + for name in siblings: + assert name in message + + # The written bundle is importable (schema information intact). + fluid_out = tmp_path / "back.fluid.yaml" + assert cmd_opds_import(_import_args(out, out=str(fluid_out)), LOG) == 0 + fluid = yaml.safe_load(fluid_out.read_text()) + names = { + f["name"] + for e in fluid["exposes"] + for f in (e.get("contract") or {}).get("schema") or [] + } + assert "order_id" in names + + def test_out_yaml_infers_yaml(self, tmp_path): + out = tmp_path / "product.yaml" + assert cmd_opds_export(_export_args(out=str(out)), LOG) == 0 + assert yaml.safe_load(out.read_text())["kind"] == "DataProduct" + assert len(list(tmp_path.glob("*.odcs.yaml"))) == 2 + + def test_explicit_format_wins_over_suffix(self, tmp_path): + out = tmp_path / "product.yaml" + assert cmd_opds_export(_export_args(out=str(out), format="json"), LOG) == 0 + assert json.loads(out.read_text())["kind"] == "DataProduct" + assert len(list(tmp_path.glob("*.odcs.json"))) == 2 + + @pytest.mark.parametrize("bad_id", ["../evil", "a/b", "/abs/x", "a\\b", ".."]) + def test_unsafe_contract_ids_rejected(self, tmp_path, bad_id): + bundle = { + "product": {"apiVersion": "v1.0.0", "kind": "DataProduct", "id": "p"}, + "contracts": {bad_id: {"id": bad_id}}, + } + out_dir = tmp_path / "nested" / "out" + out_dir.mkdir(parents=True) + out = out_dir / "product.json" + err = MagicMock() + with patch( + "fluid_build.providers.odps_standard.BitolOdpsProvider.render", return_value=bundle + ): + with patch("fluid_build.cli.opds.console_error", err): + rc = cmd_opds_export(_export_args(out=str(out)), LOG) + assert rc != 0 + assert "contractId" in " ".join(str(c) for c in err.call_args_list) + written = [p for p in tmp_path.rglob("*") if p.is_file()] + assert written == [] + + def test_unsafe_contract_ids_rejected_for_out_dir(self, tmp_path): + bundle = { + "product": {"apiVersion": "v1.0.0", "kind": "DataProduct", "id": "p"}, + "contracts": {"../evil": {"id": "../evil"}}, + } + out_dir = tmp_path / "out" + with patch( + "fluid_build.providers.odps_standard.BitolOdpsProvider.render", return_value=bundle + ): + with patch("fluid_build.cli.opds.console_error"): + rc = cmd_opds_export(_export_args(out_dir=str(out_dir)), LOG) + assert rc != 0 + assert [p for p in tmp_path.rglob("*") if p.is_file()] == [] + + def test_out_and_out_dir_together_rejected(self, tmp_path): + with patch("fluid_build.cli.opds.console_error"): + rc = cmd_opds_export( + _export_args(out=str(tmp_path / "p.json"), out_dir=str(tmp_path / "d")), LOG + ) + assert rc == 2 + assert list(tmp_path.iterdir()) == [] + + +# --------------------------------------------------------------------------- +# Findings 39 / 46 / 68 — validate/info understand Bitol +# --------------------------------------------------------------------------- + + +@pytest.fixture +def bitol_bundle(tmp_path) -> Dict[str, Path]: + ydir = tmp_path / "yaml" + jdir = tmp_path / "json" + assert cmd_opds_export(_export_args(out_dir=str(ydir), spec=SPEC_BITOL_1_0_0), LOG) == 0 + assert ( + cmd_opds_export(_export_args(out_dir=str(jdir), spec=SPEC_BITOL_1_0_0, format="json"), LOG) + == 0 + ) + return {"yaml": _product_file(ydir), "json": _product_file(jdir)} + + +class TestValidateBitol: + def test_parser_accepts_spec_on_validate_and_info(self): + ns = _parse(["opds", "validate", "p.yaml", "--spec", SPEC_BITOL_1_0_0]) + assert ns.spec == SPEC_BITOL_1_0_0 and ns.version is None + ns = _parse(["odps", "info", "--spec", SPEC_BITOL_1_1_0]) + assert ns.spec == SPEC_BITOL_1_1_0 + ns = _parse(["odps", "validate", "p.json", "--version", "4.1"]) + assert ns.version == "4.1" + + @pytest.mark.parametrize("fmt", ["yaml", "json"]) + def test_autodetects_bitol_document(self, bitol_bundle, fmt, capsys): + rc = cmd_opds_validate(_validate_args(bitol_bundle[fmt]), LOG) + assert rc == 0 + assert "commerce.orders-product" in capsys.readouterr().out + + def test_explicit_bitol_spec(self, bitol_bundle): + args = _validate_args(bitol_bundle["yaml"], spec=SPEC_BITOL_1_0_0) + assert cmd_opds_validate(args, LOG) == 0 + + def test_invalid_bitol_document_fails(self, bitol_bundle, tmp_path): + doc = yaml.safe_load(bitol_bundle["yaml"].read_text()) + del doc["id"] + bad = tmp_path / "bad.odps.yaml" + bad.write_text(yaml.safe_dump(doc)) + with patch("fluid_build.cli.opds.console_error") as err: + assert cmd_opds_validate(_validate_args(bad), LOG) == 1 + assert "id" in " ".join(str(c) for c in err.call_args_list) + + def test_schema_follows_document_api_version(self, capsys): + assert cmd_opds_validate(_validate_args(UPSTREAM_V110), LOG) == 0 + assert "v1.1.0" in capsys.readouterr().out + + def test_explicit_spec_pins_schema_version(self): + ok = _validate_args(UPSTREAM_V110, spec=SPEC_BITOL_1_1_0) + assert cmd_opds_validate(ok, LOG) == 0 + pinned = _validate_args(UPSTREAM_V110, spec=SPEC_BITOL_1_0_0) + with patch("fluid_build.cli.opds.console_error"): + assert cmd_opds_validate(pinned, LOG) == 1 + + def test_odpi_document_still_uses_odpi_validator(self, tmp_path): + doc = {"dataProductId": "x", "dataProductName": "X", "dataProductDescription": "d"} + path = tmp_path / "odpi.json" + path.write_text(json.dumps(doc)) + result = {"valid": True, "validation_type": "basic", "warnings": []} + with patch( + "fluid_build.providers.odps.validator.validate_opds_structure", return_value=result + ) as v: + assert cmd_opds_validate(_validate_args(path), LOG) == 0 + assert v.call_args.kwargs["version"] == "4.1" + assert v.call_args.args[0] == doc + + def test_odpi_document_in_yaml_is_accepted(self, tmp_path): + doc = {"dataProductId": "x", "dataProductName": "X", "dataProductDescription": "d"} + path = tmp_path / "odpi.yaml" + path.write_text(yaml.safe_dump(doc)) + result = {"valid": True, "validation_type": "basic", "warnings": []} + with patch( + "fluid_build.providers.odps.validator.validate_opds_structure", return_value=result + ) as v: + assert cmd_opds_validate(_validate_args(path), LOG) == 0 + assert v.call_args.args[0] == doc + + @pytest.mark.parametrize( + "flags", [{"version": "4.1"}, {"spec": SPEC_ODPI_4_1}], ids=["version", "spec"] + ) + def test_explicit_odpi_forces_odpi_path(self, bitol_bundle, flags): + result = {"valid": False, "validation_type": "basic", "errors": ["nope"]} + with patch( + "fluid_build.providers.odps.validator.validate_opds_structure", return_value=result + ) as v: + with patch("fluid_build.cli.opds.console_error"): + assert cmd_opds_validate(_validate_args(bitol_bundle["json"], **flags), LOG) == 1 + assert v.called + + +class TestInfoSpec: + @pytest.mark.parametrize("spec", [SPEC_BITOL_1_0_0, SPEC_BITOL_1_1_0]) + def test_info_bitol_spec(self, spec, capsys): + assert cmd_opds_info(Namespace(version=None, spec=spec, json=False), LOG) == 0 + out = capsys.readouterr().out + assert "bitol-io/open-data-product-standard" in out + assert spec.split("-")[1] in out + + def test_info_bitol_spec_json(self, capsys): + args = Namespace(version=None, spec=SPEC_BITOL_1_1_0, json=True) + assert cmd_opds_info(args, LOG) == 0 + data = json.loads(capsys.readouterr().out) + assert data[SPEC_BITOL_1_1_0]["apiVersion"] == "v1.1.0" + + def test_info_odpi_spec_matches_version_flag(self, capsys): + assert cmd_opds_info(Namespace(version=None, spec=SPEC_ODPI_4_1, json=True), LOG) == 0 + assert json.loads(capsys.readouterr().out) == {"4.1": ODPS_VERSIONS["4.1"]} + + def test_info_usage_uses_spec_not_deprecated_version(self, capsys): + assert cmd_opds_info(Namespace(version=None, spec=None, json=False), LOG) == 0 + out = capsys.readouterr().out + assert "export contract.yaml --version 4.1" not in out + assert "--spec odpi-4.1" in out + assert SPEC_BITOL_1_0_0 in out + + +# --------------------------------------------------------------------------- +# Finding 69 — --no-validate-strict reports problems as warnings +# --------------------------------------------------------------------------- + + +class TestNonStrictValidationWarns: + def test_problems_reported_as_warnings(self, tmp_path, capsys): + with patch( + "fluid_build.providers.odps_standard.validation.validate", + side_effect=ProviderError("ODPS validation failed: boom-marker"), + ): + rc = cmd_opds_export(_export_args(out_dir=str(tmp_path), validate_strict=False), LOG) + assert rc == 0 + captured = capsys.readouterr() + assert "boom-marker" in captured.err + assert "boom-marker" not in captured.out + assert _product_file(tmp_path).exists() + + def test_clean_bundle_emits_no_warnings(self, tmp_path, capsys): + rc = cmd_opds_export(_export_args(out_dir=str(tmp_path), validate_strict=False), LOG) + assert rc == 0 + assert "⚠" not in capsys.readouterr().err + + def test_strict_mode_still_fails(self, tmp_path): + with patch( + "fluid_build.providers.odps_standard.provider.validate", + side_effect=ProviderError("ODPS validation failed: boom-marker"), + ): + with patch("fluid_build.cli.opds.console_error"): + rc = cmd_opds_export(_export_args(out_dir=str(tmp_path)), LOG) + assert rc == 1 + + def test_help_text_matches_behaviour(self): + parser = argparse.ArgumentParser(prog="fluid") + register(parser.add_subparsers(dest="cmd")) + sub = parser._subparsers._group_actions[0].choices["odps"] + export = sub._subparsers._group_actions[0].choices["export"] + action = next(a for a in export._actions if "--no-validate-strict" in a.option_strings) + assert "warning" in action.help.lower() + + +# Finding 63 (--no-remote really skips the network) is pinned in +# tests/test_opds_cli_spec.py::TestNoRemoteFlag. + + +# --------------------------------------------------------------------------- +# Finding 67 — `fluid odps-bitol` prints its deprecation notice +# --------------------------------------------------------------------------- + + +class TestOdpsBitolDeprecation: + def _ns(self, argv): + from fluid_build.cli import odps_standard + + parser = argparse.ArgumentParser(prog="fluid") + odps_standard.register(parser.add_subparsers(dest="cmd")) + return parser.parse_args(argv) + + def test_export_warns(self, tmp_path, capsys): + ns = self._ns( + ["odps-bitol", "export", str(MULTI_EXPOSE_FLUID), "-o", str(tmp_path / "p.yaml")] + ) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + assert ns.func(ns, LOG) == 0 + assert any(issubclass(w.category, DeprecationWarning) for w in caught) + err = capsys.readouterr().err + assert "deprecated" in err and "fluid opds export --spec bitol-1.0.0" in err + + def test_validate_and_info_warn(self, tmp_path, capsys): + product = next(BUNDLE_DIR.glob("*.odps.yaml")) + for argv in (["odps-bitol", "validate", str(product)], ["odps-bitol", "info"]): + ns = self._ns(argv) + with warnings.catch_warnings(record=True): + warnings.simplefilter("always") + ns.func(ns, LOG) + assert "deprecated" in capsys.readouterr().err + + +# --------------------------------------------------------------------------- +# Finding 64 — top-level help describes the new default +# --------------------------------------------------------------------------- + + +class TestMainHelpLabels: + def test_odps_row_mentions_bitol_default_and_bitol_cli_deprecated(self, capsys): + from fluid_build.cli.help_formatter import print_main_help + + print_main_help(argparse.ArgumentParser(prog="fluid")) + out = capsys.readouterr().out + odps_line = next(line for line in out.splitlines() if line.strip().startswith("odps ")) + assert "Bitol" in odps_line and "ODPI" in odps_line + bitol_line = next(line for line in out.splitlines() if "odps-bitol" in line) + assert "deprecated" in bitol_line.lower() + + +# --------------------------------------------------------------------------- +# Findings 38 / 40 — docs and help match the code +# --------------------------------------------------------------------------- + + +class TestDocsMatchCode: + DOC = REPO / "docs" / "BITOL_INTEROP.md" + + def test_resolver_matrix_documented_as_implemented(self): + from fluid_build.providers.odps_standard.resolver import ( + DEFAULT_EXTENSIONS, + DEFAULT_SUBDIRS, + ) + + text = self.DOC.read_text(encoding="utf-8") + line = next(ln for ln in text.splitlines() if "Local file probes" in ln) + for ext in DEFAULT_EXTENSIONS: + assert f"`{ext}`" in line or f"{ext}," in line or f"{ext}}}" in line + for sub in DEFAULT_SUBDIRS: + assert (f"{sub}/" if sub else "") in line + assert "odcs/contracts/" not in line + assert ".yml, .json" not in line + + def test_import_not_promised_as_validated_fluid(self): + text = self.DOC.read_text(encoding="utf-8") + assert "one validated FLUID" not in text + parser = argparse.ArgumentParser(prog="fluid") + register(parser.add_subparsers(dest="cmd")) + sub = parser._subparsers._group_actions[0].choices["odps"] + importp = sub._subparsers._group_actions[0].choices["import"] + assert "validated FLUID contract" not in importp.description + assert "skeleton" in importp.description + + +# --------------------------------------------------------------------------- +# Integration seams (found when merging the review-fix workstreams) +# --------------------------------------------------------------------------- + +_MINIMAL_ODCS = Path(__file__).parent / "fixtures" / "odcs" / "contract-minimal.yaml" + + +class TestImportSeams: + def _invalid_odcs(self, tmp_path: Path) -> Path: + doc = yaml.safe_load(_MINIMAL_ODCS.read_text()) + doc["schema"][0]["properties"][0]["logicalType"] = "not-a-type" + target = tmp_path / "broken.odcs.yaml" + target.write_text(yaml.safe_dump(doc, sort_keys=False)) + return target + + def test_lone_invalid_odcs_fails_without_lenient(self, tmp_path: Path) -> None: + # ODCS import is strict again (origin/main behaviour). + assert cmd_opds_import(_import_args(self._invalid_odcs(tmp_path)), LOG) == 1 + + def test_lone_invalid_odcs_imports_with_lenient(self, tmp_path: Path) -> None: + out_file = tmp_path / "out.fluid.yaml" + args = _import_args(self._invalid_odcs(tmp_path), lenient=True, out=str(out_file)) + assert cmd_opds_import(args, LOG) == 0 + assert yaml.safe_load(out_file.read_text())["exposes"] + + def test_kind_sniffed_invalid_odcs_honours_lenient(self, tmp_path: Path) -> None: + # No .odcs suffix: routed by ``kind: DataContract``. + src = self._invalid_odcs(tmp_path) + plain = tmp_path / "broken.yaml" + plain.write_text(src.read_text()) + assert cmd_opds_import(_import_args(plain), LOG) == 1 + assert cmd_opds_import(_import_args(plain, lenient=True), LOG) == 0 + + def test_fluid_contract_is_rejected_not_imported_as_odps(self, tmp_path: Path) -> None: + fluid = tmp_path / "contract.yaml" + fluid.write_text( + yaml.safe_dump({"fluidVersion": "0.7.2", "kind": "DataProduct", "id": "x", "name": "x"}) + ) + with patch("fluid_build.cli.opds.console_error") as err: + assert cmd_opds_import(_import_args(fluid), LOG) == 1 + assert "is a FLUID contract" in err.call_args[0][0] + + def test_odcs_with_unquoted_yaml_date_validates_and_imports(self, tmp_path: Path) -> None: + doc = yaml.safe_load(_MINIMAL_ODCS.read_text()) + target = tmp_path / "dated.odcs.yaml" + text = yaml.safe_dump(doc, sort_keys=False) + # An unquoted date loads as datetime.date; JSON Schema sees its ISO string. + text += "slaProperties:\n - property: effectiveFrom\n value: 2024-01-01\n" + target.write_text(text) + assert isinstance(yaml.safe_load(text)["slaProperties"][0]["value"], datetime.date) + assert cmd_opds_import(_import_args(target), LOG) == 0 diff --git a/tests/test_spec_version_matrix.py b/tests/test_spec_version_matrix.py new file mode 100644 index 00000000..cb37e8fd --- /dev/null +++ b/tests/test_spec_version_matrix.py @@ -0,0 +1,512 @@ +# Copyright 2024-2026 Agentics Transformation Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +"""ODCS v3.1.0 / v3.2.0 and ODPS v1.0.0 / v1.1.0 — version matrix. + +Three guarantees, each checked against every fixture we have (hand-rolled +*and* the verbatim upstream examples under ``tests/fixtures/upstream``): + +1. **Lossless round-trip** — import → export reproduces the document + byte-for-byte, whatever version it declares (including the older-than- + vendored v3.0.0 adventureworks contract). +2. **Version fidelity** — the re-export declares the *same* ``apiVersion`` + the source did, and validates against the schema for that version. +3. **Version selection** — a document is judged by the schema it claims: + a v3.2.0 / v1.1.0 doc is never rejected by the older schema's ``enum``, + and a doc using new-version fields *is* rejected when explicitly checked + against the older schema. + +Refreshing the upstream fixtures (Bitol repos are Apache-2.0):: + + B=https://raw.githubusercontent.com/bitol-io + curl -sSL $B/open-data-contract-standard/main/docs/examples/all/full-example.odcs.yaml \\ + -o tests/fixtures/upstream/odcs/full-example.v3.2.0.odcs.yaml + for n in customer-data-product deprecated-data-product simple-data-product variables vendor-custom-properties; do + curl -sSL $B/open-data-product-standard/main/docs/examples/$n.odps.yaml \\ + -o tests/fixtures/upstream/odps/$n.v1.1.0.odps.yaml; done +""" + +from __future__ import annotations + +import copy +import logging +from pathlib import Path +from typing import Any, Dict + +import pytest +import yaml + +from fluid_build.providers.base import ProviderError +from fluid_build.providers.odcs import OdcsProvider +from fluid_build.providers.odcs import validation as odcs_validation +from fluid_build.providers.odcs.validation import roundtrip_check +from fluid_build.providers.odps_standard import BitolOdpsProvider +from fluid_build.providers.odps_standard import validation as odps_validation +from fluid_build.providers.odps_standard.mappers import EXPORT_PIPELINE as ODPS_EXPORT +from fluid_build.providers.odps_standard.mappers import IMPORT_PIPELINE as ODPS_IMPORT +from fluid_build.providers.odps_standard.mappers.base import ExportCtx, ImportCtx + +FIXTURES = Path(__file__).parent / "fixtures" +LOG = logging.getLogger("test_spec_version_matrix") + +ODCS_FIXTURES = sorted( + list((FIXTURES / "odcs").glob("*.yaml")) + + list((FIXTURES / "upstream" / "odcs").glob("*.yaml")) + + list((FIXTURES / "odps" / "product-bitol").glob("*.odcs.yaml")) +) +ODPS_FIXTURES = sorted( + list((FIXTURES / "odps" / "product-bitol").glob("*.odps.yaml")) + + list((FIXTURES / "upstream" / "odps").glob("*.yaml")) +) +_ids = lambda paths: [p.name for p in paths] # noqa: E731 + + +def _load(path: Path) -> Dict[str, Any]: + with open(path) as f: + return yaml.safe_load(f) + + +def _odps_mapper_roundtrip(odps: Dict[str, Any]) -> Dict[str, Any]: + """Run the ODPS mapper pipelines only (no contract resolver).""" + fluid: Dict[str, Any] = {} + for m in ODPS_IMPORT: + m.to_fluid(ImportCtx(odps=odps, fluid=fluid, logger=LOG)) + out: Dict[str, Any] = {} + ectx = ExportCtx(fluid=fluid, odps=out, logger=LOG, options={"include_custom_properties": True}) + for m in ODPS_EXPORT: + m.to_odps(ectx) + return out + + +def _fail_msg(name: str, r: Dict[str, Any]) -> str: + return ( + f"Round-trip diff for {name}:\n" + f" missing: {r['missing'][:10]}\n" + f" extra : {r['extra'][:10]}\n" + f" changed: {r['changed'][:10]}" + ) + + +# --------------------------------------------------------------------------- +# 1 + 2: lossless round-trip, version fidelity, re-export validity +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("path", ODCS_FIXTURES, ids=_ids(ODCS_FIXTURES)) +def test_odcs_roundtrip_is_lossless_and_version_faithful(path: Path) -> None: + odcs = _load(path) + prov = OdcsProvider() + out = prov.render(prov.import_contract(odcs)) + r = roundtrip_check(odcs, out) + assert r["equal"], _fail_msg(path.name, r) + assert out["apiVersion"] == odcs["apiVersion"] + prov.validate_contract(out) # judged by the version it declares + + +@pytest.mark.parametrize("path", ODPS_FIXTURES, ids=_ids(ODPS_FIXTURES)) +def test_odps_mapper_roundtrip_is_lossless_and_version_faithful(path: Path) -> None: + odps = _load(path) + out = _odps_mapper_roundtrip(odps) + r = roundtrip_check(odps, out) + assert r["equal"], _fail_msg(path.name, r) + assert out["apiVersion"] == odps["apiVersion"] + BitolOdpsProvider().validate_product(out) + + +def test_odps_output_port_contract_id_is_preserved_not_reminted() -> None: + """Regression: export used to overwrite an imported port's contractId with + the ``{productId}.{port}`` convention (input ports were already right).""" + odps = _load(FIXTURES / "upstream" / "odps" / "customer-data-product.v1.1.0.odps.yaml") + out = _odps_mapper_roundtrip(odps) + assert [p["contractId"] for p in out["outputPorts"]] == [ + p["contractId"] for p in odps["outputPorts"] + ] + + +def test_odcs_explicit_false_and_empty_list_survive() -> None: + """Regression: ``required: false`` / ``primaryKey: false`` / ``tags: []`` + were dropped by truthiness checks (800+ diffs on adventureworks).""" + odcs = { + "version": "1.0.0", + "apiVersion": "v3.1.0", + "kind": "DataContract", + "id": "t", + "status": "active", + "schema": [ + { + "name": "t", + "properties": [ + { + "name": "a", + "logicalType": "string", + "required": False, + "primaryKey": False, + "tags": [], + }, + ], + } + ], + } + prov = OdcsProvider() + out = prov.render(prov.import_contract(odcs)) + assert out["schema"][0]["properties"][0] == odcs["schema"][0]["properties"][0] + + +def test_odcs_imported_doc_without_servers_does_not_synthesize_one() -> None: + odcs = { + "version": "1.0.0", + "apiVersion": "v3.1.0", + "kind": "DataContract", + "id": "t", + "status": "active", + "schema": [ + { + "name": "t", + "physicalType": "table", + "properties": [{"name": "a", "logicalType": "string"}], + } + ], + } + prov = OdcsProvider() + out = prov.render(prov.import_contract(odcs)) + assert "servers" not in out + prov.validate_contract(out) + + +def test_odcs_fluid_edit_wins_over_stash() -> None: + """The stash is a base layer, not a freeze: a FLUID-side edit is exported.""" + odcs = _load(FIXTURES / "odcs" / "contract-full.yaml") + prov = OdcsProvider() + fluid = prov.import_contract(odcs) + fld = fluid["exposes"][0]["contract"]["schema"][1] # customer_id + fld["description"] = "EDITED" + fld["required"] = False + out = prov.render(fluid) + prop = out["schema"][0]["properties"][1] + assert prop["description"] == "EDITED" + assert prop["required"] is False + # untouched siblings still verbatim + assert out["schema"][0]["properties"][0] == odcs["schema"][0]["properties"][0] + + +# --------------------------------------------------------------------------- +# 3: version selection +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "declared, expected", + [ + ("v3.2.0", "v3.2.0"), + ("v3.1.0", "v3.1.0"), + ("3.1.0", "v3.1.0"), # bare + ("v3.0.0", "v3.2.0"), # older-than-vendored → newest superset + ("v9.9.9", "v3.2.0"), # unknown-future → newest (still fails on enum) + (None, "v3.2.0"), + ], +) +def test_odcs_resolve_schema_version(declared, expected) -> None: + doc = {"apiVersion": declared} if declared is not None else {} + assert odcs_validation.resolve_schema_version(doc) == expected + + +@pytest.mark.parametrize( + "declared, expected", + [("v1.1.0", "v1.1.0"), ("v1.0.0", "v1.0.0"), ("v0.9.0", "v1.1.0"), (None, "v1.1.0")], +) +def test_odps_resolve_schema_version(declared, expected) -> None: + doc = {"apiVersion": declared} if declared is not None else {} + assert odps_validation.resolve_schema_version(doc) == expected + + +def test_load_schema_rejects_unknown_version() -> None: + with pytest.raises(ProviderError, match="Unsupported ODCS apiVersion"): + odcs_validation.load_schema("v9.9.9") + with pytest.raises(ProviderError, match="Unsupported ODPS apiVersion"): + odps_validation.load_schema("v9.9.9") + + +def test_odcs_v320_doc_passes_auto_but_fails_v310_schema() -> None: + doc = _load(FIXTURES / "upstream" / "odcs" / "full-example.v3.2.0.odcs.yaml") + odcs_validation.validate(doc) # auto → v3.2.0 + with pytest.raises(ProviderError, match="v3.2.0"): + odcs_validation.validate(doc, odcs_validation.load_schema("v3.1.0")) + + +def test_odps_v110_features_pass_auto_but_fail_v100_schema() -> None: + doc = _load(FIXTURES / "upstream" / "odps" / "customer-data-product.v1.1.0.odps.yaml") + odps_validation.validate(doc) # auto → v1.1.0 + with pytest.raises(ProviderError): + odps_validation.validate(doc, odps_validation.load_schema("v1.0.0")) + + +def test_odps_v110_status_is_optional_but_required_in_v100() -> None: + doc = {"apiVersion": "v1.1.0", "kind": "DataProduct", "id": "x", "name": "x"} + odps_validation.validate(doc) + legacy = {**doc, "apiVersion": "v1.0.0"} + with pytest.raises(ProviderError, match="status"): + odps_validation.validate(legacy) + + +def test_older_than_vendored_odcs_validates_via_superset() -> None: + doc = _load(FIXTURES / "upstream" / "odcs" / "postgresql-adventureworks.v3.0.0.odcs.yaml") + odcs_validation.validate(doc) + + +# --------------------------------------------------------------------------- +# Emit version: default unchanged, opt-in to newer, preserved on re-export +# --------------------------------------------------------------------------- + + +@pytest.fixture +def fresh_fluid() -> Dict[str, Any]: + return _load(FIXTURES / "fluid" / "contract-multi-expose.fluid.yaml") + + +def test_fresh_export_defaults_unchanged(fresh_fluid, monkeypatch) -> None: + monkeypatch.delenv("ODCS_API_VERSION", raising=False) + monkeypatch.delenv("ODPS_API_VERSION", raising=False) + assert OdcsProvider().render(fresh_fluid)["apiVersion"] == "v3.1.0" + prov = BitolOdpsProvider() + prov.strict_validation = False + assert prov.render(fresh_fluid)["product"]["apiVersion"] == "v1.0.0" + + +def test_opt_in_newer_via_constructor(fresh_fluid) -> None: + assert OdcsProvider(api_version="v3.2.0").render(fresh_fluid)["apiVersion"] == "v3.2.0" + assert OdcsProvider(api_version="3.2.0").render(fresh_fluid)["apiVersion"] == "v3.2.0" + prov = BitolOdpsProvider(api_version="v1.1.0") + prov.strict_validation = False + assert prov.render(fresh_fluid)["product"]["apiVersion"] == "v1.1.0" + + +def test_opt_in_newer_via_env(fresh_fluid, monkeypatch) -> None: + monkeypatch.setenv("ODCS_API_VERSION", "v3.2.0") + monkeypatch.setenv("ODPS_API_VERSION", "v1.1.0") + assert OdcsProvider().render(fresh_fluid)["apiVersion"] == "v3.2.0" + prov = BitolOdpsProvider() + prov.strict_validation = False + assert prov.render(fresh_fluid)["product"]["apiVersion"] == "v1.1.0" + + +def test_fresh_v110_export_is_strictly_valid_and_uses_native_type(fresh_fluid) -> None: + fluid = copy.deepcopy(fresh_fluid) + fluid.setdefault("metadata", {})["product_type"] = "consumerAligned" + prov = BitolOdpsProvider(api_version="v1.1.0") + prov.strict_validation = True # fail-loud against the v1.1.0 schema + product = prov.render(fluid)["product"] + assert product["type"] == "consumerAligned" + assert not any(p.get("property") == "type" for p in product.get("customProperties", [])) + + +def test_fresh_v100_export_keeps_legacy_custom_property_type(fresh_fluid) -> None: + fluid = copy.deepcopy(fresh_fluid) + fluid.setdefault("metadata", {})["product_type"] = "analytical" + prov = BitolOdpsProvider(api_version="v1.0.0") + prov.strict_validation = True + product = prov.render(fluid)["product"] + assert "type" not in product + assert {"property": "type", "value": "analytical"} in product["customProperties"] + + +def test_explicit_version_overrides_preserved_on_reexport() -> None: + odcs = _load(FIXTURES / "odcs" / "contract-minimal.yaml") # declares v3.1.0 + fluid = OdcsProvider().import_contract(odcs) + out = OdcsProvider(api_version="v3.2.0").render(fluid) + assert out["apiVersion"] == "v3.2.0" + odcs_validation.validate(out) + + +# --------------------------------------------------------------------------- +# v3.2.0 content: map / vector logical types, new server engines +# --------------------------------------------------------------------------- + + +def test_map_and_vector_logical_types_round_trip() -> None: + odcs = { + "version": "1.0.0", + "apiVersion": "v3.2.0", + "kind": "DataContract", + "id": "t", + "schema": [ + { + "name": "t", + "properties": [ + { + "name": "attrs", + "logicalType": "map", + "map": { + "key": {"logicalType": "string"}, + "value": {"logicalType": "integer"}, + }, + }, + { + "name": "emb", + "logicalType": "vector", + "logicalTypeOptions": {"dimensions": 384, "elementType": "float32"}, + }, + ], + } + ], + } + prov = OdcsProvider() + fluid = prov.import_contract(odcs) + types = {f["name"]: f["type"] for f in fluid["exposes"][0]["contract"]["schema"]} + assert types == {"attrs": "map", "emb": "vector"} + out = prov.render(fluid) + r = roundtrip_check(odcs, out) + assert r["equal"], _fail_msg("map/vector", r) + prov.validate_contract(out) + + +@pytest.mark.parametrize( + "platform, location, expected_keys", + [ + ( + "iceberg", + { + "catalog": "c", + "catalogUrl": "https://x/v1", + "namespace": "db.s", + "warehouse": "w", + "junk": 1, + }, + {"catalog", "catalogUrl", "namespace", "warehouse"}, + ), + ( + "hana", + {"host": "h", "port": 30015, "database": "d", "schema": "s", "junk": 1}, + {"host", "port", "database", "schema"}, + ), + ( + "teradata", + {"host": "h", "port": 1025, "database": "d", "schema": "dropped"}, + {"host", "port", "database"}, + ), + ( + "exasol", + {"host": "h", "port": 8563, "schema": "s", "database": "dropped"}, + {"host", "port", "schema"}, + ), + ("actian", {"host": "h", "port": 1, "database": "d"}, {"host", "port", "database"}), + ], +) +def test_new_server_engines_emit_only_allowed_fields_and_validate( + platform, location, expected_keys +) -> None: + fluid = { + "id": "p", + "metadata": {"name": "p", "version": "1.0.0", "status": "active"}, + "exposes": [ + { + "id": "e", + "binding": {"platform": platform, "location": location}, + "contract": {"schema": [{"name": "a", "type": "string"}]}, + } + ], + } + out = OdcsProvider(api_version="v3.2.0").render(fluid) + server = out["servers"][0] + # ``id`` is the sanitised StableId emitted on fresh exports (as on main). + assert set(server) - {"id", "server", "type"} == expected_keys + odcs_validation.validate(out) # unevaluatedProperties: false would catch leaks + + +# --------------------------------------------------------------------------- +# Tooling gaps handled gracefully +# --------------------------------------------------------------------------- + + +def test_vowl_skips_with_warning_on_unsupported_api_version(caplog) -> None: + pytest.importorskip("vowl") + doc = _load(FIXTURES / "upstream" / "odcs" / "full-example.v3.2.0.odcs.yaml") + with caplog.at_level(logging.WARNING): + result = odcs_validation.validate_via_vowl(doc) + if result is None: + assert any("vowl second-pass skipped" in r.message for r in caplog.records) + else: # a newer vowl that supports v3.2.0 — must report it correctly + assert result["api_version"] == "v3.2.0" + + +def _install_fake_vowl(monkeypatch, exc: Exception) -> None: + """Put a stand-in ``vowl`` module on ``sys.modules`` whose ``Contract`` + raises ``exc``, so the skip/raise branches run without vowl installed.""" + import sys + import types + + fake = types.ModuleType("vowl") + + class Contract: # noqa: D401 - mimics vowl.Contract's constructor + def __init__(self, data): + raise exc + + fake.Contract = Contract + monkeypatch.setitem(sys.modules, "vowl", fake) + + +def test_vowl_unsupported_api_version_skips_with_warning(monkeypatch, caplog) -> None: + """CI-effective (no importorskip): an older vowl that predates the + contract's apiVersion must skip the second pass with a warning, not fail + the export.""" + _install_fake_vowl(monkeypatch, ValueError("Unsupported API version: v3.2.0")) + doc = _load(FIXTURES / "upstream" / "odcs" / "full-example.v3.2.0.odcs.yaml") + with caplog.at_level(logging.WARNING): + result = odcs_validation.validate_via_vowl(doc) + assert result is None + assert any("vowl second-pass skipped" in r.getMessage() for r in caplog.records) + + +def test_vowl_other_value_error_is_a_provider_error(monkeypatch) -> None: + """Only the unsupported-apiVersion tooling gap is skipped; any other vowl + rejection still surfaces as a ``vowl:`` ProviderError.""" + _install_fake_vowl(monkeypatch, ValueError("schema[0].properties: bad")) + doc = _load(FIXTURES / "upstream" / "odcs" / "full-example.v3.2.0.odcs.yaml") + with pytest.raises(ProviderError, match="^vowl: ValueError"): + odcs_validation.validate_via_vowl(doc) + + +def test_to_model_typed_view_or_clear_error() -> None: + doc = _load(FIXTURES / "odcs" / "contract-full.yaml") + prov = OdcsProvider() + try: + import open_data_contract_standard # noqa: F401 + except ImportError: + with pytest.raises(ProviderError, match="odcs-strict"): + prov.to_model(doc) + return + model = prov.to_model(doc) + assert model.id == doc["id"] + assert model.model_dump(by_alias=True, mode="json", exclude_unset=True) == doc + + +# --------------------------------------------------------------------------- +# CLI: --spec bitol-1.1.0 +# --------------------------------------------------------------------------- + + +def test_cli_spec_bitol_1_1_0_exports_v110(tmp_path: Path) -> None: + from argparse import Namespace + + from fluid_build.cli.opds import SPEC_BITOL_1_1_0, cmd_opds_export, resolve_spec + + args = Namespace( + contract=str(FIXTURES / "fluid" / "contract-multi-expose.fluid.yaml"), + spec=SPEC_BITOL_1_1_0, + version=None, + out="-", + out_dir=str(tmp_path), + format="yaml", + env=None, + validate_strict=True, + pretty=True, + ) + assert resolve_spec(args) == SPEC_BITOL_1_1_0 + assert cmd_opds_export(args, LOG) == 0 + product = _load(next(tmp_path.glob("*.odps.yaml"))) + assert product["apiVersion"] == "v1.1.0" + for odcs_file in tmp_path.glob("*.odcs.yaml"): + odcs_validation.validate(_load(odcs_file))