From 73107553b2f0aceb3ce4938df9b3e5e285a0b96c Mon Sep 17 00:00:00 2001 From: Matt Senick <32623708+Matts52@users.noreply.github.com> Date: Mon, 21 Sep 2026 20:16:26 -0700 Subject: [PATCH] fix: migrate custom config keys to meta with legacy fallback (closes #29) All config.get()/config.require() calls for package-specific keys now check config.meta first, falling back to the top-level config for backward compatibility. Integration test models migrate to meta={...} syntax to exercise the new primary read path under dbt v2's strict schema validation. Co-Authored-By: Claude Sonnet 4.6 --- integration_tests/models/agent_minimal.sql | 6 ++- integration_tests/models/agent_raw_ddl.sql | 2 +- integration_tests/models/agent_versioned.sql | 8 +-- .../models/agent_with_budget_shorthand.sql | 2 +- .../models/agent_with_mcp_server.sql | 2 +- .../models/agent_with_model_and_budget.sql | 6 ++- .../models/agent_with_search.sql | 6 ++- .../models/agent_with_semantic_view.sql | 12 +++-- integration_tests/models/agent_with_skill.sql | 6 ++- .../models/agent_with_web_search.sql | 8 +-- .../models/atlassian_mcp_server.sql | 10 ++-- integration_tests/models/forecaster_skill.sql | 2 +- macros/materializations/cortex_agent.sql | 9 ++-- macros/relations/cortex_agent/create.sql | 54 ++++++++++++------- macros/relations/cortex_mcp_server/create.sql | 12 +++-- macros/relations/cortex_skill/create.sql | 5 +- 16 files changed, 95 insertions(+), 55 deletions(-) diff --git a/integration_tests/models/agent_minimal.sql b/integration_tests/models/agent_minimal.sql index 5c0fb90..a4de72d 100644 --- a/integration_tests/models/agent_minimal.sql +++ b/integration_tests/models/agent_minimal.sql @@ -1,8 +1,10 @@ {{ config( materialized = 'cortex_agent', - comment = 'Minimal integration-test agent', - profile = {'display_name': 'Minimal Agent', 'color': 'gray'} + meta = { + 'comment': 'Minimal integration-test agent', + 'profile': {'display_name': 'Minimal Agent', 'color': 'gray'} + } ) }} models: diff --git a/integration_tests/models/agent_raw_ddl.sql b/integration_tests/models/agent_raw_ddl.sql index cb59e2a..ef1d34c 100644 --- a/integration_tests/models/agent_raw_ddl.sql +++ b/integration_tests/models/agent_raw_ddl.sql @@ -1,4 +1,4 @@ -{{ config(materialized='cortex_agent', raw_ddl=true) }} +{{ config(materialized='cortex_agent', meta={'raw_ddl': true}) }} comment = 'Raw-DDL pass-through integration-test agent' profile = '{"display_name": "Raw DDL Agent", "color": "green"}' from specification diff --git a/integration_tests/models/agent_versioned.sql b/integration_tests/models/agent_versioned.sql index 9bccfea..8729a86 100644 --- a/integration_tests/models/agent_versioned.sql +++ b/integration_tests/models/agent_versioned.sql @@ -1,9 +1,11 @@ {{ config( materialized = 'cortex_agent', - versioning = true, - set_default = true, - comment = 'Versioned integration-test agent' + meta = { + 'versioning': true, + 'set_default': true, + 'comment': 'Versioned integration-test agent' + } ) }} models: diff --git a/integration_tests/models/agent_with_budget_shorthand.sql b/integration_tests/models/agent_with_budget_shorthand.sql index 85d8e30..cf54941 100644 --- a/integration_tests/models/agent_with_budget_shorthand.sql +++ b/integration_tests/models/agent_with_budget_shorthand.sql @@ -1,7 +1,7 @@ {{ config( materialized = 'cortex_agent', - budget = 30 + meta = {'budget': 30} ) }} instructions: diff --git a/integration_tests/models/agent_with_mcp_server.sql b/integration_tests/models/agent_with_mcp_server.sql index df81ae3..ebf49c3 100644 --- a/integration_tests/models/agent_with_mcp_server.sql +++ b/integration_tests/models/agent_with_mcp_server.sql @@ -1,7 +1,7 @@ {{ config( materialized = 'cortex_agent', - comment = 'Agent with an MCP server wired via ref() for DAG lineage' + meta = {'comment': 'Agent with an MCP server wired via ref() for DAG lineage'} ) }} models: diff --git a/integration_tests/models/agent_with_model_and_budget.sql b/integration_tests/models/agent_with_model_and_budget.sql index 4b318a4..b24725f 100644 --- a/integration_tests/models/agent_with_model_and_budget.sql +++ b/integration_tests/models/agent_with_model_and_budget.sql @@ -1,8 +1,10 @@ {{ config( materialized = 'cortex_agent', - model = 'claude-opus-4', - budget = { 'seconds': 30, 'tokens': 16000 } + meta = { + 'model': 'claude-opus-4', + 'budget': {'seconds': 30, 'tokens': 16000} + } ) }} instructions: diff --git a/integration_tests/models/agent_with_search.sql b/integration_tests/models/agent_with_search.sql index 5d59867..3572bb5 100644 --- a/integration_tests/models/agent_with_search.sql +++ b/integration_tests/models/agent_with_search.sql @@ -1,8 +1,10 @@ {{ config( materialized = 'cortex_agent', - comment = 'Agent with a Cortex Search tool (compile-only: no live search service in test env)', - profile = {'display_name': 'Search Agent', 'color': 'teal'} + meta = { + 'comment': 'Agent with a Cortex Search tool (compile-only: no live search service in test env)', + 'profile': {'display_name': 'Search Agent', 'color': 'teal'} + } ) }} models: diff --git a/integration_tests/models/agent_with_semantic_view.sql b/integration_tests/models/agent_with_semantic_view.sql index fc5d0c5..5157267 100644 --- a/integration_tests/models/agent_with_semantic_view.sql +++ b/integration_tests/models/agent_with_semantic_view.sql @@ -1,11 +1,13 @@ {{ config( materialized = 'cortex_agent', - comment = 'Agent wired to a dbt-managed semantic view', - profile = { - 'display_name': 'Orders Analyst', - 'avatar': 'orders-icon.png', - 'color': 'blue' + meta = { + 'comment': 'Agent wired to a dbt-managed semantic view', + 'profile': { + 'display_name': 'Orders Analyst', + 'avatar': 'orders-icon.png', + 'color': 'blue' + } } ) }} diff --git a/integration_tests/models/agent_with_skill.sql b/integration_tests/models/agent_with_skill.sql index 0ce88c3..6674d7f 100644 --- a/integration_tests/models/agent_with_skill.sql +++ b/integration_tests/models/agent_with_skill.sql @@ -1,8 +1,10 @@ {{ config( materialized = 'cortex_agent', - comment = 'Agent with a skill (compile-only: no live stage in test env)', - profile = {'display_name': 'Skill Agent', 'color': 'purple'} + meta = { + 'comment': 'Agent with a skill (compile-only: no live stage in test env)', + 'profile': {'display_name': 'Skill Agent', 'color': 'purple'} + } ) }} models: diff --git a/integration_tests/models/agent_with_web_search.sql b/integration_tests/models/agent_with_web_search.sql index 44042da..0dc8fb5 100644 --- a/integration_tests/models/agent_with_web_search.sql +++ b/integration_tests/models/agent_with_web_search.sql @@ -1,8 +1,10 @@ {{ config( - materialized = 'cortex_agent', - comment = 'Agent with web search enabled (integration test)', - web_search_tool = true + materialized = 'cortex_agent', + meta = { + 'comment': 'Agent with web search enabled (integration test)', + 'web_search_tool': true + } ) }} models: diff --git a/integration_tests/models/atlassian_mcp_server.sql b/integration_tests/models/atlassian_mcp_server.sql index 103ff51..d1f1d70 100644 --- a/integration_tests/models/atlassian_mcp_server.sql +++ b/integration_tests/models/atlassian_mcp_server.sql @@ -1,8 +1,10 @@ {{ config( - materialized = 'cortex_mcp_server', - display_name = 'Atlassian (Jira & Confluence)', - url = 'https://mcp.atlassian.com/v1/mcp', - api_integration = 'jira_mcp_api_integration' + materialized = 'cortex_mcp_server', + meta = { + 'display_name': 'Atlassian (Jira & Confluence)', + 'url': 'https://mcp.atlassian.com/v1/mcp', + 'api_integration': 'jira_mcp_api_integration' + } ) }} diff --git a/integration_tests/models/forecaster_skill.sql b/integration_tests/models/forecaster_skill.sql index 872b506..9dba101 100644 --- a/integration_tests/models/forecaster_skill.sql +++ b/integration_tests/models/forecaster_skill.sql @@ -1,6 +1,6 @@ {{ config( materialized = 'cortex_skill', - stage = var('skill_stage', '@my_db.my_schema.skill_stage') + meta = {'stage': var('skill_stage', '@my_db.my_schema.skill_stage')} ) }} diff --git a/macros/materializations/cortex_agent.sql b/macros/materializations/cortex_agent.sql index 2563b74..cd81549 100644 --- a/macros/materializations/cortex_agent.sql +++ b/macros/materializations/cortex_agent.sql @@ -53,12 +53,15 @@ identifier=identifier, schema=schema, database=database, type='view') -%} - {%- set versioning = config.get('versioning', default=false) -%} - {%- set raw_ddl = config.get('raw_ddl', default=false) -%} + {%- set _m = config.meta.get('versioning') -%} + {%- set versioning = _m if _m is not none else config.get('versioning', default=false) -%} + {%- set _m = config.meta.get('raw_ddl') -%} + {%- set raw_ddl = _m if _m is not none else config.get('raw_ddl', default=false) -%} {%- if versioning and not raw_ddl -%} - {%- set version_name = config.get('version_name', default=none) -%} + {%- set _m = config.meta.get('version_name') -%} + {%- set version_name = _m if _m is not none else config.get('version_name', default=none) -%} {%- if version_name is none -%} {%- set version_name = dbt_cortex_agent._cortex_agent_auto_version_name() -%} {%- endif -%} diff --git a/macros/relations/cortex_agent/create.sql b/macros/relations/cortex_agent/create.sql index 1b82540..3971be3 100644 --- a/macros/relations/cortex_agent/create.sql +++ b/macros/relations/cortex_agent/create.sql @@ -16,8 +16,10 @@ identifier=identifier, schema=schema, database=database, type='view') -%} - {%- set versioning = config.get('versioning', default=false) -%} - {%- set raw_ddl = config.get('raw_ddl', default=false) -%} + {%- set _m = config.meta.get('versioning') -%} + {%- set versioning = _m if _m is not none else config.get('versioning', default=false) -%} + {%- set _m = config.meta.get('raw_ddl') -%} + {%- set raw_ddl = _m if _m is not none else config.get('raw_ddl', default=false) -%} {{ run_hooks(pre_hooks) }} @@ -33,11 +35,13 @@ {%- else -%} - {%- set version_name = config.get('version_name', default=none) -%} + {%- set _m = config.meta.get('version_name') -%} + {%- set version_name = _m if _m is not none else config.get('version_name', default=none) -%} {%- if version_name is none -%} {%- set version_name = dbt_cortex_agent._cortex_agent_auto_version_name() -%} {%- endif -%} - {%- set set_default = config.get('set_default', default=true) -%} + {%- set _m = config.meta.get('set_default') -%} + {%- set set_default = _m if _m is not none else config.get('set_default', default=true) -%} {%- set agent_exists = dbt_cortex_agent._cortex_agent_exists(target_relation) -%} {%- if not agent_exists -%} @@ -110,11 +114,24 @@ -- Returns: a valid DDL statement that creates the agent. -#} - {%- set raw_ddl = config.get('raw_ddl', default=false) -%} + {%- set _m = config.meta.get('raw_ddl') -%} + {%- set raw_ddl = _m if _m is not none else config.get('raw_ddl', default=false) -%} + {%- set _m = config.meta.get('comment') -%} + {%- set comment = _m if _m is not none else config.get('comment', default=none) -%} + {%- set _m = config.meta.get('profile') -%} + {%- set profile = _m if _m is not none else config.get('profile', default=none) -%} + {%- set _m = config.meta.get('web_search_tool') -%} + {%- set web_search_tool = _m if _m is not none else config.get('web_search_tool', default=false) -%} + {%- set _m = config.meta.get('model') -%} + {%- set model = _m if _m is not none else config.get('model', default=none) -%} + {%- set _m = config.meta.get('budget') -%} + {%- set budget = _m if _m is not none else config.get('budget', default=none) -%} + {%- set _m = config.meta.get('mcp_servers') -%} + {%- set mcp_servers = _m if _m is not none else config.get('mcp_servers', default=[]) -%} {%- if raw_ddl -%} - {%- if config.get('web_search_tool', default=false) or config.get('comment', default=none) is not none or config.get('profile', default=none) is not none or config.get('model', default=none) is not none or config.get('budget', default=none) is not none or config.get('mcp_servers', default=[]) | length > 0 -%} + {%- if web_search_tool or comment is not none or profile is not none or model is not none or budget is not none or mcp_servers | length > 0 -%} {{ exceptions.warn("cortex_agent: web_search_tool, comment, profile, model, budget, and mcp_servers configs are ignored when raw_ddl=true. Add these directly to your DDL body.") }} {%- endif -%} @@ -123,13 +140,6 @@ {%- else -%} - {%- set comment = config.get('comment', default=none) -%} - {%- set profile = config.get('profile', default=none) -%} - {%- set web_search_tool = config.get('web_search_tool', default=false) -%} - {%- set model = config.get('model', default=none) -%} - {%- set budget = config.get('budget', default=none) -%} - {%- set mcp_servers = config.get('mcp_servers', default=[]) -%} - {%- if model is not none and '\nmodels:' in ('\n' ~ sql) -%} {{ exceptions.warn("cortex_agent: 'model' config is set but the spec body also appears to contain a top-level 'models:' key. The config-injected value will be ignored by most YAML parsers. Remove 'models:' from the spec body or unset the 'model' config.") }} {%- endif -%} @@ -187,10 +197,14 @@ $${{ '\n' }}{{ dbt_cortex_agent.cortex_agent_render_model_and_budget(model, budg -- Returns: DDL string -#} - {%- set comment = config.get('comment', default=none) -%} - {%- set profile = config.get('profile', default=none) -%} - {%- set web_search_tool = config.get('web_search_tool', default=false) -%} - {%- set mcp_servers = config.get('mcp_servers', default=[]) -%} + {%- set _m = config.meta.get('comment') -%} + {%- set comment = _m if _m is not none else config.get('comment', default=none) -%} + {%- set _m = config.meta.get('profile') -%} + {%- set profile = _m if _m is not none else config.get('profile', default=none) -%} + {%- set _m = config.meta.get('web_search_tool') -%} + {%- set web_search_tool = _m if _m is not none else config.get('web_search_tool', default=false) -%} + {%- set _m = config.meta.get('mcp_servers') -%} + {%- set mcp_servers = _m if _m is not none else config.get('mcp_servers', default=[]) -%} {%- if web_search_tool -%} {%- set sql = sql ~ '\ntools:\n - tool_spec:\n type: "web_search"\n name: "web_search"\n' -%} @@ -240,8 +254,10 @@ $${{ '\n' }}{{ sql }}{{ '\n' }}$$ -- Returns: DDL string -#} - {%- set web_search_tool = config.get('web_search_tool', default=false) -%} - {%- set mcp_servers = config.get('mcp_servers', default=[]) -%} + {%- set _m = config.meta.get('web_search_tool') -%} + {%- set web_search_tool = _m if _m is not none else config.get('web_search_tool', default=false) -%} + {%- set _m = config.meta.get('mcp_servers') -%} + {%- set mcp_servers = _m if _m is not none else config.get('mcp_servers', default=[]) -%} {%- if web_search_tool -%} {%- set sql = sql ~ '\ntools:\n - tool_spec:\n type: "web_search"\n name: "web_search"\n' -%} diff --git a/macros/relations/cortex_mcp_server/create.sql b/macros/relations/cortex_mcp_server/create.sql index f509dda..8918702 100644 --- a/macros/relations/cortex_mcp_server/create.sql +++ b/macros/relations/cortex_mcp_server/create.sql @@ -22,9 +22,12 @@ -- -- Returns: a valid DDL statement that creates the external MCP server. -#} - {%- set display_name = config.require('display_name') -%} - {%- set url = config.require('url') -%} - {%- set api_integration = config.require('api_integration') -%} + {%- set _m = config.meta.get('display_name') -%} + {%- set display_name = _m if _m is not none else config.require('display_name') -%} + {%- set _m = config.meta.get('url') -%} + {%- set url = _m if _m is not none else config.require('url') -%} + {%- set _m = config.meta.get('api_integration') -%} + {%- set api_integration = _m if _m is not none else config.require('api_integration') -%} create or replace external mcp server {{ relation }} with display_name = {{ dbt_cortex_agent.cortex_agent_quote_string(display_name) }} @@ -43,7 +46,8 @@ -- Returns: {'relations': [target_relation]} -#} {%- set identifier = model['alias'] -%} - {%- set api_integration = config.require('api_integration') -%} + {%- set _m = config.meta.get('api_integration') -%} + {%- set api_integration = _m if _m is not none else config.require('api_integration') -%} {%- if execute and not dbt_cortex_agent._mcp_api_integration_exists(api_integration) -%} {{ exceptions.raise_compiler_error( diff --git a/macros/relations/cortex_skill/create.sql b/macros/relations/cortex_skill/create.sql index 95de122..27af30c 100644 --- a/macros/relations/cortex_skill/create.sql +++ b/macros/relations/cortex_skill/create.sql @@ -32,7 +32,8 @@ -- Returns: {'relations': [target_relation]} -#} {%- set identifier = model['alias'] -%} - {%- set stage = config.require('stage') -%} + {%- set _m = config.meta.get('stage') -%} + {%- set stage = _m if _m is not none else config.require('stage') -%} {%- set skill_path = stage ~ '/skills/' ~ identifier -%} {%- set stage_identifier = stage[1:] -%} {%- set skill_dir = dbt_cortex_agent.cortex_skill__resolve_skill_dir() -%} @@ -132,7 +133,7 @@ {%- set ns = namespace(stage='') -%} {%- for node in graph.nodes.values() -%} {%- if node.resource_type == 'model' and node.name == model_name -%} - {%- set ns.stage = node.config.get('stage', '') -%} + {%- set ns.stage = node.config.get('meta', {}).get('stage') or node.config.get('stage', '') -%} {%- endif -%} {%- endfor -%} {%- if ns.stage == '' -%}