diff --git a/packages/sdk/server-ai/src/ldai/client.py b/packages/sdk/server-ai/src/ldai/client.py index 623ff82..2d5ba42 100644 --- a/packages/sdk/server-ai/src/ldai/client.py +++ b/packages/sdk/server-ai/src/ldai/client.py @@ -37,13 +37,16 @@ _TRACK_SDK_INFO = '$ld:ai:sdk:info' _TRACK_USAGE_AGENT_CONFIG = '$ld:ai:usage:agent-config' +_TRACK_USAGE_AGENT_CONFIG_TEMPLATE = '$ld:ai:usage:agent-config-template' _TRACK_USAGE_AGENT_CONFIGS = '$ld:ai:usage:agent-configs' _TRACK_USAGE_COMPLETION_CONFIG = '$ld:ai:usage:completion-config' +_TRACK_USAGE_COMPLETION_CONFIG_TEMPLATE = '$ld:ai:usage:completion-config-template' _TRACK_USAGE_CREATE_AGENT = '$ld:ai:usage:create-agent' _TRACK_USAGE_CREATE_AGENT_GRAPH = '$ld:ai:usage:create-agent-graph' _TRACK_USAGE_CREATE_JUDGE = '$ld:ai:usage:create-judge' _TRACK_USAGE_CREATE_MODEL = '$ld:ai:usage:create-model' _TRACK_USAGE_JUDGE_CONFIG = '$ld:ai:usage:judge-config' +_TRACK_USAGE_JUDGE_CONFIG_TEMPLATE = '$ld:ai:usage:judge-config-template' _INIT_TRACK_CONTEXT = Context.builder('ld-internal-tracking').kind('ld_ai').anonymous(True).build() @@ -138,10 +141,11 @@ def _completion_config( default: AICompletionConfigDefault, variables: Optional[Dict[str, Any]] = None, default_ai_provider: Optional[str] = None, + interpolate: bool = True, ) -> AICompletionConfig: (model, provider, messages, instructions, tracker_factory, enabled, judge_configuration, variation) = self.__evaluate( - key, context, default.to_dict(), variables + key, context, default.to_dict(), variables, interpolate=interpolate ) evaluator = self._build_evaluator(judge_configuration, context, default_ai_provider, variables) @@ -186,12 +190,41 @@ def completion_config( key, context, default or _DISABLED_COMPLETION_DEFAULT, variables, default_ai_provider ) + def completion_config_template( + self, + key: str, + context: Context, + default: Optional[AICompletionConfigDefault] = None, + ) -> AICompletionConfig: + """ + Get the un-rendered template of a completion configuration. + + Unlike :meth:`completion_config`, Mustache placeholders in + ``messages[].content`` (e.g. ``{{name}}``) are returned as-is rather + than being substituted with variable values. Use this when you need + to inspect or store the raw prompt template before rendering it + yourself. + + :param key: The key of the completion configuration. + :param context: The context to evaluate the completion configuration in. + :param default: The default value of the completion configuration. When not provided, + a disabled config is used as the fallback. + :return: The completion configuration with un-rendered template placeholders. + """ + self._client.track(_TRACK_USAGE_COMPLETION_CONFIG_TEMPLATE, context, key, 1) + + return self._completion_config( + key, context, default or _DISABLED_COMPLETION_DEFAULT, + variables=None, interpolate=False, + ) + def _judge_config( self, key: str, context: Context, default: AIJudgeConfigDefault, variables: Optional[Dict[str, Any]] = None, + interpolate: bool = True, ) -> AIJudgeConfig: if variables is not None: if variables.get('message_history') is not None: @@ -213,7 +246,7 @@ def _judge_config( (model, provider, messages, instructions, tracker_factory, enabled, judge_configuration, variation) = self.__evaluate( - key, context, default.to_dict(), extended_variables + key, context, default.to_dict(), extended_variables, interpolate=interpolate ) def _extract_evaluation_metric_key(variation: Dict[str, Any]) -> Optional[str]: @@ -272,6 +305,35 @@ def judge_config( key, context, default or _DISABLED_JUDGE_DEFAULT, variables ) + def judge_config_template( + self, + key: str, + context: Context, + default: Optional[AIJudgeConfigDefault] = None, + ) -> AIJudgeConfig: + """ + Get the un-rendered template of a judge configuration. + + Unlike :meth:`judge_config`, Mustache placeholders in + ``messages[].content`` are returned as-is rather than being + substituted with variable values. The reserved judge placeholders + (``{{message_history}}`` and ``{{response_to_evaluate}}``) are also + preserved. Use this when you need to inspect the raw prompt template + before rendering it yourself. + + :param key: The key of the judge configuration. + :param context: The context to evaluate the judge configuration in. + :param default: The default value of the judge configuration. When not provided, + a disabled config is used as the fallback. + :return: The judge configuration with un-rendered template placeholders. + """ + self._client.track(_TRACK_USAGE_JUDGE_CONFIG_TEMPLATE, context, key, 1) + + return self._judge_config( + key, context, default or _DISABLED_JUDGE_DEFAULT, + variables=None, interpolate=False, + ) + def create_judge( self, key: str, @@ -544,6 +606,38 @@ def agent_config( key, context, default or _DISABLED_AGENT_DEFAULT, variables ) + def agent_config_template( + self, + key: str, + context: Context, + default: Optional[AIAgentConfigDefault] = None, + ) -> AIAgentConfig: + """ + Get the un-rendered template of an agent configuration. + + Unlike :meth:`agent_config`, Mustache placeholders in ``instructions`` + (e.g. ``{{topic}}``) are returned as-is rather than being substituted + with variable values. Use this when you need to inspect or store the + raw instruction template before rendering it yourself. + + :param key: The agent configuration key. + :param context: The context to evaluate the agent configuration in. + :param default: Default agent values. When not provided, a disabled config is used + as the fallback. + :return: Configured AIAgentConfig instance with un-rendered template placeholders. + """ + self._client.track( + _TRACK_USAGE_AGENT_CONFIG_TEMPLATE, + context, + key, + 1 + ) + + return self.__evaluate_agent( + key, context, default or _DISABLED_AGENT_DEFAULT, + variables=None, interpolate=False, + ) + def agent_configs( self, agent_configs: List[AIAgentConfigRequest], @@ -783,6 +877,7 @@ def __evaluate( default_dict: Dict[str, Any], variables: Optional[Dict[str, Any]] = None, graph_key: Optional[str] = None, + interpolate: bool = True, ) -> Tuple[ Optional[ModelConfig], Optional[ProviderConfig], Optional[List[LDMessage]], Optional[str], Callable[[], LDAIConfigTracker], bool, Optional[Any], Dict[str, Any] @@ -795,6 +890,8 @@ def __evaluate( :param default_dict: Default configuration as dictionary. :param variables: Variables for interpolation. :param graph_key: When set, passed to the tracker so all events include ``graphKey``. + :param interpolate: When False, Mustache placeholders in messages and instructions + are preserved as-is rather than being rendered. :return: Tuple of (model, provider, messages, instructions, tracker_factory, enabled, judge_configuration, variation). """ @@ -812,8 +909,9 @@ def __evaluate( messages = [ LDMessage( role=entry['role'], - content=self.__interpolate_template( - entry['content'], all_variables + content=( + self.__interpolate_template(entry['content'], all_variables) + if interpolate else entry['content'] ), ) for entry in variation['messages'] @@ -821,7 +919,10 @@ def __evaluate( instructions = None if 'instructions' in variation and isinstance(variation['instructions'], str): - instructions = self.__interpolate_template(variation['instructions'], all_variables) + instructions = ( + self.__interpolate_template(variation['instructions'], all_variables) + if interpolate else variation['instructions'] + ) provider_config = None if 'provider' in variation and isinstance(variation['provider'], dict): @@ -888,6 +989,7 @@ def __evaluate_agent( variables: Optional[Dict[str, Any]] = None, graph_key: Optional[str] = None, default_ai_provider: Optional[str] = None, + interpolate: bool = True, ) -> AIAgentConfig: """ Internal method to evaluate an agent configuration. @@ -898,11 +1000,12 @@ def __evaluate_agent( :param variables: Variables for interpolation. :param graph_key: When set, passed to the tracker so all events include ``graphKey``. :param default_ai_provider: Optional default AI provider for judge evaluation. + :param interpolate: When False, Mustache placeholders in instructions are preserved as-is. :return: Configured AIAgentConfig instance. """ (model, provider, messages, instructions, tracker_factory, enabled, judge_configuration, variation) = self.__evaluate( - key, context, default.to_dict(), variables, graph_key=graph_key + key, context, default.to_dict(), variables, graph_key=graph_key, interpolate=interpolate ) # For agents, prioritize instructions over messages diff --git a/packages/sdk/server-ai/tests/test_config_template.py b/packages/sdk/server-ai/tests/test_config_template.py new file mode 100644 index 0000000..ae0e00e --- /dev/null +++ b/packages/sdk/server-ai/tests/test_config_template.py @@ -0,0 +1,298 @@ +"""Tests for *_template config methods (un-interpolated variants).""" + +from unittest.mock import Mock + +import pytest +from ldclient import Config, Context, LDClient +from ldclient.integrations.test_data import TestData + +from ldai import ( + LDAIClient, + LDMessage, + ModelConfig, + ProviderConfig, +) +from ldai.models import ( + AIAgentConfigDefault, + AICompletionConfigDefault, + AIJudgeConfigDefault, +) + + +# --------------------------------------------------------------------------- +# Shared flag data +# --------------------------------------------------------------------------- + +_COMPLETION_FLAG_KEY = 'completion-template-flag' +_AGENT_FLAG_KEY = 'agent-template-flag' +_JUDGE_FLAG_KEY = 'judge-template-flag' + + +@pytest.fixture +def td() -> TestData: + td = TestData.data_source() + + td.update( + td.flag(_COMPLETION_FLAG_KEY) + .variations( + { + 'model': {'name': 'fakeModel', 'parameters': {'temperature': 0.5}}, + 'provider': {'name': 'fakeProvider'}, + 'messages': [ + {'role': 'system', 'content': 'Hello, {{name}}!'}, + {'role': 'user', 'content': 'Today is {{day}}.'}, + ], + '_ldMeta': {'enabled': True, 'variationKey': 'abcd', 'version': 1}, + } + ) + .variation_for_all(0) + ) + + td.update( + td.flag(_AGENT_FLAG_KEY) + .variations( + { + 'model': {'name': 'gpt-4', 'parameters': {'temperature': 0.3}}, + 'provider': {'name': 'openai'}, + 'instructions': 'You are a helpful assistant for {{company}}.', + '_ldMeta': {'enabled': True, 'variationKey': 'agent-v1', 'version': 1}, + } + ) + .variation_for_all(0) + ) + + td.update( + td.flag(_JUDGE_FLAG_KEY) + .variations( + { + 'model': {'name': 'gpt-4', 'parameters': {'temperature': 0.1}}, + 'provider': {'name': 'openai'}, + 'messages': [ + {'role': 'system', 'content': 'Evaluate the following: {{topic}}'}, + ], + 'evaluationMetricKey': '$ld:ai:judge:relevance', + '_ldMeta': {'enabled': True, 'variationKey': 'judge-v1', 'version': 1}, + } + ) + .variation_for_all(0) + ) + + return td + + +@pytest.fixture +def client(td: TestData) -> LDClient: + config = Config('sdk-key', update_processor_class=td, send_events=False) + return LDClient(config=config) + + +@pytest.fixture +def ldai_client(client: LDClient) -> LDAIClient: + return LDAIClient(client) + + +@pytest.fixture +def context() -> Context: + return Context.create('user-key') + + +# --------------------------------------------------------------------------- +# completion_config_template +# --------------------------------------------------------------------------- + +def test_completion_config_template_preserves_placeholders(ldai_client: LDAIClient, context: Context): + config = ldai_client.completion_config_template(_COMPLETION_FLAG_KEY, context) + + assert config.messages is not None + assert config.messages[0].content == 'Hello, {{name}}!' + assert config.messages[1].content == 'Today is {{day}}.' + + +def test_completion_config_template_returns_typed_config(ldai_client: LDAIClient, context: Context): + from ldai.models import AICompletionConfig + config = ldai_client.completion_config_template(_COMPLETION_FLAG_KEY, context) + + assert isinstance(config, AICompletionConfig) + assert config.enabled is True + assert config.model is not None + assert config.model.name == 'fakeModel' + assert config.provider is not None + assert config.provider.name == 'fakeProvider' + + +def test_completion_config_template_differs_from_rendered(ldai_client: LDAIClient, context: Context): + """Template variant should return raw placeholders; regular variant renders them.""" + template = ldai_client.completion_config_template(_COMPLETION_FLAG_KEY, context) + rendered = ldai_client.completion_config(_COMPLETION_FLAG_KEY, context, variables={'name': 'World', 'day': 'Monday'}) + + assert template.messages is not None + assert rendered.messages is not None + assert template.messages[0].content == 'Hello, {{name}}!' + assert rendered.messages[0].content == 'Hello, World!' + + +def test_completion_config_template_tracking(context: Context): + mock_client = Mock() + mock_client.variation.return_value = { + '_ldMeta': {'enabled': True, 'variationKey': 'v1', 'version': 1}, + 'model': {'name': 'test-model'}, + 'messages': [{'role': 'system', 'content': 'Hello, {{name}}!'}], + } + + ldai = LDAIClient(mock_client) + ldai.completion_config_template('my-flag', context) + + mock_client.track.assert_any_call( + '$ld:ai:usage:completion-config-template', + context, + 'my-flag', + 1, + ) + + +def test_completion_config_template_uses_default_when_flag_missing(ldai_client: LDAIClient, context: Context): + default = AICompletionConfigDefault( + enabled=True, + model=ModelConfig('default-model'), + messages=[LDMessage(role='system', content='Fallback: {{placeholder}}')], + ) + config = ldai_client.completion_config_template('missing-flag', context, default) + + assert config.messages is not None + assert config.messages[0].content == 'Fallback: {{placeholder}}' + + +# --------------------------------------------------------------------------- +# agent_config_template +# --------------------------------------------------------------------------- + +def test_agent_config_template_preserves_placeholders(ldai_client: LDAIClient, context: Context): + config = ldai_client.agent_config_template(_AGENT_FLAG_KEY, context) + + assert config.instructions == 'You are a helpful assistant for {{company}}.' + + +def test_agent_config_template_returns_typed_config(ldai_client: LDAIClient, context: Context): + from ldai.models import AIAgentConfig + config = ldai_client.agent_config_template(_AGENT_FLAG_KEY, context) + + assert isinstance(config, AIAgentConfig) + assert config.enabled is True + assert config.model is not None + assert config.model.name == 'gpt-4' + assert config.provider is not None + assert config.provider.name == 'openai' + + +def test_agent_config_template_differs_from_rendered(ldai_client: LDAIClient, context: Context): + """Template variant should return raw placeholders; regular variant renders them.""" + template = ldai_client.agent_config_template(_AGENT_FLAG_KEY, context) + rendered = ldai_client.agent_config(_AGENT_FLAG_KEY, context, variables={'company': 'Acme'}) + + assert template.instructions == 'You are a helpful assistant for {{company}}.' + assert rendered.instructions == 'You are a helpful assistant for Acme.' + + +def test_agent_config_template_tracking(context: Context): + mock_client = Mock() + mock_client.variation.return_value = { + '_ldMeta': {'enabled': True, 'variationKey': 'v1', 'version': 1}, + 'model': {'name': 'test-model'}, + 'instructions': 'Do something for {{entity}}.', + } + + ldai = LDAIClient(mock_client) + ldai.agent_config_template('my-agent-flag', context) + + mock_client.track.assert_any_call( + '$ld:ai:usage:agent-config-template', + context, + 'my-agent-flag', + 1, + ) + + +def test_agent_config_template_uses_default_when_flag_missing(ldai_client: LDAIClient, context: Context): + default = AIAgentConfigDefault( + enabled=True, + model=ModelConfig('default-model'), + instructions='Default: {{placeholder}}', + ) + config = ldai_client.agent_config_template('missing-flag', context, default) + + assert config.instructions == 'Default: {{placeholder}}' + + +# --------------------------------------------------------------------------- +# judge_config_template +# --------------------------------------------------------------------------- + +def test_judge_config_template_preserves_placeholders(ldai_client: LDAIClient, context: Context): + config = ldai_client.judge_config_template(_JUDGE_FLAG_KEY, context) + + assert config.messages is not None + assert config.messages[0].content == 'Evaluate the following: {{topic}}' + + +def test_judge_config_template_returns_typed_config(ldai_client: LDAIClient, context: Context): + from ldai.models import AIJudgeConfig + config = ldai_client.judge_config_template(_JUDGE_FLAG_KEY, context) + + assert isinstance(config, AIJudgeConfig) + assert config.enabled is True + assert config.model is not None + assert config.model.name == 'gpt-4' + assert config.evaluation_metric_key == '$ld:ai:judge:relevance' + + +def test_judge_config_template_preserves_reserved_placeholders(context: Context): + """Reserved judge placeholders must survive even in the template variant.""" + mock_client = Mock() + mock_client.variation.return_value = { + '_ldMeta': {'enabled': True, 'variationKey': 'v1', 'version': 1}, + 'model': {'name': 'gpt-4'}, + 'messages': [ + {'role': 'system', 'content': 'History: {{message_history}} Response: {{response_to_evaluate}}'}, + ], + 'evaluationMetricKey': '$ld:ai:judge:relevance', + } + + ldai = LDAIClient(mock_client) + config = ldai.judge_config_template('judge-flag', context) + + assert config.messages is not None + assert '{{message_history}}' in config.messages[0].content + assert '{{response_to_evaluate}}' in config.messages[0].content + + +def test_judge_config_template_tracking(context: Context): + mock_client = Mock() + mock_client.variation.return_value = { + '_ldMeta': {'enabled': True, 'variationKey': 'v1', 'version': 1}, + 'model': {'name': 'gpt-4'}, + 'messages': [{'role': 'system', 'content': 'You are a judge.'}], + 'evaluationMetricKey': '$ld:ai:judge:relevance', + } + + ldai = LDAIClient(mock_client) + ldai.judge_config_template('my-judge-flag', context) + + mock_client.track.assert_any_call( + '$ld:ai:usage:judge-config-template', + context, + 'my-judge-flag', + 1, + ) + + +def test_judge_config_template_uses_default_when_flag_missing(ldai_client: LDAIClient, context: Context): + default = AIJudgeConfigDefault( + enabled=True, + model=ModelConfig('default-model'), + messages=[LDMessage(role='system', content='Judge: {{placeholder}}')], + evaluation_metric_key='$ld:ai:judge:test', + ) + config = ldai_client.judge_config_template('missing-flag', context, default) + + assert config.messages is not None + assert config.messages[0].content == 'Judge: {{placeholder}}'