From b408f502da120518645781bc8feab122767061a0 Mon Sep 17 00:00:00 2001 From: Amar K P Date: Tue, 29 Sep 2026 23:23:40 +0530 Subject: [PATCH 1/2] test(demo): add integration test for demo seed and multi-tool execution - Add tests verifying existence and execution of scripts/seed_demo_db.sql - Test multi-step reasoning workflow combining sql_readonly with calculator - Test offline execution mode of examples/demo.py --- tests/integration/test_demo.py | 143 +++++++++++++++++++++++++++++++++ 1 file changed, 143 insertions(+) create mode 100644 tests/integration/test_demo.py diff --git a/tests/integration/test_demo.py b/tests/integration/test_demo.py new file mode 100644 index 0000000..e03a4e0 --- /dev/null +++ b/tests/integration/test_demo.py @@ -0,0 +1,143 @@ +"""Integration test for demo database seed and multi-tool agent execution.""" + +from __future__ import annotations + +import os +from pathlib import Path +from typing import Any + +import pytest +from sqlalchemy import text +from sqlalchemy.ext.asyncio import create_async_engine + +from agentkit.core.agent import Agent, AgentConfig +from agentkit.llm.fake import FakeLLMClient +from agentkit.tools.builtin.calculator import calculator +from agentkit.tools.builtin.sql_readonly import execute_sql_query +from agentkit.tools.models import ToolCall +from agentkit.tools.registry import ToolRegistry, tool + + +@pytest.mark.asyncio +async def test_seed_demo_db_script_exists_and_seeds(tmp_path: Path) -> None: + """Verify scripts/seed_demo_db.sql exists, is valid SQL, and populates products and orders.""" + seed_path = Path(__file__).parents[2] / "scripts" / "seed_demo_db.sql" + assert seed_path.exists(), "scripts/seed_demo_db.sql must exist" + + sql_content = seed_path.read_text(encoding="utf-8") + assert "products" in sql_content.lower() + assert "orders" in sql_content.lower() + + # Execute against SQLite database to ensure ANSI compliance + db_file = tmp_path / "test_demo.db" + db_url = f"sqlite+aiosqlite:///{db_file}" + engine = create_async_engine(db_url) + + # Split statements by semicolon and execute + statements = [stmt.strip() for stmt in sql_content.split(";") if stmt.strip()] + async with engine.begin() as conn: + for stmt in statements: + await conn.execute(text(stmt)) + + # Verify tables have rows + result = await execute_sql_query("SELECT count(*) as count FROM products", engine=engine) + assert "count" in result + assert "| 0 |" not in result + + order_result = await execute_sql_query("SELECT count(*) as count FROM orders", engine=engine) + assert "count" in order_result + assert "| 0 |" not in result + + await engine.dispose() + + +@pytest.mark.asyncio +async def test_multi_tool_end_to_end_demo_workflow(tmp_path: Path) -> None: + """Verify an agent using SQL tool + calculator resolves multi-step revenue + tax calculation.""" + seed_path = Path(__file__).parents[2] / "scripts" / "seed_demo_db.sql" + sql_content = seed_path.read_text(encoding="utf-8") + + db_file = tmp_path / "demo_workflow.db" + db_url = f"sqlite+aiosqlite:///{db_file}" + engine = create_async_engine(db_url) + + statements = [stmt.strip() for stmt in sql_content.split(";") if stmt.strip()] + async with engine.begin() as conn: + for stmt in statements: + await conn.execute(text(stmt)) + + # Register local sql_query tool pointing to this engine + registry = ToolRegistry() + registry.register(calculator) + + @tool + async def run_sql(query: str) -> str: + """Execute read-only SQL query.""" + return await execute_sql_query(query, engine=engine) + + registry.register(run_sql) + + # Setup FakeLLMClient multi-step script + # Step 1: LLM decides to query SQL for total electronics sales + # Step 2: LLM receives SQL results ($3200) and calculates sales tax (3200 * 1.085) + # Step 3: LLM returns final synthesized answer + llm = FakeLLMClient( + responses=[ + '{"thought": "I need to query total sales for Electronics in 2024."}', + '{"thought": "Total sales is 3200. Now I calculate total with 8.5% tax."}', + "Total Electronics revenue in 2024 was $3,200.00. With 8.5% sales tax, the total is $3,472.00.", + ], + tool_calls_sequence=[ + [ + ToolCall( + name="run_sql", + arguments={ + "query": "SELECT SUM(total_price) as total_rev FROM orders o JOIN products p ON o.product_id = p.id WHERE p.category = 'Electronics'" + }, + ) + ], + [ + ToolCall( + name="calculator", + arguments={"expression": "3200 * 1.085"}, + ) + ], + [], + ], + ) + + agent = Agent( + llm=llm, + tools=registry, + config=AgentConfig(max_steps=5), + ) + + prompt = "What was the total revenue from 'Electronics' category in 2024, and what is the total with 8.5% sales tax?" + result = await agent.run(prompt) + + assert result.success is True + assert result.answer is not None + assert "$3,472.00" in result.answer + assert len(result.steps) == 3 + + await engine.dispose() + + +@pytest.mark.asyncio +async def test_demo_script_runner(monkeypatch: Any) -> None: + """Verify examples/demo.py main function executes successfully in offline mode.""" + import sys + from importlib import import_module + + # Add project root to sys.path + root_path = str(Path(__file__).parents[2]) + if root_path not in sys.path: + sys.path.insert(0, root_path) + + # Verify examples/demo.py exists and can be imported + demo_module = import_module("examples.demo") + assert hasattr(demo_module, "run_demo") + + output = await demo_module.run_demo(offline=True) + assert output is not None + assert "3472" in output or "3,472" in output or "total" in output.lower() From 764faf8aa9a3139aa738807b9306aa36f37cef93 Mon Sep 17 00:00:00 2001 From: Amar K P Date: Tue, 29 Sep 2026 23:27:07 +0530 Subject: [PATCH 2/2] feat(demo): add seed_demo_db.sql and multi-tool demo.py example - Create scripts/seed_demo_db.sql with demo products and orders datasets - Implement examples/demo.py solving multi-step SQL querying + AST calculator problem - Support offline execution mode via FakeLLMClient alongside live LLM providers - Update integration tests in tests/integration/test_demo.py to verify seed and demo execution --- examples/demo.py | 207 +++++++++++++++++++++++++++++++++ scripts/seed_demo_db.sql | 38 ++++++ tests/integration/test_demo.py | 68 ++++++----- 3 files changed, 278 insertions(+), 35 deletions(-) create mode 100644 examples/demo.py create mode 100644 scripts/seed_demo_db.sql diff --git a/examples/demo.py b/examples/demo.py new file mode 100644 index 0000000..bc894a0 --- /dev/null +++ b/examples/demo.py @@ -0,0 +1,207 @@ +"""End-to-End Multi-Tool Demo for AgentKit. + +Demonstrates an autonomous ReAct agent resolving a multi-step analytical question +by combining the read-only SQL tool with the safe AST calculator. + +Problem: + "What was the total revenue from 'Electronics' category in 2024, and what would + be the total if we applied an 8.5% sales tax?" + +Usage: + python examples/demo.py --offline + python examples/demo.py --provider gemini + python examples/demo.py --provider openai +""" + +from __future__ import annotations + +import argparse +import asyncio +import logging +from pathlib import Path + +from sqlalchemy import text +from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine + +from agentkit.config import get_settings +from agentkit.core.agent import Agent, AgentConfig +from agentkit.core.trace import InMemoryTraceSink +from agentkit.llm.base import LLMClient +from agentkit.llm.factory import create_llm_client +from agentkit.llm.fake import FakeLLMClient +from agentkit.tools.builtin.calculator import calculator +from agentkit.tools.builtin.sql_readonly import execute_sql_query +from agentkit.tools.registry import ToolRegistry + +logger = logging.getLogger("agentkit.demo") + + +async def setup_demo_database(db_url: str) -> AsyncEngine: + """Initialize and seed demo database from scripts/seed_demo_db.sql.""" + engine = create_async_engine(db_url) + seed_path = Path(__file__).parents[1] / "scripts" / "seed_demo_db.sql" + + if not seed_path.exists(): + raise FileNotFoundError(f"Seed script not found at {seed_path}") + + sql_content = seed_path.read_text(encoding="utf-8") + statements = [stmt.strip() for stmt in sql_content.split(";") if stmt.strip()] + + async with engine.begin() as conn: + for stmt in statements: + await conn.execute(text(stmt)) + + logger.info("Demo database seeded successfully with products and orders.") + return engine + + +def build_fake_llm() -> LLMClient: + """Create FakeLLMClient for deterministic offline execution.""" + client = FakeLLMClient() + client.queue_tool_call( + tool_name="sql_readonly", + arguments={ + "query": ( + "SELECT SUM(o.total_price) AS total_revenue " + "FROM orders o JOIN products p ON o.product_id = p.id " + "WHERE p.category = 'Electronics' AND o.order_date >= '2024-01-01' " + "AND o.order_date <= '2024-12-31'" + ) + }, + call_id="call_sql_1", + text='{"thought": "I should query the database for 2024 orders of Electronics products."}', + ) + client.queue_tool_call( + tool_name="calculator", + arguments={"expression": "3200 * 1.085"}, + call_id="call_calc_1", + text='{"thought": "SQL query returned $3200 total revenue. Now calculate 8.5% sales tax (3200 * 1.085)."}', + ) + client.queue_text( + "The total revenue from the 'Electronics' category in 2024 was $3,200.00. " + "With an 8.5% sales tax applied, the total comes to $3,472.00." + ) + return client + + +async def run_demo( + offline: bool = False, + provider: str = "gemini", + db_url: str | None = None, +) -> str: + """Run the multi-tool AgentKit demonstration. + + Args: + offline: If True, uses FakeLLMClient with scripted ReAct turns. + provider: Provider name ('gemini', 'openai', 'fake') if offline=False. + db_url: Optional SQLAlchemy database URL override. + + Returns: + The agent's final answer string. + """ + settings = get_settings() + target_db_url = db_url or "sqlite+aiosqlite:///:memory:" + + # 1. Initialize and seed database + engine = await setup_demo_database(target_db_url) + + # 2. Register tools in an isolated registry + registry = ToolRegistry() + registry.register(calculator) + + async def execute_demo_sql(query: str) -> str: + """Execute a read-only SQL query against the database and return results as markdown table.""" + return await execute_sql_query(query, engine=engine) + + registry.register( + execute_demo_sql, + name="sql_readonly", + description="Execute a read-only SQL query against the database and return results as markdown table.", + ) + + # 3. Choose LLM Client + llm: LLMClient + if offline or provider == "fake": + llm = build_fake_llm() + else: + # Check for provider API key + if provider == "gemini" and not settings.GEMINI_API_KEY: + logger.warning("No GEMINI_API_KEY detected. Falling back to offline FakeLLMClient.") + llm = build_fake_llm() + elif provider == "openai" and not settings.OPENAI_API_KEY: + logger.warning("No OPENAI_API_KEY detected. Falling back to offline FakeLLMClient.") + llm = build_fake_llm() + else: + llm = create_llm_client(provider) + + # 4. Create Agent with trace sink + trace_sink = InMemoryTraceSink() + agent = Agent( + llm=llm, + registry=registry, + trace=trace_sink, + config=AgentConfig( + max_steps=6, + run_timeout_s=30.0, + system_prompt=( + "You are an expert data analyst assistant. Answer user questions " + "accurately using available tools (sql_readonly and calculator)." + ), + ), + ) + + query = ( + "What was the total revenue from 'Electronics' category in 2024, " + "and what would be the total if we applied an 8.5% sales tax?" + ) + + logger.info("Executing AgentKit with prompt: %s", query) + result = await agent.run(query) + + logger.info( + "Agent run finished. Status: %s, Steps: %d", + result.status.value, + result.steps_count, + ) + logger.info("Final Answer:\n%s", result.final_answer) + + await engine.dispose() + return result.final_answer or "" + + +def main() -> None: + """CLI entrypoint for running the demo.""" + logging.basicConfig( + level=logging.INFO, format="%(asctime)s [%(levelname)s] %(name)s: %(message)s" + ) + parser = argparse.ArgumentParser(description="AgentKit Multi-Tool ReAct Demo") + parser.add_argument( + "--offline", + action="store_true", + help="Run completely offline using FakeLLMClient without API keys", + ) + parser.add_argument( + "--provider", + type=str, + default="gemini", + choices=["gemini", "openai", "fake"], + help="LLM provider to use (default: gemini)", + ) + parser.add_argument( + "--db-url", + type=str, + default=None, + help="Custom SQLAlchemy database URL (defaults to in-memory SQLite)", + ) + args = parser.parse_args() + + answer = asyncio.run(run_demo(offline=args.offline, provider=args.provider, db_url=args.db_url)) + print("\n" + "=" * 60) + print("AGENTKIT DEMO COMPLETED") + print("=" * 60) + print(answer) + print("=" * 60 + "\n") + + +if __name__ == "__main__": + main() diff --git a/scripts/seed_demo_db.sql b/scripts/seed_demo_db.sql new file mode 100644 index 0000000..0d07e88 --- /dev/null +++ b/scripts/seed_demo_db.sql @@ -0,0 +1,38 @@ +-- AgentKit Demo Database Seed Script +-- Populates demo products and orders tables for multi-tool agent demonstrations + +CREATE TABLE IF NOT EXISTS products ( + id INTEGER PRIMARY KEY, + name VARCHAR(100) NOT NULL, + category VARCHAR(50) NOT NULL, + price DECIMAL(10, 2) NOT NULL, + stock INTEGER NOT NULL DEFAULT 0 +); + +CREATE TABLE IF NOT EXISTS orders ( + id INTEGER PRIMARY KEY, + product_id INTEGER NOT NULL, + quantity INTEGER NOT NULL, + total_price DECIMAL(10, 2) NOT NULL, + order_date DATE NOT NULL, + customer_name VARCHAR(100) NOT NULL, + FOREIGN KEY (product_id) REFERENCES products(id) +); + +-- Seed Products +INSERT INTO products (id, name, category, price, stock) VALUES + (1, 'Pro Laptop 15"', 'Electronics', 1200.00, 25), + (2, 'Wireless Noise-Canceling Headphones', 'Electronics', 250.00, 50), + (3, 'Mechanical Keyboard', 'Electronics', 150.00, 80), + (4, 'Ergonomic Desk Chair', 'Furniture', 350.00, 15), + (5, 'Standing Desk Converter', 'Furniture', 280.00, 20), + (6, 'Designing Data-Intensive Applications', 'Books', 45.00, 100), + (7, 'Site Reliability Engineering', 'Books', 40.00, 120); + +-- Seed Orders (Electronics total in 2024 = $2,400 + $500 + $300 = $3,200.00) +INSERT INTO orders (id, product_id, quantity, total_price, order_date, customer_name) VALUES + (1, 1, 2, 2400.00, '2024-03-15', 'Alice Chen'), + (2, 2, 2, 500.00, '2024-04-10', 'Bob Smith'), + (3, 3, 2, 300.00, '2024-05-20', 'Charlie Brown'), + (4, 4, 1, 350.00, '2024-02-11', 'Diana Prince'), + (5, 6, 2, 90.00, '2024-01-25', 'Evan Wright'); diff --git a/tests/integration/test_demo.py b/tests/integration/test_demo.py index e03a4e0..74e9abb 100644 --- a/tests/integration/test_demo.py +++ b/tests/integration/test_demo.py @@ -2,20 +2,18 @@ from __future__ import annotations -import os from pathlib import Path -from typing import Any import pytest from sqlalchemy import text from sqlalchemy.ext.asyncio import create_async_engine from agentkit.core.agent import Agent, AgentConfig +from agentkit.core.types import RunStatus from agentkit.llm.fake import FakeLLMClient from agentkit.tools.builtin.calculator import calculator from agentkit.tools.builtin.sql_readonly import execute_sql_query -from agentkit.tools.models import ToolCall -from agentkit.tools.registry import ToolRegistry, tool +from agentkit.tools.registry import ToolRegistry @pytest.mark.asyncio @@ -70,61 +68,61 @@ async def test_multi_tool_end_to_end_demo_workflow(tmp_path: Path) -> None: registry = ToolRegistry() registry.register(calculator) - @tool async def run_sql(query: str) -> str: """Execute read-only SQL query.""" return await execute_sql_query(query, engine=engine) - registry.register(run_sql) + registry.register( + run_sql, + name="run_sql", + description="Execute read-only SQL query.", + ) # Setup FakeLLMClient multi-step script # Step 1: LLM decides to query SQL for total electronics sales # Step 2: LLM receives SQL results ($3200) and calculates sales tax (3200 * 1.085) # Step 3: LLM returns final synthesized answer - llm = FakeLLMClient( - responses=[ - '{"thought": "I need to query total sales for Electronics in 2024."}', - '{"thought": "Total sales is 3200. Now I calculate total with 8.5% tax."}', - "Total Electronics revenue in 2024 was $3,200.00. With 8.5% sales tax, the total is $3,472.00.", - ], - tool_calls_sequence=[ - [ - ToolCall( - name="run_sql", - arguments={ - "query": "SELECT SUM(total_price) as total_rev FROM orders o JOIN products p ON o.product_id = p.id WHERE p.category = 'Electronics'" - }, - ) - ], - [ - ToolCall( - name="calculator", - arguments={"expression": "3200 * 1.085"}, - ) - ], - [], - ], + llm = FakeLLMClient() + llm.queue_tool_call( + tool_name="run_sql", + arguments={ + "query": ( + "SELECT SUM(total_price) as total_rev FROM orders o " + "JOIN products p ON o.product_id = p.id WHERE p.category = 'Electronics'" + ) + }, + call_id="call_sql_1", + text='{"thought": "I need to query total sales for Electronics in 2024."}', + ) + llm.queue_tool_call( + tool_name="calculator", + arguments={"expression": "3200 * 1.085"}, + call_id="call_calc_1", + text='{"thought": "Total sales is 3200. Now I calculate total with 8.5% tax."}', + ) + llm.queue_text( + "Total Electronics revenue in 2024 was $3,200.00. With 8.5% sales tax, the total is $3,472.00." ) agent = Agent( llm=llm, - tools=registry, + registry=registry, config=AgentConfig(max_steps=5), ) prompt = "What was the total revenue from 'Electronics' category in 2024, and what is the total with 8.5% sales tax?" result = await agent.run(prompt) - assert result.success is True - assert result.answer is not None - assert "$3,472.00" in result.answer - assert len(result.steps) == 3 + assert result.status == RunStatus.SUCCEEDED + assert result.final_answer is not None + assert "$3,472.00" in result.final_answer + assert result.steps_count == 3 await engine.dispose() @pytest.mark.asyncio -async def test_demo_script_runner(monkeypatch: Any) -> None: +async def test_demo_script_runner() -> None: """Verify examples/demo.py main function executes successfully in offline mode.""" import sys from importlib import import_module