test(infra): verify PostgreSQL and Redis for real, and fix the rate-limit bypass it exposed

.env.example

@@ -23,7 +23,9 @@ SEARCH_API_KEY=your_search_api_key_here
 FILE_TOOL_BASE_DIR=/tmp/agentkit_sandbox
 
 # Security & API Authentication
-# Comma-separated list of authorized client API keys
+# Comma-separated list of authorized client API keys.
+# docker-compose.yml ships no default: leave this unset and the api service
+# exits on settings validation instead of starting with a published key.
 API_KEYS=ak_test_key_12345,ak_admin_key_67890
 RATE_LIMIT_PER_MIN=30
 
@@ -33,3 +35,10 @@ RUN_TIMEOUT_S=60
 TOOL_TIMEOUT_S=15
 SESSION_TTL_S=3600
 TOOL_OUTPUT_MAX_CHARS=2000
+
+# Real-infrastructure test suite (optional, test-only)
+# `pytest` skips the live tier unless AGENTKIT_REAL_INFRA=1. Point these at a
+# DEDICATED test database: the suite truncates runs/steps.
+# AGENTKIT_REAL_INFRA=1
+# DATABASE_TEST_URL=postgresql+asyncpg://postgres:postgres@localhost:5432/agentkit_test
+# REDIS_TEST_URL=redis://localhost:6379/15

.github/workflows/ci.yml

@@ -39,13 +39,54 @@ jobs:
         run: mypy agentkit tests
 
   test:
-    name: Test (Python ${{ matrix.python-version }})
+    name: Test Hermetic (Python ${{ matrix.python-version }})
     runs-on: ubuntu-latest
     needs: [lint]
     strategy:
       fail-fast: false
       matrix:
         python-version: ["3.11", "3.12"]
+    env:
+      ENVIRONMENT: test
+      DATABASE_URL: postgresql+asyncpg://agentkit:agentkit_password@localhost:5432/agentkit_test
+      REDIS_URL: redis://localhost:6379/0
+      GEMINI_API_KEY: fake_gemini_key_for_testing
+      OPENAI_API_KEY: fake_openai_key_for_testing
+      TAVILY_API_KEY: fake_tavily_key_for_testing
+      API_KEYS: test_api_key_1,test_api_key_2
+      JWT_SECRET: test_secret_key_at_least_32_bytes_long
+      PYTHONPATH: .
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: "pip"
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e ".[dev]"
+
+      - name: Run pytest with coverage
+        run: |
+          pytest -v --cov=agentkit --cov-report=term-missing --cov-report=xml --cov-fail-under=85
+
+      - name: Upload coverage artifact
+        uses: actions/upload-artifact@v4
+        if: matrix.python-version == '3.12'
+        with:
+          name: coverage-report
+          path: coverage.xml
+
+  test-real-infra:
+    name: Test Against Live PostgreSQL and Redis
+    runs-on: ubuntu-latest
+    needs: [lint]
+    # Serialises this suite because every run truncates shared service containers.
     services:
       postgres:
         image: postgres:16-alpine
@@ -71,40 +112,28 @@ jobs:
           --health-retries 5
     env:
       ENVIRONMENT: test
-      DATABASE_URL: postgresql+asyncpg://agentkit:agentkit_password@localhost:5432/agentkit_test
+      AGENTKIT_REAL_INFRA: "1"
       DATABASE_TEST_URL: postgresql+asyncpg://agentkit:agentkit_password@localhost:5432/agentkit_test
-      REDIS_URL: redis://localhost:6379/0
-      GEMINI_API_KEY: fake_gemini_key_for_testing
-      OPENAI_API_KEY: fake_openai_key_for_testing
-      TAVILY_API_KEY: fake_tavily_key_for_testing
+      REDIS_TEST_URL: redis://localhost:6379/15
       API_KEYS: test_api_key_1,test_api_key_2
-      JWT_SECRET: test_secret_key_at_least_32_bytes_long
       PYTHONPATH: .
     steps:
       - name: Check out repository
         uses: actions/checkout@v4
 
-      - name: Set up Python ${{ matrix.python-version }}
+      - name: Set up Python 3.12
         uses: actions/setup-python@v5
         with:
-          python-version: ${{ matrix.python-version }}
+          python-version: "3.12"
           cache: "pip"
 
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
           pip install -e ".[dev]"
 
-      - name: Run Alembic migrations
-        run: alembic upgrade head
-
-      - name: Run pytest with coverage
-        run: |
-          pytest -v --cov=agentkit --cov-report=term-missing --cov-report=xml --cov-fail-under=85
-
-      - name: Upload coverage artifact
-        uses: actions/upload-artifact@v4
-        if: matrix.python-version == '3.12'
-        with:
-          name: coverage-report
-          path: coverage.xml
+      # The suite exercises native UUID/JSON/timestamptz columns, FK cascade deletes
+      # and Lua-backed rate limiting, none of which the SQLite and fakeredis based
+      # unit tests can validate.
+      - name: Run real infrastructure tests
+        run: pytest -v -m real_infra tests/real

README.md

@@ -118,8 +118,14 @@ JWT_SECRET=super_secret_jwt_key_at_least_32_bytes_long
 ```
 
 ### 4. Running with Docker Compose (Recommended)
-Launch the API, PostgreSQL, and Redis together with automatic migrations:
+Launch the API, PostgreSQL, and Redis together with automatic migrations.
+
+`docker-compose.yml` ships no default `API_KEYS`. Leave it unset and the api service
+fails immediately on settings validation rather than starting with a published key;
+`down`, `ps` and `logs` still work either way.
+
 ```bash
+export API_KEYS=ak_local_dev_key   # or place API_KEYS in your .env
 docker compose up -d --build
 
 # Verify all services are healthy
@@ -135,11 +141,20 @@ curl -s http://localhost:8000/health | jq .
 AgentKit includes a complete multi-tool demo script that seeds a database and combines SQL analysis with mathematical calculation:
 
 ```bash
-# Completely offline demo (using FakeLLMClient - no API keys required!)
+# Against real infrastructure (PostgreSQL + Redis from your .env).
+# Run and step records are persisted exactly as they would be in production.
+docker compose up -d postgres redis
+alembic upgrade head
+python examples/demo.py --provider gemini
+
+# Or completely offline (FakeLLMClient + in-memory SQLite - no services, no API keys)
 python examples/demo.py --offline
+```
 
-# Or live with Gemini:
-python examples/demo.py --provider gemini
+The demo prints the run ID; you can then inspect what it wrote:
+
+```bash
+curl -s -H "X-API-Key: $API_KEYS" "http://localhost:8000/runs/<run_id>/trace" | jq .
 ```
 
 ---
@@ -288,7 +303,7 @@ Complete step-by-step instructions, including security group rules, systemd serv
 AgentKit enforces strict code quality and comprehensive test coverage across all layers:
 
 ```bash
-# Run the complete test suite
+# Run the complete hermetic test suite (no services required)
 pytest -v
 
 # Run with test coverage report (enforcing >=85% threshold)
@@ -302,7 +317,43 @@ ruff format --check .
 mypy agentkit tests examples
 ```
 
-All pull requests trigger the automated GitHub Actions CI matrix running on Python 3.11 and 3.12 with live PostgreSQL 16 and Redis 7 service containers.
+### Two test tiers
+
+The default suite is hermetic: SQLite, `fakeredis` and `FakeLLMClient` stand in for
+PostgreSQL, Redis and the LLM providers, so `pytest` runs in about two seconds with no
+services. A second, opt-in tier runs the same components against **live** PostgreSQL and
+Redis, covering behaviour the hermetic tier structurally cannot assert:
+
+- native `uuid` / `json` / `timestamptz` columns and asyncpg parameter binding
+- real foreign-key enforcement and `ON DELETE CASCADE` (SQLite ignores FKs by default)
+- Alembic migrations applied through the real PostgreSQL dialect
+- genuine Redis TTL expiry and the hiredis codec
+- the Lua-backed atomic rate limiter under concurrency
+
+```bash
+# Start the services the tier needs
+docker compose up -d postgres redis
+
+# Run only the real-infrastructure suite
+AGENTKIT_REAL_INFRA=1 pytest -m real_infra tests/real
+
+# Or point it at any other PostgreSQL / Redis
+AGENTKIT_REAL_INFRA=1 \
+  DATABASE_TEST_URL=postgresql+asyncpg://user:pass@localhost:5432/agentkit_test \
+  REDIS_TEST_URL=redis://localhost:6379/15 \
+  pytest -m real_infra tests/real
+```
+
+Without `AGENTKIT_REAL_INFRA=1` these tests skip, so the default `pytest` invocation stays
+hermetic. The suite truncates the `runs` and `steps` tables and deletes only
+`session:*` / `ratelimit:*` Redis keys, so point it at a dedicated test database.
+
+### CI
+
+All pull requests trigger GitHub Actions with three jobs: `Lint & Type Check`; a hermetic
+`Test Hermetic` matrix across Python 3.11 and 3.12 with coverage gating; and
+`Test Against Live PostgreSQL and Redis`, which runs the `real_infra` tier against real
+PostgreSQL 16 and Redis 7 service containers.
 
 ---
 

agentkit/api/deps.py

@@ -12,13 +12,16 @@
 from agentkit.api.auth import get_current_settings
 from agentkit.config import Settings
 from agentkit.core.agent import Agent, AgentConfig
+from agentkit.core.errors import ServiceUnavailableError
 from agentkit.core.pg_trace import PostgresTraceSink
 from agentkit.core.trace import InMemoryTraceSink, TraceSink
 from agentkit.llm.base import LLMClient
 from agentkit.llm.factory import create_llm_client_from_settings
 from agentkit.llm.fake import FakeLLMClient
 from agentkit.memory.base import InMemoryMemoryStore, MemoryStore
 from agentkit.memory.pg_store import PostgresMemoryStore
+from agentkit.memory.redis_store import RedisMemoryStore
+from agentkit.memory.tiered import TieredMemoryStore
 from agentkit.tools.builtin.calculator import calculator
 from agentkit.tools.builtin.http_fetch import http_fetch
 from agentkit.tools.builtin.read_file import read_file
@@ -29,21 +32,27 @@
 logger = logging.getLogger(__name__)
 
 
-def get_session_factory(
-    request: Request,
-) -> async_sessionmaker[AsyncSession] | None:
+def get_session_factory(request: Request) -> async_sessionmaker[AsyncSession] | None:
     """Retrieve database session factory from app state."""
     return getattr(request.app.state, "session_factory", None)
 
 
 async def get_db_session(
-    request: Request,
+    factory: async_sessionmaker[AsyncSession] | None = Depends(get_session_factory),
 ) -> AsyncIterator[AsyncSession]:
-    """Provide a scoped database session."""
-    factory = get_session_factory(request)
-    if factory is not None:
-        async with factory() as session:
-            yield session
+    """Yield a scoped database session.
+
+    Fails explicitly when the application has no session factory. Returning without
+    yielding would leave FastAPI's dependency solver with an exhausted generator and
+    surface as an opaque ``generator didn't yield`` RuntimeError at request time.
+    """
+    if factory is None:
+        raise ServiceUnavailableError(
+            "database",
+            "Database is not configured. The session factory is absent from application state.",
+        )
+    async with factory() as session:
+        yield session
 
 
 def get_redis_client(request: Request) -> Redis | None:
@@ -58,10 +67,23 @@ def get_memory_store(request: Request) -> PostgresMemoryStore | MemoryStore:
         return store
 
     factory = get_session_factory(request)
-    if factory is not None:
-        return PostgresMemoryStore(factory)
+    redis_client = get_redis_client(request)
+    settings: Settings = getattr(request.app.state, "settings", None) or get_current_settings(
+        request
+    )
+
+    run_store: PostgresMemoryStore | InMemoryMemoryStore = (
+        PostgresMemoryStore(factory) if factory is not None else InMemoryMemoryStore()
+    )
+
+    if redis_client is not None:
+        session_store = RedisMemoryStore(redis_client, ttl_s=settings.SESSION_TTL_S)
+        return TieredMemoryStore(session_store=session_store, run_store=run_store)
+
+    if isinstance(run_store, PostgresMemoryStore):
+        return TieredMemoryStore(session_store=InMemoryMemoryStore(), run_store=run_store)
 
-    return InMemoryMemoryStore()
+    return run_store
 
 
 def get_trace_sink(request: Request) -> TraceSink:

agentkit/api/main.py

@@ -9,7 +9,6 @@
 from fastapi.exceptions import RequestValidationError
 from fastapi.responses import JSONResponse
 from redis.asyncio import Redis
-from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
 
 from agentkit.api.routes import discovery_router
 from agentkit.api.routes import router as runs_router
@@ -22,8 +21,10 @@
     RateLimitExceededError,
     RunNotFoundError,
     RunTimeoutError,
+    ServiceUnavailableError,
     ToolNotFoundError,
 )
+from agentkit.db.session import create_engine, create_session_factory
 
 logger = logging.getLogger(__name__)
 
@@ -40,6 +41,8 @@ def map_agentkit_error_status(exc: AgentKitError) -> int:
         return 429
     if isinstance(exc, LLMProviderError):
         return 502
+    if isinstance(exc, ServiceUnavailableError):
+        return 503
     return 400
 
 
@@ -48,8 +51,8 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
     """Manage lifecycle of database engine, session factory, and Redis connection pool."""
     settings: Settings = getattr(app.state, "settings", None) or get_settings()
 
-    engine = create_async_engine(settings.DATABASE_URL, pool_pre_ping=True)
-    session_factory = async_sessionmaker(bind=engine, class_=AsyncSession, expire_on_commit=False)
+    engine = create_engine(settings.DATABASE_URL, pool_pre_ping=True)
+    session_factory = create_session_factory(engine)
     redis_client: Redis[Any] = Redis.from_url(settings.REDIS_URL, decode_responses=True)
 
     app.state.db_engine = engine