diff --git a/.env_example b/.env_example index 0f195ff..f2df8f5 100644 --- a/.env_example +++ b/.env_example @@ -12,3 +12,5 @@ RAG_USE_REAL_MODELS=0 # Set to 1 to try loading the MS MARCO dataset from Hugging Face. RAG_USE_REAL_DATASET=0 +# set the path for metrics +LOG_PATH=app/logs/metrics.json \ No newline at end of file diff --git a/.gitignore b/.gitignore index bfb3e2f..6e99023 100644 --- a/.gitignore +++ b/.gitignore @@ -44,6 +44,7 @@ coverage.xml *.py,cover .hypothesis/ .pytest_cache/ +.pytest_tmp/ # Translations *.mo diff --git a/backend/api/routes.py b/backend/api/routes.py index 04856ff..2ded301 100644 --- a/backend/api/routes.py +++ b/backend/api/routes.py @@ -1,4 +1,5 @@ import time +from datetime import datetime, timezone from functools import lru_cache from typing import Any @@ -7,6 +8,7 @@ from backend.rag.internet_rag import InternetRag from backend.rag.neural_rag import NeuralRag +from backend.utils.logger import log_request from backend.utils.metrics import resource_delta, resource_snapshot, timed_call @@ -43,6 +45,49 @@ def get_internet_rag() -> InternetRag: return InternetRag() +def _metric_timestamp() -> str: + return datetime.now(timezone.utc).replace(microsecond=0, tzinfo=None).isoformat() + + +def _write_request_metrics( + rag_type: str, + query: str, + metrics: dict[str, float], + total_latency: float, + resource_before: dict[str, float], + resource_after: dict[str, float], + status: str = "success", +) -> None: + resources = resource_delta(resource_before, resource_after) + log_request( + { + "timestamp": _metric_timestamp(), + "rag_type": rag_type, + "query": query, + "retrieval_time": metrics.get("retrieval_time", 0.0), + "llm_time": metrics.get("llm_time", 0.0), + "total_latency": total_latency, + "cpu_usage": resources["cpu_percent_avg"], + "memory_usage": resources["system_memory_percent_avg"], + "status": status, + } + ) + + +def _run_neural_rag(request: QueryRequest) -> RagResponse: + rag = get_neural_rag() + result, elapsed = timed_call(rag.answer, request.query, k=request.k, answer_len=request.answer_len) + result["metrics"]["total_time"] = elapsed + return RagResponse(**result) + + +def _run_internet_rag(request: QueryRequest) -> RagResponse: + rag = get_internet_rag() + result, elapsed = timed_call(rag.answer, request.query, max_results=request.k, answer_len=request.answer_len) + result["metrics"]["total_time"] = elapsed + return RagResponse(**result) + + @router.get("/health") def health() -> dict[str, str]: return {"status": "ok"} @@ -50,29 +95,73 @@ def health() -> dict[str, str]: @router.post("/rag/neural", response_model=RagResponse) def neural_rag(request: QueryRequest) -> RagResponse: - rag = get_neural_rag() - result, elapsed = timed_call(rag.answer, request.query, k=request.k, answer_len=request.answer_len) - result["metrics"]["total_time"] = elapsed - return RagResponse(**result) + resource_before = resource_snapshot() + try: + response = _run_neural_rag(request) + except Exception: + resource_after = resource_snapshot() + _write_request_metrics("neural", request.query, {}, 0.0, resource_before, resource_after, "error") + raise + + resource_after = resource_snapshot() + _write_request_metrics( + "neural", + request.query, + response.metrics, + response.metrics["total_time"], + resource_before, + resource_after, + ) + return response @router.post("/rag/internet", response_model=RagResponse) def internet_rag(request: QueryRequest) -> RagResponse: - rag = get_internet_rag() - result, elapsed = timed_call(rag.answer, request.query, max_results=request.k, answer_len=request.answer_len) - result["metrics"]["total_time"] = elapsed - return RagResponse(**result) + resource_before = resource_snapshot() + try: + response = _run_internet_rag(request) + except Exception: + resource_after = resource_snapshot() + _write_request_metrics("internet", request.query, {}, 0.0, resource_before, resource_after, "error") + raise + + resource_after = resource_snapshot() + _write_request_metrics( + "internet", + request.query, + response.metrics, + response.metrics["total_time"], + resource_before, + resource_after, + ) + return response @router.post("/rag/compare", response_model=CompareResponse) def compare_rag(request: QueryRequest) -> CompareResponse: resource_before = resource_snapshot() elapsed_start = time.time() - neural = neural_rag(request) - internet = internet_rag(request) - total_time = time.time() - elapsed_start + try: + neural = _run_neural_rag(request) + internet = _run_internet_rag(request) + total_time = time.time() - elapsed_start + except Exception: + resource_after = resource_snapshot() + _write_request_metrics("compare", request.query, {}, 0.0, resource_before, resource_after, "error") + raise resource_after = resource_snapshot() metrics = resource_delta(resource_before, resource_after) metrics["total_time"] = total_time + _write_request_metrics( + "compare", + request.query, + { + "retrieval_time": neural.metrics["retrieval_time"] + internet.metrics["retrieval_time"], + "llm_time": neural.metrics["llm_time"] + internet.metrics["llm_time"], + }, + total_time, + resource_before, + resource_after, + ) return CompareResponse(neural=neural, internet=internet, metrics=metrics) diff --git a/backend/tests/test_services.py b/backend/tests/test_services.py index e30ea87..969a450 100644 --- a/backend/tests/test_services.py +++ b/backend/tests/test_services.py @@ -1,5 +1,8 @@ -import os +import json +from pathlib import Path + import numpy as np +import pytest import requests from fastapi.testclient import TestClient @@ -16,6 +19,18 @@ client = TestClient(app) +@pytest.fixture(autouse=True) +def metrics_log_path(monkeypatch, tmp_path): + log_path = tmp_path / "logs" / "metrics.json" + log_path.parent.mkdir(parents=True, exist_ok=True) + monkeypatch.setattr("backend.utils.logger.LOG_PATH", str(log_path)) + return log_path + + +def read_metric_lines(path): + return [json.loads(line) for line in Path(path).read_text().splitlines()] + + def test_neural_rag_smoke(): result = NeuralRag().answer("What is machine learning?", k=2) assert result["method"] == "neural" @@ -132,6 +147,34 @@ def test_neural_endpoint(): assert body["contexts"] assert {"retrieval_time", "llm_time", "context_size", "total_time"} <= set(body["metrics"]) + +def test_neural_endpoint_writes_request_metrics(monkeypatch, tmp_path): + log_path = tmp_path / "logs" / "metrics.json" + monkeypatch.setattr("backend.utils.logger.LOG_PATH", str(log_path)) + + response = client.post("/api/rag/neural", json={"query": "What is machine learning?", "k": 2}) + + assert response.status_code == 200 + metrics = read_metric_lines(log_path) + assert len(metrics) == 1 + assert { + "timestamp", + "rag_type", + "query", + "retrieval_time", + "llm_time", + "total_latency", + "cpu_usage", + "memory_usage", + "status", + } <= set(metrics[0]) + assert metrics[0]["rag_type"] == "neural" + assert metrics[0]["query"] == "What is machine learning?" + assert metrics[0]["status"] == "success" + assert metrics[0]["retrieval_time"] >= 0 + assert metrics[0]["llm_time"] >= 0 + assert metrics[0]["total_latency"] >= 0 + # Test that the internet endpoint returns an answer and contexts with metrics def test_internet_endpoint(): response = client.post("/api/rag/internet", json={"query": "What is machine learning?", "k": 2}) @@ -142,6 +185,23 @@ def test_internet_endpoint(): assert body["contexts"] assert {"retrieval_time", "llm_time", "context_size", "total_time"} <= set(body["metrics"]) + +def test_internet_endpoint_writes_request_metrics(monkeypatch, tmp_path): + log_path = tmp_path / "logs" / "metrics.json" + monkeypatch.setattr("backend.utils.logger.LOG_PATH", str(log_path)) + + response = client.post("/api/rag/internet", json={"query": "What is machine learning?", "k": 2}) + + assert response.status_code == 200 + metrics = read_metric_lines(log_path) + assert len(metrics) == 1 + assert metrics[0]["rag_type"] == "internet" + assert metrics[0]["query"] == "What is machine learning?" + assert metrics[0]["status"] == "success" + assert metrics[0]["retrieval_time"] >= 0 + assert metrics[0]["llm_time"] >= 0 + assert metrics[0]["total_latency"] >= 0 + # Test that the compare endpoint returns both neural and internet results with metrics def test_compare_endpoint_includes_resource_metrics(): response = client.post("/api/rag/compare", json={"query": "What is machine learning?", "k": 2}) @@ -153,6 +213,23 @@ def test_compare_endpoint_includes_resource_metrics(): assert "memory_rss_mb_after" in body["metrics"] assert "system_memory_percent_avg" in body["metrics"] + +def test_compare_endpoint_writes_single_request_metrics(monkeypatch, tmp_path): + log_path = tmp_path / "logs" / "metrics.json" + monkeypatch.setattr("backend.utils.logger.LOG_PATH", str(log_path)) + + response = client.post("/api/rag/compare", json={"query": "What is machine learning?", "k": 2}) + + assert response.status_code == 200 + metrics = read_metric_lines(log_path) + assert len(metrics) == 1 + assert metrics[0]["rag_type"] == "compare" + assert metrics[0]["query"] == "What is machine learning?" + assert metrics[0]["status"] == "success" + assert metrics[0]["retrieval_time"] >= 0 + assert metrics[0]["llm_time"] >= 0 + assert metrics[0]["total_latency"] >= 0 + # Additional tests for query validation and edge cases def test_query_validation(): response = client.post("/api/rag/neural", json={"query": "", "k": 2}) diff --git a/backend/utils/logger.py b/backend/utils/logger.py index e424b17..49e549e 100644 --- a/backend/utils/logger.py +++ b/backend/utils/logger.py @@ -1,6 +1,24 @@ +import json import logging import os from pathlib import Path +import json +from datetime import datetime + + +BASE_DIR = Path(__file__).resolve().parent.parent.parent +LOG_DIR = BASE_DIR / "logs" +LOG_DIR.mkdir(parents=True, exist_ok=True) + +LOG_PATH = LOG_DIR / "metrics.json" + + +def log_request(data: dict): + with open(LOG_PATH, "a") as f: + f.write(json.dumps({ + "timestamp": datetime.utcnow().isoformat(), + **data + }) + "\n") # Utility functions for environment variable loading and logging setup diff --git a/conftest.py b/conftest.py new file mode 100644 index 0000000..e642bb6 --- /dev/null +++ b/conftest.py @@ -0,0 +1,29 @@ +"""Project-wide pytest configuration. + +Two things this file handles before any test runs: + +* Force the offline / deterministic code paths for the RAG services so tests do + not download models, stream datasets, or call external APIs. Backend modules + call ``load_project_env()`` at import time which reads ``.env`` only for keys + that are NOT already set, so these assignments must happen before any + ``backend.*`` module is imported. +* Pin pytest's ``basetemp`` to a workspace-local folder. The default Windows + temp directory is locked on some workstations, which makes ``tmp_path`` raise + ``PermissionError`` during fixture setup. +""" + +import os +from pathlib import Path + + +os.environ["RAG_USE_REAL_MODELS"] = "0" +os.environ["RAG_USE_REAL_DATASET"] = "0" +os.environ["TAVILY_API_KEY"] = "" + + +ROOT = Path(__file__).resolve().parent + + +def pytest_configure(config): + if not config.option.basetemp: + config.option.basetemp = str(ROOT / ".pytest_tmp") diff --git a/docker/Dockerfile b/docker/Dockerfile index 0980e1d..02f537a 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -15,6 +15,12 @@ RUN useradd --create-home --shell /usr/sbin/nologin appuser COPY --chown=appuser:appuser backend ./backend +# Create writable logs directory +RUN mkdir -p /app/logs + +# Give ownership to appuser +RUN chown -R appuser:appuser /app + USER appuser EXPOSE 8000 diff --git a/requirements.txt b/requirements.txt index 4a4fba8..ccf6119 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,3 +5,5 @@ numpy>=2.0.0 psutil>=7.0.0 requests>=2.32.0 beautifulsoup4>=4.12.0 +pytest>=7.0.0 +pytest-asyncio>=0.20.0 \ No newline at end of file