diff --git a/src/gitmate/cli.py b/src/gitmate/cli.py index 336cbe3..fea1192 100644 --- a/src/gitmate/cli.py +++ b/src/gitmate/cli.py @@ -40,6 +40,83 @@ def commit( raise typer.Exit(code) +@app.command() +def stats( + days: int | None = typer.Option(None, "--days", "-d", help="Limit stats to the last N days."), + month: bool = typer.Option( + False, "--month", "-m", help="Show stats for current calendar month." + ), + raw: bool = typer.Option( + False, "--raw", help="Output raw JSON for scripting or README tables." + ), +) -> None: + """Display telemetry metrics, cache hit rate, token usage, and estimated spend.""" + import json + from dataclasses import asdict + from datetime import UTC, datetime, timedelta + + from gitmate.metrics import get_aggregate_stats + + since: datetime | None = None + now = datetime.now(UTC) + if month: + since = datetime(now.year, now.month, 1, tzinfo=UTC) + elif days is not None: + if days <= 0: + raise typer.BadParameter("--days must be a positive integer.") + since = now - timedelta(days=days) + + stats_data = get_aggregate_stats(since=since) + + if raw: + typer.echo(json.dumps(asdict(stats_data), indent=2)) + return + + if stats_data.total_invocations == 0: + console.print( + "[yellow]No invocations recorded yet. Run 'gitmate commit' to start collecting metrics.[/yellow]" + ) + return + + time_label = "Current Month" if month else (f"Last {days} Days" if days else "All Time") + table = Table(title=f"gitmate stats ({time_label})", show_header=True) + table.add_column("Metric", style="bold") + table.add_column("Value", justify="right") + + table.add_row("Total Invocations", str(stats_data.total_invocations)) + table.add_row("Cache Hits", f"{stats_data.cache_hits} ({stats_data.cache_hit_rate:.1f}%)") + table.add_row( + "Tokens (In / Out / Total)", + f"{stats_data.total_tokens_in:,} / {stats_data.total_tokens_out:,} / {stats_data.total_tokens:,}", + ) + table.add_row("Estimated Spend", f"${stats_data.total_cost_usd:.4f}") + table.add_row("Avg Latency (Overall)", f"{stats_data.avg_latency_ms:.0f} ms") + table.add_row("Avg Latency (Cache Hit)", f"{stats_data.avg_latency_hit_ms:.0f} ms") + table.add_row("Avg Latency (Cache Miss)", f"{stats_data.avg_latency_miss_ms:.0f} ms") + table.add_row("Fallback Templates Used", str(stats_data.fallback_count)) + + console.print(table) + + if stats_data.by_command: + cmd_table = Table(title="By Command", show_header=True) + cmd_table.add_column("Command") + cmd_table.add_column("Runs", justify="right") + cmd_table.add_column("Hit Rate", justify="right") + cmd_table.add_column("Tokens", justify="right") + cmd_table.add_column("Spend", justify="right") + cmd_table.add_column("Avg Latency", justify="right") + for cmd, b in sorted(stats_data.by_command.items()): + cmd_table.add_row( + cmd, + str(b.invocations), + f"{b.cache_hit_rate:.1f}%", + f"{b.total_tokens:,}", + f"${b.cost_usd:.4f}", + f"{b.avg_latency_ms:.0f} ms", + ) + console.print(cmd_table) + + @app.command() def pr_summary() -> None: """Generate a PR description for the current branch.""" diff --git a/src/gitmate/committer.py b/src/gitmate/committer.py index 297a5bf..2b7606d 100644 --- a/src/gitmate/committer.py +++ b/src/gitmate/committer.py @@ -19,7 +19,7 @@ from gitmate.config import GitmateConfig from gitmate.diff_extractor import DiffExtractor, GitCommandError from gitmate.fallback import generate_commit_message -from gitmate.metrics import record_invocation +from gitmate.metrics import get_monthly_spend, record_invocation from gitmate.providers.base import LLMProvider from gitmate.token_budget import TokenCounter @@ -39,6 +39,7 @@ def _safe_record_telemetry( cache_hit: bool, latency_ms: int | None, fallback_used: bool, + estimated_cost_usd: float | None = None, ) -> None: """Record invocation to metrics database without propagating errors.""" try: @@ -50,6 +51,7 @@ def _safe_record_telemetry( cache_hit=cache_hit, latency_ms=latency_ms, fallback_used=fallback_used, + estimated_cost_usd=estimated_cost_usd, ) except Exception as exc: # noqa: BLE001 logger.warning("Failed to record telemetry: %s", exc) @@ -232,6 +234,23 @@ def commit_flow( ) return 1 + # 3. Monthly budget cap enforcement + if active_cfg.budget_cap_usd is not None: + monthly_spend = get_monthly_spend() + if monthly_spend >= active_cfg.budget_cap_usd: + con.print( + f"[red]error:[/red] Monthly budget cap exceeded " + f"(${monthly_spend:.2f} >= ${active_cfg.budget_cap_usd:.2f}). " + "Aborting to prevent further LLM spend.\n" + "To adjust or remove the cap, run: gitmate config set budget_cap_usd " + ) + return 1 + if monthly_spend >= 0.8 * active_cfg.budget_cap_usd: + con.print( + f"[yellow]warning: Monthly spend has reached {int(monthly_spend / active_cfg.budget_cap_usd * 100)}% " + f"of budget cap (${monthly_spend:.2f} / ${active_cfg.budget_cap_usd:.2f}).[/yellow]" + ) + active_counter = counter if active_counter is None and provider is not None: @@ -241,7 +260,7 @@ def count_tokens(self, text: str, model: str) -> int: active_counter = _SimpleCounter() - # 3. Initial generation & telemetry recording + # 4. Initial generation & telemetry recording t0 = time.perf_counter() gen_result = generate_commit_message( diffs=staged_diffs, @@ -262,18 +281,19 @@ def count_tokens(self, text: str, model: str) -> int: cache_hit=gen_result.cache_hit, latency_ms=latency_ms, fallback_used=gen_result.is_fallback, + estimated_cost_usd=gen_result.estimated_cost_usd, ) current_message = gen_result.text current_model = gen_result.model current_is_fallback = gen_result.is_fallback - # 4. Fast path: --yes with allow_noninteractive_commit enabled + # 5. Fast path: --yes with allow_noninteractive_commit enabled if yes: code = _execute_commit(current_message, console=con, cwd=repo_dir) return code - # 5. Interactive review loop + # 6. Interactive review loop _render_panel(con, current_message, current_model, current_is_fallback) while True: @@ -305,6 +325,22 @@ def count_tokens(self, text: str, model: str) -> int: _render_panel(con, current_message, current_model, current_is_fallback) elif choice in ("r", "regenerate"): + if active_cfg.budget_cap_usd is not None: + monthly_spend = get_monthly_spend() + if monthly_spend >= active_cfg.budget_cap_usd: + con.print( + f"[red]error:[/red] Monthly budget cap exceeded " + f"(${monthly_spend:.2f} >= ${active_cfg.budget_cap_usd:.2f}). " + "Aborting to prevent further LLM spend.\n" + "To adjust or remove the cap, run: gitmate config set budget_cap_usd " + ) + continue + if monthly_spend >= 0.8 * active_cfg.budget_cap_usd: + con.print( + f"[yellow]warning: Monthly spend has reached {int(monthly_spend / active_cfg.budget_cap_usd * 100)}% " + f"of budget cap (${monthly_spend:.2f} / ${active_cfg.budget_cap_usd:.2f}).[/yellow]" + ) + con.print("Regenerating commit message with fresh API call...") t0 = time.perf_counter() gen_result = generate_commit_message( @@ -326,6 +362,7 @@ def count_tokens(self, text: str, model: str) -> int: cache_hit=False, latency_ms=latency_ms, fallback_used=gen_result.is_fallback, + estimated_cost_usd=gen_result.estimated_cost_usd, ) current_message = gen_result.text diff --git a/src/gitmate/fallback.py b/src/gitmate/fallback.py index 8bac2a5..9593ed8 100644 --- a/src/gitmate/fallback.py +++ b/src/gitmate/fallback.py @@ -11,6 +11,7 @@ from gitmate.chunking import summarize_omitted_diffs from gitmate.config import API_KEY_ACCOUNT, GitmateConfig from gitmate.diff_extractor import FileDiff +from gitmate.metrics import calculate_cost from gitmate.prompt import load_template, render_commit_prompt from gitmate.providers.base import LLMProvider, ProviderUnavailable from gitmate.providers.cache import CachedProvider, cache_key @@ -54,6 +55,7 @@ class GenerationResult: input_tokens: int | None = None output_tokens: int | None = None cache_hit: bool = False + estimated_cost_usd: float = 0.0 def _extract_scope(path_str: str) -> str | None: @@ -242,7 +244,7 @@ def generate_commit_message( active_provider: LLMProvider if provider is not None: if bypass_cache and isinstance(provider, CachedProvider): - active_provider = provider._provider + active_provider = provider.underlying else: active_provider = provider else: @@ -273,10 +275,11 @@ def generate_commit_message( ) # 2. Generation / Chunk-and-summarize with graceful fallback on provider failure - try: - total_input_tokens = 0 - total_output_tokens = 0 + total_input_tokens = 0 + total_output_tokens = 0 + chunk_cost = 0.0 + try: if decision.strategy is BudgetStrategy.NEEDS_CHUNKING: if not decision.omitted_diffs: if console is not None: @@ -298,6 +301,14 @@ def generate_commit_message( ) total_input_tokens += chunk_in total_output_tokens += chunk_out + if chunk_in or chunk_out: + chunk_cost = calculate_cost( + model=cfg.model, + tokens_in=chunk_in or None, + tokens_out=chunk_out or None, + cache_hit=False, + fallback_used=False, + ) summary_note = ( f"{decision.summary_note}\n\nSummaries of large omitted files:\n{merged_summaries}" @@ -327,6 +338,15 @@ def generate_commit_message( if resp.output_tokens is not None: total_output_tokens += resp.output_tokens + final_cost = calculate_cost( + model=resp.model, + tokens_in=resp.input_tokens, + tokens_out=resp.output_tokens, + cache_hit=cache_hit, + fallback_used=False, + ) + cost = round(chunk_cost + final_cost, 6) + return GenerationResult( text=resp.text, is_fallback=False, @@ -334,6 +354,7 @@ def generate_commit_message( input_tokens=total_input_tokens or None, output_tokens=total_output_tokens or None, cache_hit=cache_hit, + estimated_cost_usd=cost, ) except ProviderUnavailable as exc: @@ -345,5 +366,8 @@ def generate_commit_message( text=fallback_msg, is_fallback=True, model=cfg.model, + input_tokens=total_input_tokens or None, + output_tokens=total_output_tokens or None, + estimated_cost_usd=chunk_cost, fallback_reason=str(exc), ) diff --git a/src/gitmate/metrics.py b/src/gitmate/metrics.py index e7cc77b..0a82a3d 100644 --- a/src/gitmate/metrics.py +++ b/src/gitmate/metrics.py @@ -16,6 +16,19 @@ DEFAULT_DB_FILENAME = "metrics.db" METRICS_DIR_ENV_VAR = "GITMATE_METRICS_DIR" +#: Model pricing in USD per 1,000,000 tokens (input_rate, output_rate). +MODEL_PRICING: dict[str, tuple[float, float]] = { + "gemini-3.5-flash-lite": (0.30, 2.50), + "gemini-flash-lite": (0.30, 2.50), + "gemini-3.5-flash": (0.50, 3.00), + "gemini-3-flash": (0.50, 3.00), + "gemini-3-flash-preview": (0.50, 3.00), + "gemini-flash": (0.50, 3.00), + "claude-3-5-haiku": (0.80, 4.00), + "claude-3-5-sonnet": (3.00, 15.00), +} +DEFAULT_PRICING: tuple[float, float] = (0.50, 3.00) + CREATE_TABLE_SQL = """ CREATE TABLE IF NOT EXISTS invocations ( id INTEGER PRIMARY KEY AUTOINCREMENT, @@ -26,7 +39,8 @@ tokens_out INTEGER, cache_hit INTEGER NOT NULL, latency_ms INTEGER, - fallback_used INTEGER NOT NULL + fallback_used INTEGER NOT NULL, + estimated_cost_usd REAL ); CREATE INDEX IF NOT EXISTS idx_invocations_timestamp ON invocations(timestamp); CREATE INDEX IF NOT EXISTS idx_invocations_command ON invocations(command); @@ -46,6 +60,67 @@ class InvocationRecord: cache_hit: bool latency_ms: int | None fallback_used: bool + estimated_cost_usd: float = 0.0 + + +@dataclass(frozen=True) +class BreakdownStats: + """Aggregated metrics for a specific subset (e.g. command or model).""" + + invocations: int + cache_hits: int + cache_hit_rate: float + tokens_in: int + tokens_out: int + total_tokens: int + cost_usd: float + avg_latency_ms: float + + +@dataclass(frozen=True) +class AggregateStats: + """Overall aggregated telemetry metrics and category breakdowns.""" + + total_invocations: int + cache_hits: int + cache_hit_rate: float + total_tokens_in: int + total_tokens_out: int + total_tokens: int + total_cost_usd: float + avg_latency_ms: float + avg_latency_hit_ms: float + avg_latency_miss_ms: float + fallback_count: int + by_command: dict[str, BreakdownStats] + by_model: dict[str, BreakdownStats] + + +def calculate_cost( + model: str, + tokens_in: int | None, + tokens_out: int | None, + cache_hit: bool = False, + fallback_used: bool = False, +) -> float: + """Calculate estimated cost in USD based on model pricing table. + + Cache hits and template fallbacks consume zero billed API tokens ($0.00). + """ + if cache_hit or fallback_used: + return 0.0 + in_tokens = tokens_in or 0 + out_tokens = tokens_out or 0 + if in_tokens == 0 and out_tokens == 0: + return 0.0 + + if model not in MODEL_PRICING: + logger.warning("Unknown model '%s'; estimated cost recorded as $0.0", model) + return 0.0 + + in_rate, out_rate = MODEL_PRICING[model] + cost = (in_tokens / 1_000_000.0 * in_rate) + (out_tokens / 1_000_000.0 * out_rate) + return round(cost, 6) def get_metrics_db_path(db_path: Path | str | None = None) -> Path: @@ -72,16 +147,23 @@ def get_metrics_db_path(db_path: Path | str | None = None) -> Path: return base / APP_NAME / DEFAULT_DB_FILENAME -def init_db(db_path: Path | str | None = None) -> Path: - """Initialize the metrics database and invocations table if not present. +def _ensure_schema(conn: sqlite3.Connection) -> None: + """Ensure table exists and perform non-destructive schema migrations.""" + conn.executescript(CREATE_TABLE_SQL) + cursor = conn.cursor() + cursor.execute("PRAGMA table_info(invocations)") + columns = {row[1] for row in cursor.fetchall()} + if "estimated_cost_usd" not in columns: + conn.execute("ALTER TABLE invocations ADD COLUMN estimated_cost_usd REAL") - Returns the resolved database path. - """ + +def init_db(db_path: Path | str | None = None) -> Path: + """Initialize the metrics database and ensure schema is up-to-date.""" target = get_metrics_db_path(db_path) target.parent.mkdir(parents=True, exist_ok=True) conn = sqlite3.connect(target, timeout=5.0) - with contextlib.closing(conn): - conn.executescript(CREATE_TABLE_SQL) + with contextlib.closing(conn), conn: + _ensure_schema(conn) return target @@ -93,6 +175,7 @@ def record_invocation( cache_hit: bool, latency_ms: int | None, fallback_used: bool, + estimated_cost_usd: float | None = None, db_path: Path | str | None = None, timestamp: str | None = None, ) -> None: @@ -105,17 +188,28 @@ def record_invocation( target = get_metrics_db_path(db_path) target.parent.mkdir(parents=True, exist_ok=True) ts = timestamp if timestamp is not None else datetime.now(UTC).isoformat() + cost = ( + estimated_cost_usd + if estimated_cost_usd is not None + else calculate_cost( + model=model, + tokens_in=tokens_in, + tokens_out=tokens_out, + cache_hit=cache_hit, + fallback_used=fallback_used, + ) + ) conn = sqlite3.connect(target, timeout=5.0) with contextlib.closing(conn): - conn.executescript(CREATE_TABLE_SQL) + _ensure_schema(conn) with conn: conn.execute( """ INSERT INTO invocations ( timestamp, command, model, tokens_in, tokens_out, - cache_hit, latency_ms, fallback_used - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?) + cache_hit, latency_ms, fallback_used, estimated_cost_usd + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) """, ( ts, @@ -126,6 +220,7 @@ def record_invocation( 1 if cache_hit else 0, latency_ms, 1 if fallback_used else 0, + cost, ), ) except Exception as exc: # noqa: BLE001 @@ -136,20 +231,20 @@ def get_invocations( db_path: Path | str | None = None, limit: int | None = None, ) -> list[InvocationRecord]: - """Retrieve recorded invocations from the metrics database. - - Returns an empty list if the database file does not exist or errors occur. - """ + """Retrieve recorded invocations from the metrics database.""" try: target = get_metrics_db_path(db_path) if not target.exists(): return [] conn = sqlite3.connect(target, timeout=5.0) with contextlib.closing(conn): + _ensure_schema(conn) conn.row_factory = sqlite3.Row query = ( "SELECT id, timestamp, command, model, tokens_in, tokens_out, " - "cache_hit, latency_ms, fallback_used FROM invocations ORDER BY id ASC" + "cache_hit, latency_ms, fallback_used, " + "COALESCE(estimated_cost_usd, 0.0) as estimated_cost_usd " + "FROM invocations ORDER BY id ASC" ) if limit is not None: query += f" LIMIT {int(limit)}" @@ -167,6 +262,7 @@ def get_invocations( cache_hit=bool(row["cache_hit"]), latency_ms=row["latency_ms"], fallback_used=bool(row["fallback_used"]), + estimated_cost_usd=float(row["estimated_cost_usd"]), ) for row in rows ] @@ -175,6 +271,142 @@ def get_invocations( return [] +def get_monthly_spend( + year: int | None = None, + month: int | None = None, + db_path: Path | str | None = None, +) -> float: + """Calculate total estimated spend in USD for a given calendar month. + + Defaults to current UTC year and month if not specified. + """ + now = datetime.now(UTC) + y = year if year is not None else now.year + m = month if month is not None else now.month + prefix = f"{y:04d}-{m:02d}%" + + try: + target = get_metrics_db_path(db_path) + if not target.exists(): + return 0.0 + conn = sqlite3.connect(target, timeout=5.0) + with contextlib.closing(conn): + _ensure_schema(conn) + cursor = conn.cursor() + cursor.execute( + "SELECT COALESCE(SUM(estimated_cost_usd), 0.0) FROM invocations WHERE timestamp LIKE ?", + (prefix,), + ) + row = cursor.fetchone() + val = float(row[0]) if row and row[0] is not None else 0.0 + return round(val, 6) + except Exception as exc: # noqa: BLE001 + logger.warning("Failed to calculate monthly spend: %s", exc) + return 0.0 + + +def _build_breakdown(records: list[InvocationRecord]) -> BreakdownStats: + """Build BreakdownStats from a list of InvocationRecords.""" + total = len(records) + hits = sum(1 for r in records if r.cache_hit) + hit_rate = (hits / total * 100.0) if total > 0 else 0.0 + t_in = sum(r.tokens_in or 0 for r in records) + t_out = sum(r.tokens_out or 0 for r in records) + cost = sum(r.estimated_cost_usd for r in records) + valid_latencies = [r.latency_ms for r in records if r.latency_ms is not None] + avg_lat = sum(valid_latencies) / len(valid_latencies) if valid_latencies else 0.0 + + return BreakdownStats( + invocations=total, + cache_hits=hits, + cache_hit_rate=round(hit_rate, 1), + tokens_in=t_in, + tokens_out=t_out, + total_tokens=t_in + t_out, + cost_usd=round(cost, 6), + avg_latency_ms=round(avg_lat, 1), + ) + + +def get_aggregate_stats( + since: datetime | None = None, + command: str | None = None, + db_path: Path | str | None = None, +) -> AggregateStats: + """Compute aggregated metrics and breakdowns across invocations.""" + records = get_invocations(db_path=db_path) + + if since is not None: + since_iso = since.isoformat() + records = [r for r in records if r.timestamp >= since_iso] + + if command is not None: + records = [r for r in records if r.command == command] + + if not records: + return AggregateStats( + total_invocations=0, + cache_hits=0, + cache_hit_rate=0.0, + total_tokens_in=0, + total_tokens_out=0, + total_tokens=0, + total_cost_usd=0.0, + avg_latency_ms=0.0, + avg_latency_hit_ms=0.0, + avg_latency_miss_ms=0.0, + fallback_count=0, + by_command={}, + by_model={}, + ) + + total_invocations = len(records) + cache_hits = sum(1 for r in records if r.cache_hit) + cache_hit_rate = round(cache_hits / total_invocations * 100.0, 1) + total_tokens_in = sum(r.tokens_in or 0 for r in records) + total_tokens_out = sum(r.tokens_out or 0 for r in records) + total_tokens = total_tokens_in + total_tokens_out + total_cost = round(sum(r.estimated_cost_usd for r in records), 6) + fallback_count = sum(1 for r in records if r.fallback_used) + + valid_latencies = [r.latency_ms for r in records if r.latency_ms is not None] + avg_latency = round(sum(valid_latencies) / len(valid_latencies), 1) if valid_latencies else 0.0 + + hit_latencies = [r.latency_ms for r in records if r.cache_hit and r.latency_ms is not None] + avg_hit_lat = round(sum(hit_latencies) / len(hit_latencies), 1) if hit_latencies else 0.0 + + miss_latencies = [r.latency_ms for r in records if not r.cache_hit and r.latency_ms is not None] + avg_miss_lat = round(sum(miss_latencies) / len(miss_latencies), 1) if miss_latencies else 0.0 + + # Group by command + by_command: dict[str, list[InvocationRecord]] = {} + for r in records: + by_command.setdefault(r.command, []).append(r) + cmd_breakdown = {cmd: _build_breakdown(recs) for cmd, recs in by_command.items()} + + # Group by model + by_model: dict[str, list[InvocationRecord]] = {} + for r in records: + by_model.setdefault(r.model, []).append(r) + model_breakdown = {m: _build_breakdown(recs) for m, recs in by_model.items()} + + return AggregateStats( + total_invocations=total_invocations, + cache_hits=cache_hits, + cache_hit_rate=cache_hit_rate, + total_tokens_in=total_tokens_in, + total_tokens_out=total_tokens_out, + total_tokens=total_tokens, + total_cost_usd=total_cost, + avg_latency_ms=avg_latency, + avg_latency_hit_ms=avg_hit_lat, + avg_latency_miss_ms=avg_miss_lat, + fallback_count=fallback_count, + by_command=cmd_breakdown, + by_model=model_breakdown, + ) + + def clear_metrics(db_path: Path | str | None = None) -> None: """Remove all records from invocations table (helper for tests).""" try: diff --git a/src/gitmate/providers/cache.py b/src/gitmate/providers/cache.py index 5c20800..e26e9f2 100644 --- a/src/gitmate/providers/cache.py +++ b/src/gitmate/providers/cache.py @@ -47,6 +47,11 @@ def __init__( self._ttl = ttl self._template_version = template_version + @property + def underlying(self) -> LLMProvider: + """Return the wrapped LLMProvider.""" + return self._provider + def generate(self, prompt: str, model: str) -> LLMResponse: """Return a cached response when present, else generate and store.""" key = cache_key(prompt, model, self._template_version) diff --git a/tests/test_cli.py b/tests/test_cli.py index 4705f4c..5319752 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -293,3 +293,121 @@ def test_config_show_displays_allow_noninteractive_commit(tmp_config_dir: Path) assert res.exit_code == 0 assert "allow_noninteractive_commit" in res.output assert "False" in res.output + + +def test_stats_cli_empty_database(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("GITMATE_METRICS_DIR", str(tmp_path / "empty_metrics")) + res = runner.invoke(cli.app, ["stats"]) + assert res.exit_code == 0 + assert "No invocations recorded yet" in res.output + + +def test_stats_cli_populated_database(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + from gitmate.metrics import record_invocation + + m_dir = tmp_path / "cli_metrics" + monkeypatch.setenv("GITMATE_METRICS_DIR", str(m_dir)) + + record_invocation( + command="commit", + model="gemini-3.5-flash-lite", + tokens_in=5000, + tokens_out=500, + cache_hit=False, + latency_ms=450, + fallback_used=False, + ) + record_invocation( + command="commit", + model="gemini-3.5-flash-lite", + tokens_in=5000, + tokens_out=500, + cache_hit=True, + latency_ms=12, + fallback_used=False, + ) + + # 1. Standard table + res = runner.invoke(cli.app, ["stats"]) + assert res.exit_code == 0 + assert "gitmate stats (All Time)" in res.output + assert "Total Invocations" in res.output + assert "Cache Hits" in res.output + assert "50.0%" in res.output + assert "Estimated Spend" in res.output + assert "By Command" in res.output + + # 2. Time-scoped + res_month = runner.invoke(cli.app, ["stats", "--month"]) + assert res_month.exit_code == 0 + assert "gitmate stats (Current Month)" in res_month.output + + res_days = runner.invoke(cli.app, ["stats", "--days", "7"]) + assert res_days.exit_code == 0 + assert "gitmate stats (Last 7 Days)" in res_days.output + + +def test_stats_cli_raw_json(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + import json + + from gitmate.metrics import record_invocation + + m_dir = tmp_path / "json_metrics" + monkeypatch.setenv("GITMATE_METRICS_DIR", str(m_dir)) + + record_invocation( + command="commit", + model="gemini-3.5-flash-lite", + tokens_in=1000, + tokens_out=100, + cache_hit=True, + latency_ms=15, + fallback_used=False, + ) + + res = runner.invoke(cli.app, ["stats", "--raw"]) + assert res.exit_code == 0 + data = json.loads(res.output) + assert data["total_invocations"] == 1 + assert data["cache_hits"] == 1 + assert data["cache_hit_rate"] == 100.0 + assert "commit" in data["by_command"] + + +def test_stats_cli_raw_json_long_model_name_no_rewrapping( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Ensure long model names (100+ chars) do not get line-wrapped and break JSON parsing.""" + import json + + from gitmate.metrics import record_invocation + + m_dir = tmp_path / "long_model_metrics" + monkeypatch.setenv("GITMATE_METRICS_DIR", str(m_dir)) + + long_model = "custom-organization-deployment-fine-tuned-gemini-3-flash-preview-endpoint-accelerated-v2-long-identifier-string" + assert len(long_model) > 100 + + record_invocation( + command="commit", + model=long_model, + tokens_in=5000, + tokens_out=250, + cache_hit=False, + latency_ms=120, + fallback_used=False, + ) + + res = runner.invoke(cli.app, ["stats", "--raw"]) + assert res.exit_code == 0 + # Must parse cleanly without JSONDecodeError caused by line-wrapping + data = json.loads(res.output) + assert long_model in data["by_model"] + assert data["by_model"][long_model]["invocations"] == 1 + + +def test_stats_cli_invalid_days(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("GITMATE_METRICS_DIR", str(tmp_path / "metrics")) + res = runner.invoke(cli.app, ["stats", "--days", "0"]) + assert res.exit_code != 0 + assert "positive integer" in res.output diff --git a/tests/test_committer.py b/tests/test_committer.py index a58ac24..52c4768 100644 --- a/tests/test_committer.py +++ b/tests/test_committer.py @@ -501,3 +501,75 @@ def _mock_run(cmd: Any, *args: Any, **kwargs: Any) -> subprocess.CompletedProces code = commit_flow(console=con, provider=FakeProvider("msg")) assert code == 1 + + +# --- 10. Budget Cap Enforcement --- + + +def test_commit_flow_budget_cap_exceeded_aborts_immediately( + git_repo: Path, metrics_dir: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + from datetime import UTC, datetime + + from gitmate.metrics import record_invocation + + stage_file(git_repo, "budget.py", b"b = 1\n") + monkeypatch.setattr(sys.stdin, "isatty", lambda: True) + + # Record previous spend in current month exceeding cap + now_iso = datetime.now(UTC).isoformat() + record_invocation( + command="commit", + model="gemini-3.5-flash-lite", + tokens_in=1000, + tokens_out=200, + cache_hit=False, + latency_ms=200, + fallback_used=False, + estimated_cost_usd=1.50, + timestamp=now_iso, + ) + + cfg = GitmateConfig(budget_cap_usd=1.00) + provider = FakeProvider("feat: should not generate") + con = Console() + + code = commit_flow(cfg=cfg, console=con, provider=provider) + assert code == 1 + assert provider.call_count == 0 # Provider never called + + +def test_commit_flow_budget_cap_warning_allows_commit( + git_repo: Path, metrics_dir: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + from datetime import UTC, datetime + + from gitmate.metrics import record_invocation + + stage_file(git_repo, "warn.py", b"w = 1\n") + monkeypatch.setattr(sys.stdin, "isatty", lambda: True) + + # Record spend at 85% of $1.00 cap + now_iso = datetime.now(UTC).isoformat() + record_invocation( + command="commit", + model="gemini-3.5-flash-lite", + tokens_in=1000, + tokens_out=200, + cache_hit=False, + latency_ms=200, + fallback_used=False, + estimated_cost_usd=0.85, + timestamp=now_iso, + ) + + cfg = GitmateConfig(budget_cap_usd=1.00) + provider = FakeProvider("feat: allowed commit") + con = Console() + monkeypatch.setattr(con, "input", lambda prompt="": "a") + + code = commit_flow(cfg=cfg, console=con, provider=provider) + assert code == 0 + assert provider.call_count == 1 + messages = _get_git_log_messages(git_repo) + assert messages[0] == "feat: allowed commit" diff --git a/tests/test_metrics.py b/tests/test_metrics.py index 119111f..bbc91ca 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -212,3 +212,256 @@ def test_record_invocation_suppresses_sqlite_and_io_errors( # 2. Non-existent path get_invocations returns empty list without error non_existent = tmp_path / "does_not_exist" / "metrics.db" assert get_invocations(db_path=non_existent) == [] + + +def test_calculate_cost() -> None: + from gitmate.metrics import calculate_cost + + # Gemini Flash-Lite: $0.30 / 1M in, $2.50 / 1M out + # 100_000 in ($0.03) + 10_000 out ($0.025) = $0.055 + cost = calculate_cost( + model="gemini-3.5-flash-lite", + tokens_in=100_000, + tokens_out=10_000, + cache_hit=False, + fallback_used=False, + ) + assert cost == 0.055 + + # Gemini 3 Flash / Preview: $0.50 / 1M in, $3.00 / 1M out + # 100_000 in ($0.05) + 10_000 out ($0.03) = $0.08 + cost_flash = calculate_cost( + model="gemini-3-flash", + tokens_in=100_000, + tokens_out=10_000, + cache_hit=False, + fallback_used=False, + ) + assert cost_flash == 0.08 + + # Unknown model warns and returns $0.0 + cost_unknown = calculate_cost( + model="unknown-custom-model", + tokens_in=100_000, + tokens_out=10_000, + ) + assert cost_unknown == 0.0 + + # Cache hit is always $0.0 + assert ( + calculate_cost( + model="gemini-3.5-flash-lite", + tokens_in=100_000, + tokens_out=10_000, + cache_hit=True, + ) + == 0.0 + ) + + # Fallback template is always $0.0 + assert ( + calculate_cost( + model="gemini-3.5-flash-lite", + tokens_in=100_000, + tokens_out=10_000, + fallback_used=True, + ) + == 0.0 + ) + + # Zero tokens is $0.0 + assert calculate_cost("gemini-3.5-flash-lite", 0, 0) == 0.0 + assert calculate_cost("gemini-3.5-flash-lite", None, None) == 0.0 + + # Claude 3.5 Sonnet: $3.00 / 1M in, $15.00 / 1M out + # 10_000 in ($0.03) + 1_000 out ($0.015) = $0.045 + cost_claude = calculate_cost("claude-3-5-sonnet", 10_000, 1_000) + assert cost_claude == 0.045 + + +def test_schema_migration_adds_estimated_cost_column(tmp_path: Path) -> None: + db_file = tmp_path / "legacy_metrics.db" + # Create Phase 5 legacy schema without estimated_cost_usd + legacy_sql = """ + CREATE TABLE invocations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + timestamp TEXT NOT NULL, + command TEXT NOT NULL, + model TEXT NOT NULL, + tokens_in INTEGER, + tokens_out INTEGER, + cache_hit INTEGER NOT NULL, + latency_ms INTEGER, + fallback_used INTEGER NOT NULL + ); + INSERT INTO invocations ( + timestamp, command, model, tokens_in, tokens_out, cache_hit, latency_ms, fallback_used + ) VALUES ('2026-09-01T12:00:00Z', 'commit', 'gemini-3.5-flash-lite', 100, 20, 0, 200, 0); + """ + conn = sqlite3.connect(db_file) + conn.executescript(legacy_sql) + conn.close() + + # Now call init_db and get_invocations on the legacy file + init_db(db_file) + records = get_invocations(db_file) + assert len(records) == 1 + assert records[0].estimated_cost_usd == 0.0 + + # Record a new invocation on the migrated DB + record_invocation( + command="commit", + model="gemini-3.5-flash-lite", + tokens_in=1_000_000, + tokens_out=1_000_000, + cache_hit=False, + latency_ms=300, + fallback_used=False, + db_path=db_file, + ) + records_after = get_invocations(db_file) + assert len(records_after) == 2 + # Second record has calculated cost: 0.30 + 2.50 = 2.80 + assert records_after[1].estimated_cost_usd == 2.80 + + +def test_get_monthly_spend(metrics_dir: Path) -> None: + from gitmate.metrics import get_monthly_spend + + # September 2026 + record_invocation( + command="commit", + model="m", + tokens_in=100, + tokens_out=100, + cache_hit=False, + latency_ms=100, + fallback_used=False, + estimated_cost_usd=0.05, + timestamp="2026-09-10T10:00:00Z", + ) + record_invocation( + command="commit", + model="m", + tokens_in=100, + tokens_out=100, + cache_hit=False, + latency_ms=100, + fallback_used=False, + estimated_cost_usd=0.03, + timestamp="2026-09-15T12:00:00Z", + ) + # August 2026 + record_invocation( + command="commit", + model="m", + tokens_in=100, + tokens_out=100, + cache_hit=False, + latency_ms=100, + fallback_used=False, + estimated_cost_usd=0.20, + timestamp="2026-08-20T12:00:00Z", + ) + + sep_spend = get_monthly_spend(year=2026, month=9) + assert sep_spend == 0.08 + + aug_spend = get_monthly_spend(year=2026, month=8) + assert aug_spend == 0.20 + + oct_spend = get_monthly_spend(year=2026, month=10) + assert oct_spend == 0.0 + + +def test_get_aggregate_stats(metrics_dir: Path) -> None: + from datetime import UTC, datetime + + from gitmate.metrics import get_aggregate_stats + + # Empty DB stats + empty_stats = get_aggregate_stats() + assert empty_stats.total_invocations == 0 + assert empty_stats.cache_hit_rate == 0.0 + assert empty_stats.total_cost_usd == 0.0 + + # Populate with diverse entries + # 1. Commit cache miss + record_invocation( + command="commit", + model="gemini-3.5-flash-lite", + tokens_in=1000, + tokens_out=200, + cache_hit=False, + latency_ms=500, + fallback_used=False, + estimated_cost_usd=0.001, + timestamp="2026-09-10T10:00:00Z", + ) + # 2. Commit cache hit + record_invocation( + command="commit", + model="gemini-3.5-flash-lite", + tokens_in=1000, + tokens_out=200, + cache_hit=True, + latency_ms=10, + fallback_used=False, + estimated_cost_usd=0.0, + timestamp="2026-09-11T10:00:00Z", + ) + # 3. Fallback commit + record_invocation( + command="commit", + model="gemini-3.5-flash-lite", + tokens_in=0, + tokens_out=0, + cache_hit=False, + latency_ms=5, + fallback_used=True, + estimated_cost_usd=0.0, + timestamp="2026-09-12T10:00:00Z", + ) + # 4. PR summary command + record_invocation( + command="pr_summary", + model="gemini-3.5-flash", + tokens_in=2000, + tokens_out=500, + cache_hit=False, + latency_ms=800, + fallback_used=False, + estimated_cost_usd=0.005, + timestamp="2026-09-15T10:00:00Z", + ) + + stats = get_aggregate_stats() + assert stats.total_invocations == 4 + assert stats.cache_hits == 1 + assert stats.cache_hit_rate == 25.0 + assert stats.total_tokens_in == 4000 + assert stats.total_tokens_out == 900 + assert stats.total_tokens == 4900 + assert stats.total_cost_usd == 0.006 + assert stats.fallback_count == 1 + assert stats.avg_latency_hit_ms == 10.0 + assert stats.avg_latency_miss_ms == 435.0 # (500 + 5 + 800) / 3 = 435.0 + + # Check command breakdown + assert "commit" in stats.by_command + assert stats.by_command["commit"].invocations == 3 + assert stats.by_command["commit"].cache_hits == 1 + assert stats.by_command["commit"].cache_hit_rate == 33.3 + + assert "pr_summary" in stats.by_command + assert stats.by_command["pr_summary"].invocations == 1 + + # Check model breakdown + assert "gemini-3.5-flash-lite" in stats.by_model + assert "gemini-3.5-flash" in stats.by_model + + # Check since filtering + since_dt = datetime(2026, 9, 14, tzinfo=UTC) + filtered = get_aggregate_stats(since=since_dt) + assert filtered.total_invocations == 1 + assert filtered.by_command["pr_summary"].invocations == 1 diff --git a/tests/test_orchestrator.py b/tests/test_orchestrator.py index 2f09892..570f3eb 100644 --- a/tests/test_orchestrator.py +++ b/tests/test_orchestrator.py @@ -2,6 +2,7 @@ from __future__ import annotations +from pathlib import Path from unittest.mock import MagicMock import pytest @@ -275,3 +276,107 @@ def test_generate_commit_message_custom_secret_store() -> None: assert result.is_fallback is False assert result.text == "feat(auth): add login feature" + + +class ChunkSuccessFinalFailProvider: + """Mock provider where chunk summarization succeeds but the final prompt fails.""" + + def __init__(self, error_msg: str = "503 Service Unavailable") -> None: + self.error_msg = error_msg + self.call_count = 0 + self.prompts: list[str] = [] + + def generate(self, prompt: str, model: str) -> LLMResponse: + self.call_count += 1 + self.prompts.append(prompt) + if self.call_count == 1: + # Chunk summarization succeeds + return LLMResponse( + text="Summarized login logic", + model=model, + input_tokens=150, + output_tokens=25, + ) + # Final commit message prompt fails + raise ProviderUnavailable(self.error_msg) + + +def test_generate_commit_message_chunking_success_final_call_fails_preserves_chunk_tokens_and_cost() -> ( + None +): + """If chunk summarization succeeds but final prompt fails, chunk tokens and spend are preserved in fallback result.""" + cfg = GitmateConfig(max_context_tokens=2_600, model="gemini-3.5-flash-lite") + provider = ChunkSuccessFinalFailProvider(error_msg="503 Service Unavailable") + counter = FakeCounter() + console = MagicMock(spec=Console) + + result = generate_commit_message( + diffs=_sample_diff(), + cfg=cfg, + provider=provider, + counter=counter, + console=console, + ) + + assert result.is_fallback is True + assert result.text == "feat(auth): add login.py" + assert result.fallback_reason == "503 Service Unavailable" + # Chunking tokens MUST be preserved + assert result.input_tokens == 150 + assert result.output_tokens == 25 + # Spend MUST be non-zero (150 in / 25 out at $0.30 / $2.50 per 1M) + assert result.estimated_cost_usd > 0.0 + expected_chunk_cost = round((150 / 1_000_000 * 0.30) + (25 / 1_000_000 * 2.50), 6) + assert result.estimated_cost_usd == expected_chunk_cost + console.print.assert_called_once_with( + "[yellow]⚠ API unavailable, using template fallback[/yellow]" + ) + + +def test_generate_commit_message_chunking_with_cached_final_call_preserves_chunk_cost( + tmp_path: Path, +) -> None: + """When final prompt hits cache, chunking cost is still preserved rather than zeroed.""" + from gitmate.metrics import calculate_cost + from gitmate.providers.cache import CachedProvider + + cache_dir = tmp_path / "chunk_cache" + cfg = GitmateConfig( + max_context_tokens=2_600, + model="gemini-3.5-flash-lite", + cache_dir=str(cache_dir), + ) + counter = FakeCounter() + + raw_provider = FakeProvider(response="feat(auth): login service") + cached = CachedProvider(provider=raw_provider, cache_dir=str(cache_dir)) + + # 1. First invocation: cache miss + res1 = generate_commit_message( + diffs=_sample_diff(), + cfg=cfg, + provider=cached, + counter=counter, + bypass_cache=False, + ) + assert res1.is_fallback is False + assert res1.cache_hit is False + assert res1.input_tokens == 300 # 150 chunk + 150 final + assert res1.output_tokens == 50 # 25 chunk + 25 final + chunk_cost = calculate_cost(model="gemini-3.5-flash-lite", tokens_in=150, tokens_out=25) + final_cost = calculate_cost(model="gemini-3.5-flash-lite", tokens_in=150, tokens_out=25) + expected_full_cost = round(chunk_cost + final_cost, 6) + assert res1.estimated_cost_usd == expected_full_cost + + # 2. Second invocation: final prompt hits cache + res2 = generate_commit_message( + diffs=_sample_diff(), + cfg=cfg, + provider=cached, + counter=counter, + bypass_cache=False, + ) + assert res2.is_fallback is False + assert res2.cache_hit is True + # Final call cost was 0 (cache hit), but chunk_cost MUST still be preserved! + assert res2.estimated_cost_usd == chunk_cost diff --git a/tests/test_providers.py b/tests/test_providers.py index 4e376d8..2053c94 100644 --- a/tests/test_providers.py +++ b/tests/test_providers.py @@ -209,3 +209,12 @@ def test_retryable_error_alias() -> None: assert RetryableError is RetryableProviderError err = RetryableError("transient") assert isinstance(err, RetryableProviderError) + + +def test_cached_provider_underlying_property(tmp_path: Path) -> None: + fake = FakeProvider("output") + cached = CachedProvider(provider=fake, cache_dir=tmp_path) + try: + assert cached.underlying is fake + finally: + cached.close()