diff --git a/benchmarks/openapi_request_recall/design_cases.json b/benchmarks/openapi_request_recall/design_cases.json new file mode 100644 index 00000000..676bd99c --- /dev/null +++ b/benchmarks/openapi_request_recall/design_cases.json @@ -0,0 +1,649 @@ +{ + "format_version": 1, + "split": "design", + "note": "Design-only cases. Requests were already inspected during development; do not use them to claim held-out improvement.", + "sources": { + "gitea": { + "url": "https://raw.githubusercontent.com/go-gitea/gitea/08c6ea6728ee35a16f9a39588e725044b7f389cd/templates/swagger/v1_json.tmpl", + "sha256": "c418b5362113c352e012d82d9e9eb7a430c41cd8310ec938aa944d70a8dcdcee" + }, + "wiremock": { + "url": "https://raw.githubusercontent.com/wiremock/wiremock/1a9a61507fbca5c7d6da72de4b22d54ceef48760/src/main/resources/swagger/wiremock-admin-api.json", + "sha256": "ab12a954320dbad1cebc1a54898aaab2d4ffe4b92165768fded650c9891a68fd" + } + }, + "cases": [ + { + "id": "search-gitea-owned-repos-en", + "system": "gitea", + "language": "en", + "query": "Show the repositories owned by my signed-in account.", + "primary": { + "method": "get", + "path": "/user/repos" + }, + "alternatives": [], + "origin": "xgen-cli tool-search-evaluation 2026-10-06" + }, + { + "id": "search-gitea-owned-repos-ko", + "system": "gitea", + "language": "ko", + "query": "내 계정이 소유한 저장소 목록을 보여줘.", + "primary": { + "method": "get", + "path": "/user/repos" + }, + "alternatives": [], + "origin": "xgen-cli tool-search-evaluation 2026-10-06" + }, + { + "id": "search-gitea-org-repos-en", + "system": "gitea", + "language": "en", + "query": "Which repositories belong to this organization?", + "primary": { + "method": "get", + "path": "/orgs/{org}/repos" + }, + "alternatives": [], + "origin": "xgen-cli tool-search-evaluation 2026-10-06" + }, + { + "id": "search-gitea-org-repos-ko", + "system": "gitea", + "language": "ko", + "query": "이 조직에 속한 저장소들을 찾아줘.", + "primary": { + "method": "get", + "path": "/orgs/{org}/repos" + }, + "alternatives": [], + "origin": "xgen-cli tool-search-evaluation 2026-10-06" + }, + { + "id": "search-gitea-repo-branches-en", + "system": "gitea", + "language": "en", + "query": "Show all branches of this repository.", + "primary": { + "method": "get", + "path": "/repos/{owner}/{repo}/branches" + }, + "alternatives": [], + "origin": "xgen-cli tool-search-evaluation 2026-10-06" + }, + { + "id": "search-gitea-repo-branches-ko", + "system": "gitea", + "language": "ko", + "query": "이 저장소의 브랜치 목록을 알려줘.", + "primary": { + "method": "get", + "path": "/repos/{owner}/{repo}/branches" + }, + "alternatives": [], + "origin": "xgen-cli tool-search-evaluation 2026-10-06" + }, + { + "id": "search-gitea-repo-releases-en", + "system": "gitea", + "language": "en", + "query": "Show the releases published for this repository.", + "primary": { + "method": "get", + "path": "/repos/{owner}/{repo}/releases" + }, + "alternatives": [], + "origin": "xgen-cli tool-search-evaluation 2026-10-06" + }, + { + "id": "search-gitea-repo-releases-ko", + "system": "gitea", + "language": "ko", + "query": "이 저장소에 게시된 릴리즈 목록을 보여줘.", + "primary": { + "method": "get", + "path": "/repos/{owner}/{repo}/releases" + }, + "alternatives": [], + "origin": "xgen-cli tool-search-evaluation 2026-10-06" + }, + { + "id": "search-gitea-repo-issues-en", + "system": "gitea", + "language": "en", + "query": "List the issues reported in this repository.", + "primary": { + "method": "get", + "path": "/repos/{owner}/{repo}/issues" + }, + "alternatives": [], + "origin": "xgen-cli tool-search-evaluation 2026-10-06" + }, + { + "id": "search-gitea-repo-issues-ko", + "system": "gitea", + "language": "ko", + "query": "이 저장소에 등록된 이슈 목록을 조회해줘.", + "primary": { + "method": "get", + "path": "/repos/{owner}/{repo}/issues" + }, + "alternatives": [], + "origin": "xgen-cli tool-search-evaluation 2026-10-06" + }, + { + "id": "search-gitea-repo-pulls-en", + "system": "gitea", + "language": "en", + "query": "List the pull requests for this repository.", + "primary": { + "method": "get", + "path": "/repos/{owner}/{repo}/pulls" + }, + "alternatives": [], + "origin": "xgen-cli tool-search-evaluation 2026-10-06" + }, + { + "id": "search-gitea-repo-pulls-ko", + "system": "gitea", + "language": "ko", + "query": "이 저장소의 풀 리퀘스트 목록을 조회해줘.", + "primary": { + "method": "get", + "path": "/repos/{owner}/{repo}/pulls" + }, + "alternatives": [], + "origin": "xgen-cli tool-search-evaluation 2026-10-06" + }, + { + "id": "search-gitea-issue-comments-en", + "system": "gitea", + "language": "en", + "query": "Read every comment on this issue.", + "primary": { + "method": "get", + "path": "/repos/{owner}/{repo}/issues/{index}/comments" + }, + "alternatives": [], + "origin": "xgen-cli tool-search-evaluation 2026-10-06" + }, + { + "id": "search-gitea-issue-comments-ko", + "system": "gitea", + "language": "ko", + "query": "이 이슈에 달린 댓글들을 읽어줘.", + "primary": { + "method": "get", + "path": "/repos/{owner}/{repo}/issues/{index}/comments" + }, + "alternatives": [], + "origin": "xgen-cli tool-search-evaluation 2026-10-06" + }, + { + "id": "search-gitea-file-content-en", + "system": "gitea", + "language": "en", + "query": "Read the contents of a file at a given path in this repository.", + "primary": { + "method": "get", + "path": "/repos/{owner}/{repo}/contents/{filepath}" + }, + "alternatives": [], + "origin": "xgen-cli tool-search-evaluation 2026-10-06" + }, + { + "id": "search-gitea-file-content-ko", + "system": "gitea", + "language": "ko", + "query": "이 저장소에서 지정한 경로의 파일 내용을 읽어줘.", + "primary": { + "method": "get", + "path": "/repos/{owner}/{repo}/contents/{filepath}" + }, + "alternatives": [], + "origin": "xgen-cli tool-search-evaluation 2026-10-06" + }, + { + "id": "search-wiremock-all-mappings-en", + "system": "wiremock", + "language": "en", + "query": "Show all configured stub mappings.", + "primary": { + "method": "get", + "path": "/__admin/mappings" + }, + "alternatives": [], + "origin": "xgen-cli tool-search-evaluation 2026-10-06" + }, + { + "id": "search-wiremock-all-mappings-ko", + "system": "wiremock", + "language": "ko", + "query": "설정된 스텁 매핑 목록을 보여줘.", + "primary": { + "method": "get", + "path": "/__admin/mappings" + }, + "alternatives": [], + "origin": "xgen-cli tool-search-evaluation 2026-10-06" + }, + { + "id": "search-wiremock-mapping-detail-en", + "system": "wiremock", + "language": "en", + "query": "Read the stub mapping with this ID.", + "primary": { + "method": "get", + "path": "/__admin/mappings/{stubMappingId}" + }, + "alternatives": [], + "origin": "xgen-cli tool-search-evaluation 2026-10-06" + }, + { + "id": "search-wiremock-mapping-detail-ko", + "system": "wiremock", + "language": "ko", + "query": "이 ID에 해당하는 스텁 매핑을 조회해줘.", + "primary": { + "method": "get", + "path": "/__admin/mappings/{stubMappingId}" + }, + "alternatives": [], + "origin": "xgen-cli tool-search-evaluation 2026-10-06" + }, + { + "id": "search-wiremock-request-journal-en", + "system": "wiremock", + "language": "en", + "query": "Show the requests recorded in the request journal.", + "primary": { + "method": "get", + "path": "/__admin/requests" + }, + "alternatives": [], + "origin": "xgen-cli tool-search-evaluation 2026-10-06" + }, + { + "id": "search-wiremock-request-journal-ko", + "system": "wiremock", + "language": "ko", + "query": "요청 로그에 기록된 요청 목록을 보여줘.", + "primary": { + "method": "get", + "path": "/__admin/requests" + }, + "alternatives": [], + "origin": "xgen-cli tool-search-evaluation 2026-10-06" + }, + { + "id": "search-wiremock-request-detail-en", + "system": "wiremock", + "language": "en", + "query": "Read the logged request identified by this ID.", + "primary": { + "method": "get", + "path": "/__admin/requests/{requestId}" + }, + "alternatives": [], + "origin": "xgen-cli tool-search-evaluation 2026-10-06" + }, + { + "id": "search-wiremock-request-detail-ko", + "system": "wiremock", + "language": "ko", + "query": "이 ID로 기록된 요청의 상세 내용을 조회해줘.", + "primary": { + "method": "get", + "path": "/__admin/requests/{requestId}" + }, + "alternatives": [], + "origin": "xgen-cli tool-search-evaluation 2026-10-06" + }, + { + "id": "search-wiremock-unmatched-requests-en", + "system": "wiremock", + "language": "en", + "query": "Find received requests that matched no stub mapping.", + "primary": { + "method": "get", + "path": "/__admin/requests/unmatched" + }, + "alternatives": [], + "origin": "xgen-cli tool-search-evaluation 2026-10-06" + }, + { + "id": "search-wiremock-unmatched-requests-ko", + "system": "wiremock", + "language": "ko", + "query": "스텁 매핑과 일치하지 않은 요청들을 찾아줘.", + "primary": { + "method": "get", + "path": "/__admin/requests/unmatched" + }, + "alternatives": [], + "origin": "xgen-cli tool-search-evaluation 2026-10-06" + }, + { + "id": "search-wiremock-recording-status-en", + "system": "wiremock", + "language": "en", + "query": "Check whether request recording is currently running.", + "primary": { + "method": "get", + "path": "/__admin/recordings/status" + }, + "alternatives": [], + "origin": "xgen-cli tool-search-evaluation 2026-10-06" + }, + { + "id": "search-wiremock-recording-status-ko", + "system": "wiremock", + "language": "ko", + "query": "요청 녹화가 지금 실행 중인지 확인해줘.", + "primary": { + "method": "get", + "path": "/__admin/recordings/status" + }, + "alternatives": [], + "origin": "xgen-cli tool-search-evaluation 2026-10-06" + }, + { + "id": "search-wiremock-scenarios-en", + "system": "wiremock", + "language": "en", + "query": "Show all stateful scenarios and their current states.", + "primary": { + "method": "get", + "path": "/__admin/scenarios" + }, + "alternatives": [], + "origin": "xgen-cli tool-search-evaluation 2026-10-06" + }, + { + "id": "search-wiremock-scenarios-ko", + "system": "wiremock", + "language": "ko", + "query": "모든 시나리오와 현재 상태를 보여줘.", + "primary": { + "method": "get", + "path": "/__admin/scenarios" + }, + "alternatives": [], + "origin": "xgen-cli tool-search-evaluation 2026-10-06" + }, + { + "id": "search-wiremock-files-en", + "system": "wiremock", + "language": "en", + "query": "List the names of all stored response files.", + "primary": { + "method": "get", + "path": "/__admin/files" + }, + "alternatives": [], + "origin": "xgen-cli tool-search-evaluation 2026-10-06" + }, + { + "id": "search-wiremock-files-ko", + "system": "wiremock", + "language": "ko", + "query": "저장된 응답 파일의 이름 목록을 보여줘.", + "primary": { + "method": "get", + "path": "/__admin/files" + }, + "alternatives": [], + "origin": "xgen-cli tool-search-evaluation 2026-10-06" + }, + { + "id": "choice-gitea-branch-tip-en", + "system": "gitea", + "language": "en", + "query": "For the branch main in repository acme/notes, show its latest commit details.", + "primary": { + "method": "get", + "path": "/repos/{owner}/{repo}/branches/{branch}" + }, + "alternatives": [ + { + "method": "get", + "path": "/repos/{owner}/{repo}/branches" + } + ], + "origin": "xgen-cli tool-choice-evaluation 2026-10-06" + }, + { + "id": "choice-gitea-branch-tip-ko", + "system": "gitea", + "language": "ko", + "query": "acme/notes 저장소의 main 브랜치에서 가장 최근 커밋 정보를 보고 싶어.", + "primary": { + "method": "get", + "path": "/repos/{owner}/{repo}/branches/{branch}" + }, + "alternatives": [ + { + "method": "get", + "path": "/repos/{owner}/{repo}/branches" + } + ], + "origin": "xgen-cli tool-choice-evaluation 2026-10-06" + }, + { + "id": "choice-gitea-commit-tree-en", + "system": "gitea", + "language": "en", + "query": "Show the file tree associated with a particular Git tree SHA, including nested entries.", + "primary": { + "method": "get", + "path": "/repos/{owner}/{repo}/git/trees/{sha}" + }, + "alternatives": [], + "origin": "xgen-cli tool-choice-evaluation 2026-10-06" + }, + { + "id": "choice-gitea-commit-tree-ko", + "system": "gitea", + "language": "ko", + "query": "특정 Git tree SHA에 대응하는 파일 트리를 하위 항목까지 보여줘.", + "primary": { + "method": "get", + "path": "/repos/{owner}/{repo}/git/trees/{sha}" + }, + "alternatives": [], + "origin": "xgen-cli tool-choice-evaluation 2026-10-06" + }, + { + "id": "choice-gitea-language-bytes-en", + "system": "gitea", + "language": "en", + "query": "How many source-code bytes does each programming language account for in acme/notes?", + "primary": { + "method": "get", + "path": "/repos/{owner}/{repo}/languages" + }, + "alternatives": [], + "origin": "xgen-cli tool-choice-evaluation 2026-10-06" + }, + { + "id": "choice-gitea-language-bytes-ko", + "system": "gitea", + "language": "ko", + "query": "acme/notes 저장소에서 프로그래밍 언어별 소스 코드 용량을 바이트로 알려줘.", + "primary": { + "method": "get", + "path": "/repos/{owner}/{repo}/languages" + }, + "alternatives": [], + "origin": "xgen-cli tool-choice-evaluation 2026-10-06" + }, + { + "id": "choice-gitea-public-user-en", + "system": "gitea", + "language": "en", + "query": "Show the public profile of the account whose username is riverdev.", + "primary": { + "method": "get", + "path": "/users/{username}" + }, + "alternatives": [ + { + "method": "get", + "path": "/users/search" + } + ], + "origin": "xgen-cli tool-choice-evaluation 2026-10-06" + }, + { + "id": "choice-gitea-public-user-ko", + "system": "gitea", + "language": "ko", + "query": "사용자명이 riverdev인 계정의 공개 프로필을 보여줘.", + "primary": { + "method": "get", + "path": "/users/{username}" + }, + "alternatives": [ + { + "method": "get", + "path": "/users/search" + } + ], + "origin": "xgen-cli tool-choice-evaluation 2026-10-06" + }, + { + "id": "choice-wiremock-unused-stubs-en", + "system": "wiremock", + "language": "en", + "query": "Which stub definitions have not matched any recorded requests?", + "primary": { + "method": "get", + "path": "/__admin/mappings/unmatched" + }, + "alternatives": [], + "origin": "xgen-cli tool-choice-evaluation 2026-10-06" + }, + { + "id": "choice-wiremock-unused-stubs-ko", + "system": "wiremock", + "language": "ko", + "query": "기록된 요청에 한 번도 매칭되지 않은 스텁 정의들을 찾아줘.", + "primary": { + "method": "get", + "path": "/__admin/mappings/unmatched" + }, + "alternatives": [], + "origin": "xgen-cli tool-choice-evaluation 2026-10-06" + }, + { + "id": "choice-wiremock-near-misses-en", + "system": "wiremock", + "language": "en", + "query": "Show closest stub matches for received requests that failed to match any stub.", + "primary": { + "method": "get", + "path": "/__admin/requests/unmatched/near-misses" + }, + "alternatives": [], + "origin": "xgen-cli tool-choice-evaluation 2026-10-06" + }, + { + "id": "choice-wiremock-near-misses-ko", + "system": "wiremock", + "language": "ko", + "query": "어느 스텁에도 일치하지 않은 수신 요청들의 가장 가까운 스텁 매칭 후보를 보여줘.", + "primary": { + "method": "get", + "path": "/__admin/requests/unmatched/near-misses" + }, + "alternatives": [], + "origin": "xgen-cli tool-choice-evaluation 2026-10-06" + }, + { + "id": "choice-wiremock-server-version-en", + "system": "wiremock", + "language": "en", + "query": "Which software version is this WireMock server running?", + "primary": { + "method": "get", + "path": "/__admin/version" + }, + "alternatives": [], + "origin": "xgen-cli tool-choice-evaluation 2026-10-06" + }, + { + "id": "choice-wiremock-server-version-ko", + "system": "wiremock", + "language": "ko", + "query": "이 WireMock 서버가 실행 중인 소프트웨어 버전을 알려줘.", + "primary": { + "method": "get", + "path": "/__admin/version" + }, + "alternatives": [], + "origin": "xgen-cli tool-choice-evaluation 2026-10-06" + }, + { + "id": "choice-wiremock-server-health-en", + "system": "wiremock", + "language": "en", + "query": "Check whether this WireMock instance is currently healthy.", + "primary": { + "method": "get", + "path": "/__admin/health" + }, + "alternatives": [], + "origin": "xgen-cli tool-choice-evaluation 2026-10-06" + }, + { + "id": "choice-wiremock-server-health-ko", + "system": "wiremock", + "language": "ko", + "query": "이 WireMock 인스턴스의 현재 상태가 정상인지 점검해줘.", + "primary": { + "method": "get", + "path": "/__admin/health" + }, + "alternatives": [], + "origin": "xgen-cli tool-choice-evaluation 2026-10-06" + }, + { + "id": "choice-gitea-unsupported-air-en", + "system": "gitea", + "language": "en", + "query": "Measure current airborne particulate concentration at a monitoring station.", + "primary": null, + "alternatives": [], + "origin": "xgen-cli tool-choice-evaluation 2026-10-06" + }, + { + "id": "choice-gitea-unsupported-air-ko", + "system": "gitea", + "language": "ko", + "query": "관측소의 현재 공기 중 미세먼지 농도를 측정해줘.", + "primary": null, + "alternatives": [], + "origin": "xgen-cli tool-choice-evaluation 2026-10-06" + }, + { + "id": "choice-wiremock-unsupported-audio-en", + "system": "wiremock", + "language": "en", + "query": "Transcribe a spoken audio recording into text.", + "primary": null, + "alternatives": [], + "origin": "xgen-cli tool-choice-evaluation 2026-10-06" + }, + { + "id": "choice-wiremock-unsupported-audio-ko", + "system": "wiremock", + "language": "ko", + "query": "음성 녹음 파일을 받아서 발화 내용을 텍스트로 변환해줘.", + "primary": null, + "alternatives": [], + "origin": "xgen-cli tool-choice-evaluation 2026-10-06" + } + ] +} \ No newline at end of file diff --git a/benchmarks/openapi_request_recall/run.py b/benchmarks/openapi_request_recall/run.py new file mode 100644 index 00000000..b7c1c3bf --- /dev/null +++ b/benchmarks/openapi_request_recall/run.py @@ -0,0 +1,164 @@ +"""Natural-request recall benchmark for persisted OpenAPI collection graphs. + +Measures whether a user request retrieves an acceptable endpoint in the +top-k results. It uses the same public path a product adapter uses: +``build_openapi_collection_artifact`` -> ``ToolGraph.load`` -> +``retrieve_with_scores``. Only the public API is imported, so the script can +be run against an installed release for a baseline:: + + uv run --no-project --with graph-tool-call==0.46.0 \ + python benchmarks/openapi_request_recall/run.py \ + --cases cases.json --spec gitea=gitea.json --output out.json + +Case format (JSON list or ``{"cases": [...]}``):: + + {"id": "...", "system": "gitea", "language": "en", "query": "...", + "primary": {"method": "get", "path": "/user/repos"} | null, + "alternatives": [{"method": "get", "path": "..."}]} + +Cases with ``primary: null`` are unsupported requests and are excluded from +recall metrics. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import statistics +import sys +import tempfile +import time +from collections import defaultdict +from pathlib import Path +from typing import Any + +import graph_tool_call +from graph_tool_call import ToolGraph +from graph_tool_call.graphify.collection_artifact import build_openapi_collection_artifact + + +def _load_cases(path: Path) -> list[dict[str, Any]]: + data = json.loads(path.read_text(encoding="utf-8")) + cases = data["cases"] if isinstance(data, dict) else data + for case in cases: + if "primary" not in case and "expected_path" in case: + case["primary"] = {"method": case["expected_method"], "path": case["expected_path"]} + case.setdefault("alternatives", []) + return cases + + +def _accepted(case: dict[str, Any]) -> set[tuple[str, str]]: + labels = [case["primary"], *case.get("alternatives", [])] + return {(label["method"].lower(), label["path"]) for label in labels} + + +def _build_graph(spec_paths: list[Path]) -> tuple[ToolGraph, float]: + specs = [json.loads(path.read_text(encoding="utf-8")) for path in spec_paths] + started = time.perf_counter() + artifact = build_openapi_collection_artifact(specs if len(specs) > 1 else specs[0]) + with tempfile.TemporaryDirectory(prefix="gtc-recall-") as directory: + path = Path(directory) / "graph.json" + path.write_text(json.dumps(artifact, ensure_ascii=False), encoding="utf-8") + graph = ToolGraph.load(path) + return graph, time.perf_counter() - started + + +def _endpoint(tool: Any) -> tuple[str, str]: + metadata = tool.metadata or {} + return (str(metadata.get("method", "")).lower(), str(metadata.get("path", ""))) + + +def _summarize(rows: list[dict[str, Any]]) -> dict[str, Any]: + scored = [row for row in rows if row["supported"]] + if not scored: + return {"n": 0} + return { + "n": len(scored), + "hit1": sum(row["rank"] == 1 for row in scored), + "hit5": sum(row["rank"] is not None for row in scored), + "mrr5": round(sum(1 / row["rank"] for row in scored if row["rank"]) / len(scored), 4), + } + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--cases", type=Path, required=True) + parser.add_argument( + "--spec", + action="append", + required=True, + help="system=path[,path...]; repeat per system", + ) + parser.add_argument("--top-k", type=int, default=5) + parser.add_argument("--output", type=Path) + parser.add_argument("--quiet", action="store_true") + args = parser.parse_args(argv) + + spec_map: dict[str, list[Path]] = {} + for item in args.spec: + system, _, paths = item.partition("=") + spec_map[system] = [Path(path) for path in paths.split(",")] + + cases = _load_cases(args.cases) + rows: list[dict[str, Any]] = [] + build: dict[str, Any] = {} + for system in sorted({case["system"] for case in cases}): + graph, seconds = _build_graph(spec_map[system]) + endpoints = {_endpoint(tool) for tool in graph.tools.values()} + build[system] = {"tools": len(graph.tools), "build_seconds": round(seconds, 2)} + for case in (case for case in cases if case["system"] == system): + supported = case.get("primary") is not None + accepted = _accepted(case) if supported else set() + started = time.perf_counter() + hits = graph.retrieve_with_scores(case["query"], top_k=args.top_k) + elapsed = time.perf_counter() - started + candidates = [_endpoint(hit.tool) for hit in hits] + rank = next( + (index for index, endpoint in enumerate(candidates, 1) if endpoint in accepted), + None, + ) + rows.append( + { + "id": case["id"], + "system": system, + "language": case.get("language"), + "supported": supported, + "indexed": bool(accepted & endpoints) if supported else None, + "rank": rank, + "candidates": [hit.tool.name for hit in hits], + "seconds": round(elapsed, 4), + } + ) + + groups: dict[str, list[dict[str, Any]]] = defaultdict(list) + for row in rows: + groups[f"{row['system']}/{row['language']}"].append(row) + latencies = [row["seconds"] for row in rows] + result = { + "graph_tool_call_version": graph_tool_call.__version__, + "cases_sha256": hashlib.sha256(args.cases.read_bytes()).hexdigest(), + "top_k": args.top_k, + "build": build, + "summary": { + "all": _summarize(rows), + "groups": {key: _summarize(value) for key, value in sorted(groups.items())}, + "latency_median_seconds": round(statistics.median(latencies), 4), + }, + "rows": rows, + } + if args.output: + args.output.write_text(json.dumps(result, ensure_ascii=False, indent=1), encoding="utf-8") + if not args.quiet: + print( + json.dumps( + {"version": result["graph_tool_call_version"], **result["summary"]}, + ensure_ascii=False, + indent=1, + ) + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/graph_tool_call/ingest/openapi.py b/graph_tool_call/ingest/openapi.py index 64e59767..cc9e6c6c 100644 --- a/graph_tool_call/ingest/openapi.py +++ b/graph_tool_call/ingest/openapi.py @@ -2523,64 +2523,6 @@ def _infer_annotations(method: str) -> MCPAnnotations | None: return _ANNOTATION_BY_METHOD.get(method.lower()) -def _enrich_description(description: str, method: str, path: str) -> str: - """Append path-derived context to short/generic descriptions. - - Many large APIs (e.g. Kubernetes) share identical descriptions across operations - that differ only in scope or sub-resource. This enrichment adds discriminative - signals that BM25 and embedding can use. - - Only activates when the path has enough depth (3+ segments) to indicate - a complex API with scope disambiguation needs. - """ - if not path: - return description - - segments = [s for s in path.split("/") if s and not s.startswith("{")] - # Only enrich for complex paths — simple APIs (e.g. /items, /users/{id}) - # don't need scope/sub-resource disambiguation. - if len(segments) < 3: - return description - - suffixes: list[str] = [] - - has_ns = "{namespace}" in path or "{ns}" in path - has_name = "{name}" in path - - if has_ns: - suffixes.append("namespaced") - elif not has_name and method.lower() in ("get", "delete"): - suffixes.append("cluster-wide") - - # Sub-resource detection from path suffix - if segments: - resource = segments[-1] - sub_resources = { - "exec", - "attach", - "portforward", - "proxy", - "log", - "status", - "scale", - "finalize", - "binding", - "eviction", - "ephemeralcontainers", - } - if resource.lower() in sub_resources and len(segments) >= 2: - parent = segments[-2] - suffixes.append(f"{resource} of {parent}") - - # Collection delete - if method.lower() == "delete" and not has_name: - suffixes.append("collection") - - if suffixes: - return f"{description} ({', '.join(suffixes)})" - return description - - def _resolve_server_url( operation: dict[str, Any], path_item: dict[str, Any] | None, @@ -2746,9 +2688,6 @@ def _operation_to_tool( parts.append(f"[{', '.join(tags)}]") description = " ".join(parts) - # Enrich generic descriptions with path-derived context - description = _enrich_description(description, method, path) - if is_swagger2: parameters = _extract_params_swagger2( operation, diff --git a/graph_tool_call/retrieval/engine.py b/graph_tool_call/retrieval/engine.py index caeb38e4..fd019910 100644 --- a/graph_tool_call/retrieval/engine.py +++ b/graph_tool_call/retrieval/engine.py @@ -15,6 +15,11 @@ from graph_tool_call.retrieval.keyword import BM25Scorer from graph_tool_call.retrieval.ranking import stable_score_items +# Strong lexical winners are lifted into this fixed head of the ranking. It must not +# follow the requested top_k, or the first results would change with how many +# results the caller asks for. +_DOMINANT_KEYWORD_HEAD = 5 + _CLAUSE_ACTION_PATTERN = re.compile( r"\b(" r"find|get|retrieve|search|calculate|compute|convert|make|create|return|fit|" @@ -398,7 +403,7 @@ def _run_pipeline( self._preserve_dominant_keyword_candidates( keyword_scores, final_scores, - top_k, + _DOMINANT_KEYWORD_HEAD, allow_top_rank_promotion=not self._has_diverse_actionable_clauses(query), ) semantic_scores = self._boost_structured_semantics(query, final_scores) @@ -971,7 +976,7 @@ def _boost_embedding_rerank(self, query: str, scores: dict[str, float]) -> None: def _preserve_dominant_keyword_candidates( keyword_scores: dict[str, float], final_scores: dict[str, float], - top_k: int, + head: int, *, allow_top_rank_promotion: bool = True, ) -> None: @@ -980,13 +985,13 @@ def _preserve_dominant_keyword_candidates( wRRF, clause injection, and semantic boosts are rank-based by design, which keeps unrelated raw score scales from dominating the whole pipeline. The tradeoff is that a very strong BM25 winner can sometimes - slide just below ``top_k`` or sit behind a close sibling when multiple + slide below the first ``head`` results or sit behind a close sibling when multiple auxiliary hints agree. This conservative guard only lifts top lexical candidates whose raw BM25 score is both strong in absolute terms and close to the keyword leader, preserving exact operationId/schema evidence without making weak keyword tails noisy. """ - if top_k <= 0 or not keyword_scores or not final_scores: + if head <= 0 or not keyword_scores or not final_scores: return ranked_keyword = [ @@ -1005,8 +1010,8 @@ def _preserve_dominant_keyword_candidates( if not ranked_final: return - top_names = {name for name, _score in ranked_final[:top_k]} - boundary_score = ranked_final[min(top_k - 1, len(ranked_final) - 1)][1] + top_names = {name for name, _score in ranked_final[:head]} + boundary_score = ranked_final[min(head - 1, len(ranked_final) - 1)][1] if boundary_score <= 0: return @@ -1020,9 +1025,9 @@ def _preserve_dominant_keyword_candidates( and top_keyword_name in top_names ): keyword_lead = top_keyword_score / max(keyword_by_name.get(top_final_name, 0.0), 1e-9) - final_rank = final_rank_by_name.get(top_keyword_name, top_k + 1) + final_rank = final_rank_by_name.get(top_keyword_name, head + 1) final_ratio = final_scores[top_keyword_name] / max(top_final_score, 1e-9) - if final_rank < top_k and keyword_lead >= 1.02 and final_ratio >= 0.88: + if final_rank < head and keyword_lead >= 1.02 and final_ratio >= 0.88: final_scores[top_keyword_name] = max( final_scores[top_keyword_name], top_final_score * 1.02 ) @@ -1038,9 +1043,9 @@ def _preserve_dominant_keyword_candidates( candidate_score = boundary_score * lift if allow_top_rank_promotion and keyword_rank == 1 and name != top_final_name: keyword_lead = score / max(keyword_by_name.get(top_final_name, 0.0), 1e-9) - final_rank = final_rank_by_name.get(name, top_k + 10) + final_rank = final_rank_by_name.get(name, head + 10) final_ratio = final_scores[name] / max(top_final_score, 1e-9) - if final_rank <= top_k + 2 and keyword_lead >= 1.02 and final_ratio >= 0.8: + if final_rank <= head + 2 and keyword_lead >= 1.02 and final_ratio >= 0.8: candidate_score = max(candidate_score, top_final_score * 1.02) final_scores[name] = max(final_scores[name], candidate_score) diff --git a/graph_tool_call/retrieval/keyword.py b/graph_tool_call/retrieval/keyword.py index 3a94e28b..2d94d6fa 100644 --- a/graph_tool_call/retrieval/keyword.py +++ b/graph_tool_call/retrieval/keyword.py @@ -8,6 +8,48 @@ from graph_tool_call.core.tool import ToolSchema +_PATH_SUB_RESOURCES = frozenset( + { + "exec", + "attach", + "portforward", + "proxy", + "log", + "status", + "scale", + "finalize", + "binding", + "eviction", + "ephemeralcontainers", + } +) + + +def _path_context_text(method: str, path: str) -> str: + """Describe scope and sub-resource hints derived from a deep OpenAPI path. + + Large APIs (e.g. Kubernetes) repeat one description across operations that + differ only in scope or sub-resource. These hints are index-only: they are + inferred from the path, not stated by the source, so they must not be shown + as the tool description. + """ + segments = [s for s in path.split("/") if s and not s.startswith("{")] + if len(segments) < 3: + return "" + method = method.lower() + has_name = "{name}" in path + hints: list[str] = [] + if "{namespace}" in path or "{ns}" in path: + hints.append("namespaced") + elif not has_name and method in ("get", "delete"): + hints.append("cluster-wide") + if segments[-1].lower() in _PATH_SUB_RESOURCES: + hints.append(f"{segments[-1]} of {segments[-2]}") + if method == "delete" and not has_name: + hints.append("collection") + return ", ".join(hints) + + # Baseline stopwords — always removed regardless of corpus. # These are common filler words that never carry discriminative value. _BASE_STOPWORDS = frozenset( @@ -751,6 +793,8 @@ def _extract_metadata_tokens(self, tool: ToolSchema) -> list[str]: if not path: return tokens + tokens.extend(self._tokenize_fn(_path_context_text(method, path))) + # Split path into segments, skip empty and {param} placeholders segments = [s for s in path.split("/") if s and not s.startswith("{")] for seg in segments: diff --git a/tests/test_bm25.py b/tests/test_bm25.py index e049ea38..ace918df 100644 --- a/tests/test_bm25.py +++ b/tests/test_bm25.py @@ -5,7 +5,7 @@ import pytest from graph_tool_call.core.tool import ToolParameter, ToolSchema -from graph_tool_call.retrieval.keyword import BM25Scorer +from graph_tool_call.retrieval.keyword import BM25Scorer, _path_context_text def _make_tool( @@ -409,3 +409,33 @@ def test_mixed_korean_english(): assert "해지" in tokens # English parts from camelCase split — "order" and "cancel" are part of mixed tokens # They remain as part of their original tokens since there's no camelCase boundary + + +def test_path_context_text_describes_deep_paths_only(): + assert _path_context_text("get", "/items/{id}") == "" + assert _path_context_text("get", "/api/v1/nodes/list") == "cluster-wide" + assert ( + _path_context_text("get", "/api/v1/namespaces/{namespace}/pods/{name}/status") + == "namespaced, status of pods" + ) + assert ( + _path_context_text("DELETE", "/api/v1/namespaces/{namespace}/pods") + == "namespaced, collection" + ) + + +def test_path_context_is_indexed_without_changing_description(): + tool = ToolSchema( + name="readPodStatus", + description="read the specified Pod", + metadata={"method": "get", "path": "/api/v1/namespaces/{namespace}/pods/{name}/status"}, + ) + other = ToolSchema( + name="readPod", + description="read the specified Pod", + metadata={"method": "get", "path": "/api/v1/namespaces/{namespace}/pods/{name}"}, + ) + scorer = BM25Scorer({tool.name: tool, other.name: other}) + scores = scorer.score("status of a pod") + assert scores["readPodStatus"] > scores.get("readPod", 0.0) + assert tool.description == "read the specified Pod" diff --git a/tests/test_ingest_openapi.py b/tests/test_ingest_openapi.py index 627cc602..78a7f67f 100644 --- a/tests/test_ingest_openapi.py +++ b/tests/test_ingest_openapi.py @@ -2360,6 +2360,43 @@ def test_whitespace_only_description_gets_fallback(self) -> None: tools, _ = ingest_openapi(spec) assert tools[0].description == "POST /items [items, crud]" + def test_deep_paths_keep_source_description(self) -> None: + """Path-derived scope hints are index-only and never appended to descriptions.""" + spec: dict = { + "openapi": "3.0.0", + "info": {"title": "Test", "version": "1.0.0"}, + "paths": { + "/admin/menus/list": { + "get": { + "operationId": "listMenus", + "summary": "메뉴 목록 조회", + "responses": {"200": {"description": "OK"}}, + } + }, + "/api/v1/namespaces/{namespace}/pods/{name}/status": { + "get": { + "operationId": "readPodStatus", + "summary": "read status of the specified Pod", + "responses": {"200": {"description": "OK"}}, + } + }, + "/api/v1/namespaces/{namespace}/pods": { + "delete": { + "operationId": "deletePods", + "summary": "delete collection of Pod", + "responses": {"200": {"description": "OK"}}, + } + }, + }, + } + tools, _ = ingest_openapi(spec) + descriptions = {tool.name: tool.description for tool in tools} + assert descriptions == { + "listMenus": "메뉴 목록 조회", + "readPodStatus": "read status of the specified Pod", + "deletePods": "delete collection of Pod", + } + class TestMalformedParameters: def test_malformed_param_without_name_skipped(self) -> None: diff --git a/tests/test_retrieval.py b/tests/test_retrieval.py index 9f7d5d6b..da32816c 100644 --- a/tests/test_retrieval.py +++ b/tests/test_retrieval.py @@ -247,7 +247,7 @@ def test_dominant_keyword_candidate_preserved_near_top_k_boundary(): "auxiliary_c": 18.0, } - RetrievalEngine._preserve_dominant_keyword_candidates(keyword_scores, final_scores, top_k=5) + RetrievalEngine._preserve_dominant_keyword_candidates(keyword_scores, final_scores, head=5) ranked = sorted(final_scores, key=final_scores.get, reverse=True) assert "exact_keyword_target" in ranked[:5] @@ -268,7 +268,7 @@ def test_dominant_keyword_leader_just_outside_boundary_can_win_close_sibling(): "auxiliary_a": 20.0, } - RetrievalEngine._preserve_dominant_keyword_candidates(keyword_scores, final_scores, top_k=5) + RetrievalEngine._preserve_dominant_keyword_candidates(keyword_scores, final_scores, head=5) ranked = sorted(final_scores, key=final_scores.get, reverse=True) assert ranked[0] == "exact_keyword_target" @@ -289,7 +289,7 @@ def test_dominant_keyword_candidate_guard_ignores_weak_tail_matches(): "weak_keyword_tail": 12.0, } - RetrievalEngine._preserve_dominant_keyword_candidates(keyword_scores, final_scores, top_k=5) + RetrievalEngine._preserve_dominant_keyword_candidates(keyword_scores, final_scores, head=5) ranked = sorted(final_scores, key=final_scores.get, reverse=True) assert "weak_keyword_tail" not in ranked[:5] @@ -309,7 +309,7 @@ def test_dominant_keyword_leader_can_win_close_sibling_rank(): "auxiliary_a": 20.0, } - RetrievalEngine._preserve_dominant_keyword_candidates(keyword_scores, final_scores, top_k=5) + RetrievalEngine._preserve_dominant_keyword_candidates(keyword_scores, final_scores, head=5) ranked = sorted(final_scores, key=final_scores.get, reverse=True) assert ranked[0] == "exact_keyword_target" @@ -331,7 +331,7 @@ def test_dominant_keyword_leader_top_rank_promotion_can_be_disabled(): RetrievalEngine._preserve_dominant_keyword_candidates( keyword_scores, final_scores, - top_k=5, + head=5, allow_top_rank_promotion=False, ) @@ -352,7 +352,7 @@ def test_dominant_keyword_leader_does_not_jump_large_score_gap(): "semantic_sibling": 60.0, } - RetrievalEngine._preserve_dominant_keyword_candidates(keyword_scores, final_scores, top_k=5) + RetrievalEngine._preserve_dominant_keyword_candidates(keyword_scores, final_scores, head=5) ranked = sorted(final_scores, key=final_scores.get, reverse=True) assert ranked[0] == "semantic_sibling" @@ -567,3 +567,21 @@ def test_clause_diversity_gate_detects_distinct_subtasks(): ) assert engine._has_diverse_actionable_clauses(query) + + +def test_dominant_keyword_guard_head_does_not_follow_top_k(monkeypatch): + seen = [] + original = RetrievalEngine._preserve_dominant_keyword_candidates + + def record(keyword_scores, final_scores, head, **kwargs): + seen.append(head) + return original(keyword_scores, final_scores, head, **kwargs) + + monkeypatch.setattr( + RetrievalEngine, "_preserve_dominant_keyword_candidates", staticmethod(record) + ) + tg = _build_file_tools_graph() + for top_k in (3, 10, 20): + tg.retrieve_with_scores("read the file contents", top_k=top_k) + assert len(seen) == 3 + assert len(set(seen)) == 1