diff --git a/README.md b/README.md index 9421235..715941a 100644 --- a/README.md +++ b/README.md @@ -119,42 +119,7 @@ Goldens are stored as JSON next to the suite file key), keyed by case name. A changed response fails the case with a `snapshot` assertion result ("re-run with `--record` to update"). A `snapshot: true` case with no recorded golden fails with a hint to record -one first, so the workflow never silently skips. - -## Suite files - -```yaml -name: sentiment-classifier - -model: - base_url: https://api.openai.com/v1 # any OpenAI-compatible endpoint - name: gpt-4o-mini - api_key_env: OPENAI_API_KEY # env var holding the key - temperature: 0 - -prompt: | - Classify the sentiment of this review. Respond with JSON only. - Review: {{review}} - -# Only used by --mock runs: deterministic offline responses. -mock: - default: '{"sentiment": "neutral"}' - rules: - - match: "absolutely loved" # substring, or /regex/ - response: '{"sentiment": "positive"}' - -cases: - - name: glowing review - vars: {review: "I absolutely loved this keyboard."} - assertions: - - contains: '"positive"' - - max_chars: 100 - - json_schema: - type: object - required: [sentiment] - properties: - sentiment: {type: string, enum: [positive, negative, neutral]} -``` +it. See `examples/snapshots.yaml` and the matching golden file. Templates support `{{variable}}` placeholders only — a suite is data, not code. A case may set `mock_response` to override the mock rules for that @@ -186,6 +151,14 @@ each turn's user message). `snapshot: true` and `--record` apply to the conversation's final reply. See `examples/multiturn.yaml` for a runnable example (`promptproof run examples/multiturn.yaml --mock`). +### Retrieval-augmented answers (deterministic only) + +When the prompt is a RAG-style answer grounded in provided context, you can +check the required facts, forbid invented citations, and enforce brevity +with only deterministic assertions — no `llm_judge` needed. See +`examples/rag.yaml` for a complete offline example +(`promptproof run examples/rag.yaml --mock`). + ## Assertions | Assertion | Example | Passes when… | diff --git a/examples/rag.yaml b/examples/rag.yaml new file mode 100644 index 0000000..486edea --- /dev/null +++ b/examples/rag.yaml @@ -0,0 +1,66 @@ +# PromptProof example suite — a retrieval-augmented answer checker. +# +# Demonstrates purely deterministic assertions for a RAG-style response: +# required facts must all be present, invented citations / internal +# markers must be absent, and the answer must stay concise. +# +# Runs fully offline (no API key, no llm_judge): +# promptproof run examples/rag.yaml --mock + +name: rag-answer-checker + +model: + base_url: https://api.openai.com/v1 + name: gpt-4o-mini + api_key_env: OPENAI_API_KEY + temperature: 0 + +prompt: | + Answer the question using only the provided context. Cite facts from + the context. Do not invent sources, paper titles, or document IDs. + Keep the answer under 200 characters. + + Context: + {{context}} + + Question: {{question}} + +# Offline mock rules. First matching rule wins; default is a safe fallback. +mock: + default: "I cannot answer from the provided context." + rules: + - match: "founded in 2012" + response: "Acme Corp was founded in 2012 by Jane Doe in Seattle." + - match: "Series B" + response: "Acme raised a $40M Series B in 2023 led by Example Ventures." + +cases: + - name: founding facts + vars: + context: "Acme Corp was founded in 2012 by Jane Doe in Seattle." + question: "When and where was Acme Corp founded, and by whom?" + assertions: + - contains_all: ["2012", "Jane Doe", "Seattle"] + - not_contains: "according to a 2015 paper" + - not_regex: "\\[\\d+\\]" # no bracketed citation numbers + - max_chars: 200 + + - name: funding round + vars: + context: "In 2023 Acme raised a $40 million Series B round led by Example Ventures." + question: "What was the size of Acme's Series B and who led it?" + assertions: + - contains_all: ["$40M", "Series B", "Example Ventures"] + - not_contains: "https://arxiv.org" + - not_regex: "doc_id:\\s*\\d+" # no invented document identifiers + - max_chars: 200 + + - name: insufficient context + vars: + context: "Acme Corp sells widgets." + question: "What is Acme's current market cap?" + mock_response: "The provided context does not contain market-cap information." + assertions: + - not_contains: "billion" + - not_regex: "\\$[0-9]+\\s*[BM]?" # must not invent a dollar figure + - max_chars: 200