From da117056d16fc668d4fae4bd3baeb46cc6cf67fa Mon Sep 17 00:00:00 2001 From: Copilot <223556219+Copilot@users.noreply.github.com> Date: Tue, 21 Jul 2026 10:32:50 -0700 Subject: [PATCH 1/2] DOC Add scorer composition diagram Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: e451626a-e8bd-49a8-9b71-8598fbbe4ea6 --- doc/code/scoring/3_combining_scorers.ipynb | 58 +++++++++++++++++++++- doc/code/scoring/3_combining_scorers.py | 56 +++++++++++++++++++++ 2 files changed, 113 insertions(+), 1 deletion(-) diff --git a/doc/code/scoring/3_combining_scorers.ipynb b/doc/code/scoring/3_combining_scorers.ipynb index 9646871026..7557fd6cd5 100644 --- a/doc/code/scoring/3_combining_scorers.ipynb +++ b/doc/code/scoring/3_combining_scorers.ipynb @@ -23,7 +23,63 @@ "\n", "These wrappers are themselves scorers, so they plug into attacks and the batch scorer\n", "exactly like the leaf scorers on the [True/False](1_true_false_scorers.ipynb) and\n", - "[Float-scale](2_float_scale_scorers.ipynb) pages." + "[Float-scale](2_float_scale_scorers.ipynb) pages.\n", + "\n", + "The [class hierarchy](0_scoring.ipynb#the-class-hierarchy) explains what each wrapper\n", + "*is*. This diagram instead shows runtime composition: what each wrapper may contain.\n", + "Solid arrows pass a scorer through `scorer=` or `scorers=`, while dashed arrows show\n", + "the scorer base implemented by the resulting wrapper. An \"any\" input can therefore be\n", + "a leaf scorer or an already composed wrapper with that base, which enables stacking.\n", + "\n", + "```mermaid\n", + "flowchart LR\n", + " subgraph inputs[\"Supported inputs\"]\n", + " direction TB\n", + " TF[\"Any TrueFalseScorer
(leaf or previously wrapped)\"]\n", + " FS[\"Any FloatScaleScorer
(leaf or previously wrapped)\"]\n", + " end\n", + "\n", + " subgraph wrappers[\"Composition wrappers\"]\n", + " direction TB\n", + " COMP[\"TrueFalseCompositeScorer
AND · OR · MAJORITY\"]\n", + " INV[\"TrueFalseInverterScorer
negates one result\"]\n", + " THRESH[\"FloatScaleThresholdScorer
score ≥ threshold\"]\n", + " CONV[\"create_conversation_scorer()
scores concatenated history\"]\n", + " end\n", + "\n", + " subgraph outputs[\"Resulting scorer kind\"]\n", + " direction TB\n", + " TFOUT[\"TrueFalseScorer
can be stacked again\"]\n", + " FSOUT[\"FloatScaleScorer
can be stacked again\"]\n", + " end\n", + "\n", + " TF -->|\"1+ via scorers=\"| COMP\n", + " TF -->|\"1 via scorer=\"| INV\n", + " FS -->|\"1 via scorer=\"| THRESH\n", + " TF -->|\"1 via scorer=\"| CONV\n", + " FS -->|\"1 via scorer=\"| CONV\n", + "\n", + " COMP -. is a .-> TFOUT\n", + " INV -. is a .-> TFOUT\n", + " THRESH -. is a .-> TFOUT\n", + " CONV -. \"for true/false input\" .-> TFOUT\n", + " CONV -. \"for float-scale input\" .-> FSOUT\n", + "\n", + " classDef input fill:#e8f0fe,stroke:#4285f4,color:#15233a;\n", + " classDef wrapper fill:#fff4e5,stroke:#f29900,color:#3d2600;\n", + " classDef output fill:#e6f4ea,stroke:#34a853,color:#17351f;\n", + " class TF,FS input;\n", + " class COMP,INV,THRESH,CONV wrapper;\n", + " class TFOUT,FSOUT output;\n", + "```\n", + "\n", + "`TrueFalseCompositeScorer` requires at least one `TrueFalseScorer` and combines their\n", + "single results with `AND`, `OR`, or `MAJORITY`; `TrueFalseInverterScorer` accepts one\n", + "`TrueFalseScorer`. `FloatScaleThresholdScorer` is the cross-kind adapter: it accepts one\n", + "`FloatScaleScorer` and produces a `TrueFalseScorer`. `create_conversation_scorer()`\n", + "accepts only those two base types and returns a dynamic wrapper that remains the same\n", + "scorer kind as its input. For example, float-scale → conversation → threshold →\n", + "inversion is supported; a generic `Scorer` outside those base types is not." ] }, { diff --git a/doc/code/scoring/3_combining_scorers.py b/doc/code/scoring/3_combining_scorers.py index 2455e20240..16adedf2cf 100644 --- a/doc/code/scoring/3_combining_scorers.py +++ b/doc/code/scoring/3_combining_scorers.py @@ -19,6 +19,62 @@ # These wrappers are themselves scorers, so they plug into attacks and the batch scorer # exactly like the leaf scorers on the [True/False](1_true_false_scorers.ipynb) and # [Float-scale](2_float_scale_scorers.ipynb) pages. +# +# The [class hierarchy](0_scoring.ipynb#the-class-hierarchy) explains what each wrapper +# *is*. This diagram instead shows runtime composition: what each wrapper may contain. +# Solid arrows pass a scorer through `scorer=` or `scorers=`, while dashed arrows show +# the scorer base implemented by the resulting wrapper. An "any" input can therefore be +# a leaf scorer or an already composed wrapper with that base, which enables stacking. +# +# ```mermaid +# flowchart LR +# subgraph inputs["Supported inputs"] +# direction TB +# TF["Any TrueFalseScorer
(leaf or previously wrapped)"] +# FS["Any FloatScaleScorer
(leaf or previously wrapped)"] +# end +# +# subgraph wrappers["Composition wrappers"] +# direction TB +# COMP["TrueFalseCompositeScorer
AND · OR · MAJORITY"] +# INV["TrueFalseInverterScorer
negates one result"] +# THRESH["FloatScaleThresholdScorer
score ≥ threshold"] +# CONV["create_conversation_scorer()
scores concatenated history"] +# end +# +# subgraph outputs["Resulting scorer kind"] +# direction TB +# TFOUT["TrueFalseScorer
can be stacked again"] +# FSOUT["FloatScaleScorer
can be stacked again"] +# end +# +# TF -->|"1+ via scorers="| COMP +# TF -->|"1 via scorer="| INV +# FS -->|"1 via scorer="| THRESH +# TF -->|"1 via scorer="| CONV +# FS -->|"1 via scorer="| CONV +# +# COMP -. is a .-> TFOUT +# INV -. is a .-> TFOUT +# THRESH -. is a .-> TFOUT +# CONV -. "for true/false input" .-> TFOUT +# CONV -. "for float-scale input" .-> FSOUT +# +# classDef input fill:#e8f0fe,stroke:#4285f4,color:#15233a; +# classDef wrapper fill:#fff4e5,stroke:#f29900,color:#3d2600; +# classDef output fill:#e6f4ea,stroke:#34a853,color:#17351f; +# class TF,FS input; +# class COMP,INV,THRESH,CONV wrapper; +# class TFOUT,FSOUT output; +# ``` +# +# `TrueFalseCompositeScorer` requires at least one `TrueFalseScorer` and combines their +# single results with `AND`, `OR`, or `MAJORITY`; `TrueFalseInverterScorer` accepts one +# `TrueFalseScorer`. `FloatScaleThresholdScorer` is the cross-kind adapter: it accepts one +# `FloatScaleScorer` and produces a `TrueFalseScorer`. `create_conversation_scorer()` +# accepts only those two base types and returns a dynamic wrapper that remains the same +# scorer kind as its input. For example, float-scale → conversation → threshold → +# inversion is supported; a generic `Scorer` outside those base types is not. # %% from pyrit.setup import IN_MEMORY, initialize_pyrit_async From 18f2b2bfe4ad462e358bec4d4fb47369de7b6397 Mon Sep 17 00:00:00 2001 From: Copilot <223556219+Copilot@users.noreply.github.com> Date: Wed, 22 Jul 2026 16:01:00 -0700 Subject: [PATCH 2/2] DOC Separate scorer composition example Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- doc/code/scoring/3_combining_scorers.ipynb | 4 +++- doc/code/scoring/3_combining_scorers.py | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/doc/code/scoring/3_combining_scorers.ipynb b/doc/code/scoring/3_combining_scorers.ipynb index 7557fd6cd5..76952b51b6 100644 --- a/doc/code/scoring/3_combining_scorers.ipynb +++ b/doc/code/scoring/3_combining_scorers.ipynb @@ -78,7 +78,9 @@ "`TrueFalseScorer`. `FloatScaleThresholdScorer` is the cross-kind adapter: it accepts one\n", "`FloatScaleScorer` and produces a `TrueFalseScorer`. `create_conversation_scorer()`\n", "accepts only those two base types and returns a dynamic wrapper that remains the same\n", - "scorer kind as its input. For example, float-scale → conversation → threshold →\n", + "scorer kind as its input.\n", + "\n", + "For example, float-scale → conversation → threshold →\n", "inversion is supported; a generic `Scorer` outside those base types is not." ] }, diff --git a/doc/code/scoring/3_combining_scorers.py b/doc/code/scoring/3_combining_scorers.py index 16adedf2cf..c575bc4f5e 100644 --- a/doc/code/scoring/3_combining_scorers.py +++ b/doc/code/scoring/3_combining_scorers.py @@ -73,7 +73,9 @@ # `TrueFalseScorer`. `FloatScaleThresholdScorer` is the cross-kind adapter: it accepts one # `FloatScaleScorer` and produces a `TrueFalseScorer`. `create_conversation_scorer()` # accepts only those two base types and returns a dynamic wrapper that remains the same -# scorer kind as its input. For example, float-scale → conversation → threshold → +# scorer kind as its input. +# +# For example, float-scale → conversation → threshold → # inversion is supported; a generic `Scorer` outside those base types is not. # %% from pyrit.setup import IN_MEMORY, initialize_pyrit_async