diff --git a/doc/code/scoring/3_combining_scorers.ipynb b/doc/code/scoring/3_combining_scorers.ipynb index 9646871026..76952b51b6 100644 --- a/doc/code/scoring/3_combining_scorers.ipynb +++ b/doc/code/scoring/3_combining_scorers.ipynb @@ -23,7 +23,65 @@ "\n", "These wrappers are themselves scorers, so they plug into attacks and the batch scorer\n", "exactly like the leaf scorers on the [True/False](1_true_false_scorers.ipynb) and\n", - "[Float-scale](2_float_scale_scorers.ipynb) pages." + "[Float-scale](2_float_scale_scorers.ipynb) pages.\n", + "\n", + "The [class hierarchy](0_scoring.ipynb#the-class-hierarchy) explains what each wrapper\n", + "*is*. This diagram instead shows runtime composition: what each wrapper may contain.\n", + "Solid arrows pass a scorer through `scorer=` or `scorers=`, while dashed arrows show\n", + "the scorer base implemented by the resulting wrapper. An \"any\" input can therefore be\n", + "a leaf scorer or an already composed wrapper with that base, which enables stacking.\n", + "\n", + "```mermaid\n", + "flowchart LR\n", + " subgraph inputs[\"Supported inputs\"]\n", + " direction TB\n", + " TF[\"Any TrueFalseScorer
(leaf or previously wrapped)\"]\n", + " FS[\"Any FloatScaleScorer
(leaf or previously wrapped)\"]\n", + " end\n", + "\n", + " subgraph wrappers[\"Composition wrappers\"]\n", + " direction TB\n", + " COMP[\"TrueFalseCompositeScorer
AND · OR · MAJORITY\"]\n", + " INV[\"TrueFalseInverterScorer
negates one result\"]\n", + " THRESH[\"FloatScaleThresholdScorer
score ≥ threshold\"]\n", + " CONV[\"create_conversation_scorer()
scores concatenated history\"]\n", + " end\n", + "\n", + " subgraph outputs[\"Resulting scorer kind\"]\n", + " direction TB\n", + " TFOUT[\"TrueFalseScorer
can be stacked again\"]\n", + " FSOUT[\"FloatScaleScorer
can be stacked again\"]\n", + " end\n", + "\n", + " TF -->|\"1+ via scorers=\"| COMP\n", + " TF -->|\"1 via scorer=\"| INV\n", + " FS -->|\"1 via scorer=\"| THRESH\n", + " TF -->|\"1 via scorer=\"| CONV\n", + " FS -->|\"1 via scorer=\"| CONV\n", + "\n", + " COMP -. is a .-> TFOUT\n", + " INV -. is a .-> TFOUT\n", + " THRESH -. is a .-> TFOUT\n", + " CONV -. \"for true/false input\" .-> TFOUT\n", + " CONV -. \"for float-scale input\" .-> FSOUT\n", + "\n", + " classDef input fill:#e8f0fe,stroke:#4285f4,color:#15233a;\n", + " classDef wrapper fill:#fff4e5,stroke:#f29900,color:#3d2600;\n", + " classDef output fill:#e6f4ea,stroke:#34a853,color:#17351f;\n", + " class TF,FS input;\n", + " class COMP,INV,THRESH,CONV wrapper;\n", + " class TFOUT,FSOUT output;\n", + "```\n", + "\n", + "`TrueFalseCompositeScorer` requires at least one `TrueFalseScorer` and combines their\n", + "single results with `AND`, `OR`, or `MAJORITY`; `TrueFalseInverterScorer` accepts one\n", + "`TrueFalseScorer`. `FloatScaleThresholdScorer` is the cross-kind adapter: it accepts one\n", + "`FloatScaleScorer` and produces a `TrueFalseScorer`. `create_conversation_scorer()`\n", + "accepts only those two base types and returns a dynamic wrapper that remains the same\n", + "scorer kind as its input.\n", + "\n", + "For example, float-scale → conversation → threshold →\n", + "inversion is supported; a generic `Scorer` outside those base types is not." ] }, { diff --git a/doc/code/scoring/3_combining_scorers.py b/doc/code/scoring/3_combining_scorers.py index 2455e20240..c575bc4f5e 100644 --- a/doc/code/scoring/3_combining_scorers.py +++ b/doc/code/scoring/3_combining_scorers.py @@ -19,6 +19,64 @@ # These wrappers are themselves scorers, so they plug into attacks and the batch scorer # exactly like the leaf scorers on the [True/False](1_true_false_scorers.ipynb) and # [Float-scale](2_float_scale_scorers.ipynb) pages. +# +# The [class hierarchy](0_scoring.ipynb#the-class-hierarchy) explains what each wrapper +# *is*. This diagram instead shows runtime composition: what each wrapper may contain. +# Solid arrows pass a scorer through `scorer=` or `scorers=`, while dashed arrows show +# the scorer base implemented by the resulting wrapper. An "any" input can therefore be +# a leaf scorer or an already composed wrapper with that base, which enables stacking. +# +# ```mermaid +# flowchart LR +# subgraph inputs["Supported inputs"] +# direction TB +# TF["Any TrueFalseScorer
(leaf or previously wrapped)"] +# FS["Any FloatScaleScorer
(leaf or previously wrapped)"] +# end +# +# subgraph wrappers["Composition wrappers"] +# direction TB +# COMP["TrueFalseCompositeScorer
AND · OR · MAJORITY"] +# INV["TrueFalseInverterScorer
negates one result"] +# THRESH["FloatScaleThresholdScorer
score ≥ threshold"] +# CONV["create_conversation_scorer()
scores concatenated history"] +# end +# +# subgraph outputs["Resulting scorer kind"] +# direction TB +# TFOUT["TrueFalseScorer
can be stacked again"] +# FSOUT["FloatScaleScorer
can be stacked again"] +# end +# +# TF -->|"1+ via scorers="| COMP +# TF -->|"1 via scorer="| INV +# FS -->|"1 via scorer="| THRESH +# TF -->|"1 via scorer="| CONV +# FS -->|"1 via scorer="| CONV +# +# COMP -. is a .-> TFOUT +# INV -. is a .-> TFOUT +# THRESH -. is a .-> TFOUT +# CONV -. "for true/false input" .-> TFOUT +# CONV -. "for float-scale input" .-> FSOUT +# +# classDef input fill:#e8f0fe,stroke:#4285f4,color:#15233a; +# classDef wrapper fill:#fff4e5,stroke:#f29900,color:#3d2600; +# classDef output fill:#e6f4ea,stroke:#34a853,color:#17351f; +# class TF,FS input; +# class COMP,INV,THRESH,CONV wrapper; +# class TFOUT,FSOUT output; +# ``` +# +# `TrueFalseCompositeScorer` requires at least one `TrueFalseScorer` and combines their +# single results with `AND`, `OR`, or `MAJORITY`; `TrueFalseInverterScorer` accepts one +# `TrueFalseScorer`. `FloatScaleThresholdScorer` is the cross-kind adapter: it accepts one +# `FloatScaleScorer` and produces a `TrueFalseScorer`. `create_conversation_scorer()` +# accepts only those two base types and returns a dynamic wrapper that remains the same +# scorer kind as its input. +# +# For example, float-scale → conversation → threshold → +# inversion is supported; a generic `Scorer` outside those base types is not. # %% from pyrit.setup import IN_MEMORY, initialize_pyrit_async