Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
60 changes: 59 additions & 1 deletion doc/code/scoring/3_combining_scorers.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,65 @@
"\n",
"These wrappers are themselves scorers, so they plug into attacks and the batch scorer\n",
"exactly like the leaf scorers on the [True/False](1_true_false_scorers.ipynb) and\n",
"[Float-scale](2_float_scale_scorers.ipynb) pages."
"[Float-scale](2_float_scale_scorers.ipynb) pages.\n",
"\n",
"The [class hierarchy](0_scoring.ipynb#the-class-hierarchy) explains what each wrapper\n",
"*is*. This diagram instead shows runtime composition: what each wrapper may contain.\n",
"Solid arrows pass a scorer through `scorer=` or `scorers=`, while dashed arrows show\n",
"the scorer base implemented by the resulting wrapper. An \"any\" input can therefore be\n",
"a leaf scorer or an already composed wrapper with that base, which enables stacking.\n",
"\n",
"```mermaid\n",
"flowchart LR\n",
" subgraph inputs[\"Supported inputs\"]\n",
" direction TB\n",
" TF[\"Any TrueFalseScorer<br/>(leaf or previously wrapped)\"]\n",
" FS[\"Any FloatScaleScorer<br/>(leaf or previously wrapped)\"]\n",
" end\n",
"\n",
" subgraph wrappers[\"Composition wrappers\"]\n",
" direction TB\n",
" COMP[\"TrueFalseCompositeScorer<br/>AND · OR · MAJORITY\"]\n",
" INV[\"TrueFalseInverterScorer<br/>negates one result\"]\n",
" THRESH[\"FloatScaleThresholdScorer<br/>score ≥ threshold\"]\n",
" CONV[\"create_conversation_scorer()<br/>scores concatenated history\"]\n",
" end\n",
"\n",
" subgraph outputs[\"Resulting scorer kind\"]\n",
" direction TB\n",
" TFOUT[\"TrueFalseScorer<br/>can be stacked again\"]\n",
" FSOUT[\"FloatScaleScorer<br/>can be stacked again\"]\n",
" end\n",
"\n",
" TF -->|\"1+ via scorers=\"| COMP\n",
" TF -->|\"1 via scorer=\"| INV\n",
" FS -->|\"1 via scorer=\"| THRESH\n",
" TF -->|\"1 via scorer=\"| CONV\n",
" FS -->|\"1 via scorer=\"| CONV\n",
"\n",
" COMP -. is a .-> TFOUT\n",
" INV -. is a .-> TFOUT\n",
" THRESH -. is a .-> TFOUT\n",
" CONV -. \"for true/false input\" .-> TFOUT\n",
" CONV -. \"for float-scale input\" .-> FSOUT\n",
"\n",
" classDef input fill:#e8f0fe,stroke:#4285f4,color:#15233a;\n",
" classDef wrapper fill:#fff4e5,stroke:#f29900,color:#3d2600;\n",
" classDef output fill:#e6f4ea,stroke:#34a853,color:#17351f;\n",
" class TF,FS input;\n",
" class COMP,INV,THRESH,CONV wrapper;\n",
" class TFOUT,FSOUT output;\n",
"```\n",
"\n",
"`TrueFalseCompositeScorer` requires at least one `TrueFalseScorer` and combines their\n",
"single results with `AND`, `OR`, or `MAJORITY`; `TrueFalseInverterScorer` accepts one\n",
"`TrueFalseScorer`. `FloatScaleThresholdScorer` is the cross-kind adapter: it accepts one\n",
"`FloatScaleScorer` and produces a `TrueFalseScorer`. `create_conversation_scorer()`\n",
"accepts only those two base types and returns a dynamic wrapper that remains the same\n",
"scorer kind as its input.\n",
"\n",
"For example, float-scale → conversation → threshold →\n",
"inversion is supported; a generic `Scorer` outside those base types is not."
]
},
{
Expand Down
58 changes: 58 additions & 0 deletions doc/code/scoring/3_combining_scorers.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,64 @@
# These wrappers are themselves scorers, so they plug into attacks and the batch scorer
# exactly like the leaf scorers on the [True/False](1_true_false_scorers.ipynb) and
# [Float-scale](2_float_scale_scorers.ipynb) pages.
#
# The [class hierarchy](0_scoring.ipynb#the-class-hierarchy) explains what each wrapper
# *is*. This diagram instead shows runtime composition: what each wrapper may contain.
# Solid arrows pass a scorer through `scorer=` or `scorers=`, while dashed arrows show
# the scorer base implemented by the resulting wrapper. An "any" input can therefore be
# a leaf scorer or an already composed wrapper with that base, which enables stacking.
#
# ```mermaid
# flowchart LR
# subgraph inputs["Supported inputs"]
# direction TB
# TF["Any TrueFalseScorer<br/>(leaf or previously wrapped)"]
# FS["Any FloatScaleScorer<br/>(leaf or previously wrapped)"]
# end
#
# subgraph wrappers["Composition wrappers"]
# direction TB
# COMP["TrueFalseCompositeScorer<br/>AND · OR · MAJORITY"]
# INV["TrueFalseInverterScorer<br/>negates one result"]
# THRESH["FloatScaleThresholdScorer<br/>score ≥ threshold"]
# CONV["create_conversation_scorer()<br/>scores concatenated history"]
# end
#
# subgraph outputs["Resulting scorer kind"]
# direction TB
# TFOUT["TrueFalseScorer<br/>can be stacked again"]
# FSOUT["FloatScaleScorer<br/>can be stacked again"]
# end
#
# TF -->|"1+ via scorers="| COMP
# TF -->|"1 via scorer="| INV
# FS -->|"1 via scorer="| THRESH
# TF -->|"1 via scorer="| CONV
# FS -->|"1 via scorer="| CONV
#
# COMP -. is a .-> TFOUT
# INV -. is a .-> TFOUT
# THRESH -. is a .-> TFOUT
# CONV -. "for true/false input" .-> TFOUT
# CONV -. "for float-scale input" .-> FSOUT
#
# classDef input fill:#e8f0fe,stroke:#4285f4,color:#15233a;
# classDef wrapper fill:#fff4e5,stroke:#f29900,color:#3d2600;
# classDef output fill:#e6f4ea,stroke:#34a853,color:#17351f;
# class TF,FS input;
# class COMP,INV,THRESH,CONV wrapper;
# class TFOUT,FSOUT output;
# ```
#
# `TrueFalseCompositeScorer` requires at least one `TrueFalseScorer` and combines their
# single results with `AND`, `OR`, or `MAJORITY`; `TrueFalseInverterScorer` accepts one
# `TrueFalseScorer`. `FloatScaleThresholdScorer` is the cross-kind adapter: it accepts one
# `FloatScaleScorer` and produces a `TrueFalseScorer`. `create_conversation_scorer()`
# accepts only those two base types and returns a dynamic wrapper that remains the same
# scorer kind as its input.
#
# For example, float-scale → conversation → threshold →
# inversion is supported; a generic `Scorer` outside those base types is not.
# %%
from pyrit.setup import IN_MEMORY, initialize_pyrit_async

Expand Down