diff --git a/docs/testing.md b/docs/testing.md index e5d2143..da9c4fb 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -70,7 +70,7 @@ PHP branch support was verified locally on 2026-09-17 on PHP 8.2.33, 8.3.33, 8.4 | Layer | Scope and counting | | --- | --- | -| Python worker | 406 tests: lifecycle, validation and HTTP, the per-token capacity cap (one token refused at its cap with its own `Retry-After` while another submits, a repeated key answered there, the total still enforced with the `Retry-After` of whichever limit applies, jobs restored from the table counted until they expire, the default unchanged, and every out-of-range value refused at start), the optional JSON log (every line parses and holds only its event's keys, an input, token, header, query string, idempotency key and a failing task's exception text never reach it, `off` writes nothing and a bad `BRIDGE_LOG` stops the start, the route templates, the 401 without an owner, consistent job timings, and a stderr that fails or blocks costing a request nothing), the drain (submissions refused with 503, queued jobs failed at once, running ones finished within `BRIDGE_DRAIN_SECONDS`, a second signal), `/readyz` and its reasons for any configured token, the 503 for a full connection budget and the 431 for an oversized request line or header block, per-client tokens (each source and every way it is refused at start, a job and its idempotency key scoped to the submitting token, through the job table and a table from before owners, the `owner` metrics series, and no token in an error, the metrics, the table, a child's environment or a start message), the job table in SQLite (a store reopened on its file fails unfinished jobs as `service_restarted`, serves retained results, deletes expired rows from the file and never holds an input; a submission the table refuses is a 429, a later write it refuses leaves the job served, and a key whose row outlived a failed delete runs again) and idempotency keys, process crashes, cancellation, timeouts, retention, the Retry-After a full or closing store gives and its header on every 429, allocation/exit races, the warm pool (that successive jobs share a child in which no live container or frame holds a reference to a finished job's input or result, after a success, a failure and real task payloads, with the detector checked against a planted needle; that a cancelled, timed-out or crashed child is replaced; that a child is recycled after its share of jobs; that reads, submissions and cancellations answer at once while a child is being started, that a child whose start ends after the store closed does not outlive it, and that a job assigned to or waiting for a live child does not fail when no replacement can start; that a job whose child died before taking it is tried once more on another, only once, that a job sent but not yet taken can still be cancelled or time out, and that a job whose child crashed after taking it is not; that a child, idle or running a job, ignores the SIGINT a terminal's Ctrl+C sends the whole process group; that the service token is absent from a child's environment and from its environment block when the service started it; that a message for an earlier job lands nowhere; that a child that never reports ready is replaced and that children dying at start are not restarted in a loop; and that a cancellation does not hold up reads), redaction rules including IPv6, lower-case IBANs and email addresses in any script and their speed on hostile input, NER aggregation, name joining, the second title-cased reading, the uncased model's reading of the whole text and the merge driven by stubs, the fetcher's pins and the model directories an ONNX worker checks for before it starts, batched cross-encoder scoring, the NER measurement's sample, scoring and WNUT-17 reader, the load measurement's settings, percentiles, error counting and table, the ranking measurement's metrics, BM25, dataset reader and fit to the worker's limits, the public datasets' pins and licence gate, the workflows' runners, action pins and checkout credentials, both Claude Code hooks (the post-edit rules, and the pre-call push guard through its own cases), progress bounds, the service's reading of its BRIDGE_* settings (defaults, every variable, malformed and out-of-range values, a state directory that cannot hold the table), and the model-name table every result reports from | +| Python worker | 407 tests: lifecycle, validation and HTTP, the per-token capacity cap (one token refused at its cap with its own `Retry-After` while another submits, a repeated key answered there, the total still enforced with the `Retry-After` of whichever limit applies, jobs restored from the table counted until they expire, the default unchanged, and every out-of-range value refused at start), the optional JSON log (every line parses and holds only its event's keys, an input, token, header, query string, idempotency key and a failing task's exception text never reach it, `off` writes nothing and a bad `BRIDGE_LOG` stops the start, the route templates, the 401 without an owner, consistent job timings, and a stderr that fails or blocks costing a request nothing), the drain (submissions refused with 503, queued jobs failed at once, running ones finished within `BRIDGE_DRAIN_SECONDS`, a second signal), `/readyz` and its reasons for any configured token, the 503 for a full connection budget and the 431 for an oversized request line or header block, per-client tokens (each source and every way it is refused at start, a job and its idempotency key scoped to the submitting token, through the job table and a table from before owners, the `owner` metrics series, and no token in an error, the metrics, the table, a child's environment or a start message), the job table in SQLite (a store reopened on its file fails unfinished jobs as `service_restarted`, serves retained results, deletes expired rows from the file and never holds an input; a submission the table refuses is a 429, a later write it refuses leaves the job served, and a key whose row outlived a failed delete runs again) and idempotency keys, process crashes, cancellation, timeouts, retention, the Retry-After a full or closing store gives and its header on every 429, allocation/exit races, the warm pool (that successive jobs share a child in which no live container or frame holds a reference to a finished job's input or result, after a success, a failure and real task payloads, with the detector checked against a planted needle; that a cancelled, timed-out or crashed child is replaced; that a child is recycled after its share of jobs; that reads, submissions and cancellations answer at once while a child is being started, that a child whose start ends after the store closed does not outlive it, and that a job assigned to or waiting for a live child does not fail when no replacement can start; that a job whose child died before taking it is tried once more on another, only once, that a job sent but not yet taken can still be cancelled or time out, and that a job whose child crashed after taking it is not; that a child, idle or running a job, ignores the SIGINT a terminal's Ctrl+C sends the whole process group; that the service token is absent from a child's environment and from its environment block when the service started it; that a message for an earlier job lands nowhere; that a child that never reports ready is replaced and that children dying at start are not restarted in a loop; and that a cancellation does not hold up reads), redaction rules including IPv6, lower-case IBANs and email addresses in any script and their speed on hostile input, NER aggregation, name joining, the second title-cased reading, the uncased model's reading of the whole text and the merge driven by stubs, the fetcher's pins and the model directories an ONNX worker checks for before it starts, batched cross-encoder scoring, the NER measurement's sample, scoring and WNUT-17 reader, the load measurement's settings, percentiles, error counting and table, the ranking measurement's metrics, BM25, dataset reader and fit to the worker's limits, the public datasets' pins and licence gate, the workflows' runners, action pins and checkout credentials, both Claude Code hooks (the post-edit rules, and the pre-call push guard through its own cases), progress bounds, the service's reading of its BRIDGE_* settings (defaults, every variable, malformed and out-of-range values, a state directory that cannot hold the table), and the model-name table every result reports from | | MCP server | 125 tests, offline with hash-pinned wheels: the protocol client against an in-process worker, every tool through the SDK's in-memory client with test tasks enabled, the entry point as a subprocess, result validators on synthetic data, the index builder (chunking, the format and each corruption it refuses, building against the worker, caching, its command line) `bridge_search` (the model gate, path confinement, reloads, rerank on and off), the reachability of every answer in the LLM evaluation, that a default search sees only part of its corpus and that its two-step question takes two steps, that the demo worker reaches none of the semantic evaluation's answers, that a tool waits out a full worker for its Retry-After within the call's budget or with its own backoff without one, before each of a search's jobs and within one budget, and says so when it stays full, the evaluation runner's command, isolation checks, scoring and per-backend launcher and index check without starting `claude`, a check that the server's model names match the worker's, that every path parameter says where it resolves and every position in a result says what it counts, and a check that every timing claim in a tool description is a measured median | | PHP contracts + HTTP | 192 checks per PHP version, a full fault-worker's Retry-After read through the client among them and a two-token `tokens-worker` where another token's read and cancel are 404 and the same idempotency key is two jobs (on which `tests/integration.py` then fills one token's `BRIDGE_OWNER_CAPACITY` of 8 while the other still submits up to the worker's capacity of 12); this **includes** the 148 contract checks (Retry-After from a loopback peer, present, absent, malformed and repeated, and a header block over 16 KiB refused), not 192 + 148; the contract checks include a fake transport's view of each request, the wait deadline across a transport failure, and that no file in `src/` outside `Laravel/` and `Symfony/` names a framework | | PSR-18 transport | The 148 contract checks plus 30 through `Psr18Transport` and in-repo fakes of the PSR-7, PSR-17 and PSR-18 interfaces (`tests/php/psr18_fakes.php`): a round trip with its method, URL, headers and body, a body at the 262144-byte cap and one over it read one byte past the cap and no further, five redirect statuses refused unread, a non-JSON or missing content type, malformed JSON, a 429's Retry-After from one header, a repeated one and none, a client exception and a body read failure as `transport_error` without their text; 178 checks by `php-psr18-tests` with `.cache/php-vendor` on the include path, once, in the MCP server job; `php-tests` skips them with one printed line; no real Symfony HttpClient or Guzzle runs | @@ -230,29 +230,29 @@ docker rm -f bridge-load-lexical bridge-load-worker ## Measured NER precision and recall (informational) -`scripts/ner_measure.py` runs every sentence of `tests/fixtures/ner-sample.json` through the ONNX worker's redact task with PER, ORG and LOC, at the task's default threshold and a lower one, each its own job, and compares the models' spans with the sample's labels; rule spans are not counted. The model workflow prints a fresh table to its job summary on every run, and fails only if a job fails, never on a score. The table below is from that workflow, run 36902794112 on pull request #19 at branch head `e49e388`; the commit in its header is GitHub's merge ref. The sample is invented and small, so these figures describe these pinned models on these three registers, not a quality guarantee. +`scripts/ner_measure.py` runs every sentence of `tests/fixtures/ner-sample.json` through the ONNX worker's redact task with PER, ORG and LOC, at the task's default threshold and a lower one, each its own job, and compares the models' spans with the sample's labels; rule spans are not counted. The model workflow prints a fresh table to its job summary on every run, and fails only if a job fails, never on a score. The table below was measured locally with this script over `main` at `dd2cd9d`, the commit its header names; a label's Masked counts that label's entities hidden under any label, so it can exceed the label's overlapping recall. The sample is invented and small, so these figures describe these pinned models on these three registers, not a quality guarantee. -Measured 2026-10-01 UTC, commit `d5b3edc039b6370b51f5fe455785b88ef2aa5e29`, runner: GitHub-hosted ubuntu-latest, CPU: AMD EPYC 7763 64-Core Processor. -Model `Xenova/bert-base-NER:int8+Xenova/bert-base-NER-uncased:int8`; `tests/fixtures/ner-sample.json`, 48 invented sentences with 104 labelled entities; one redact job per sentence and threshold, entities PER, ORG and LOC. Exact: same label and boundaries. Overlapping: same label and a shared character, one gold span per predicted span. Masked: a shared character under any label, the share of entities a redaction hides. +Measured 2026-10-26 UTC, commit `dd2cd9d`, runner: local, Windows 11 Pro with Docker Desktop 29.8.2, CPU: AMD Ryzen 9 9950X 16-Core Processor. +Model `Xenova/bert-base-NER:int8+Xenova/bert-base-NER-uncased:int8`; `tests/fixtures/ner-sample.json`, 48 invented sentences with 104 labelled entities; one redact job per sentence and threshold, entities PER, ORG and LOC. Exact: same label and boundaries. Overlapping: same label and a shared character, one gold span per predicted span. Masked: a shared character under any label, one gold span per predicted span, the share of entities a redaction hides; a label's row counts that label's entities hidden under any label. | Threshold | Register or label | Gold | Predicted | Precision, exact | Recall, exact | Precision, overlapping | Recall, overlapping | Masked | | --- | --- | --- | --- | --- | --- | --- | --- | --- | | 0.85 | news | 38 | 38 | 1.00 | 1.00 | 1.00 | 1.00 | 1.00 | -| 0.85 | informal | 30 | 25 | 0.72 | 0.60 | 0.84 | 0.70 | 0.80 | +| 0.85 | informal | 30 | 24 | 0.75 | 0.60 | 0.88 | 0.70 | 0.80 | | 0.85 | records | 36 | 36 | 0.94 | 0.94 | 0.94 | 0.94 | 1.00 | -| 0.85 | PER | 42 | 37 | 0.92 | 0.81 | 1.00 | 0.88 | 0.88 | -| 0.85 | ORG | 24 | 24 | 0.88 | 0.88 | 0.88 | 0.88 | 0.88 | -| 0.85 | LOC | 38 | 38 | 0.92 | 0.92 | 0.92 | 0.92 | 0.92 | +| 0.85 | PER | 42 | 37 | 0.92 | 0.81 | 1.00 | 0.88 | 0.93 | +| 0.85 | ORG | 24 | 23 | 0.91 | 0.88 | 0.91 | 0.88 | 0.96 | +| 0.85 | LOC | 38 | 38 | 0.92 | 0.92 | 0.92 | 0.92 | 0.95 | | 0.5 | news | 38 | 38 | 1.00 | 1.00 | 1.00 | 1.00 | 1.00 | -| 0.5 | informal | 30 | 33 | 0.67 | 0.73 | 0.73 | 0.80 | 1.00 | +| 0.5 | informal | 30 | 31 | 0.68 | 0.70 | 0.74 | 0.77 | 0.97 | | 0.5 | records | 36 | 36 | 0.94 | 0.94 | 0.94 | 0.94 | 1.00 | -| 0.5 | PER | 42 | 38 | 0.95 | 0.86 | 1.00 | 0.90 | 0.90 | -| 0.5 | ORG | 24 | 30 | 0.73 | 0.92 | 0.73 | 0.92 | 0.92 | -| 0.5 | LOC | 38 | 39 | 0.92 | 0.95 | 0.92 | 0.95 | 0.95 | +| 0.5 | PER | 42 | 38 | 0.95 | 0.86 | 1.00 | 0.90 | 1.00 | +| 0.5 | ORG | 24 | 28 | 0.75 | 0.88 | 0.75 | 0.88 | 0.96 | +| 0.5 | LOC | 38 | 39 | 0.92 | 0.95 | 0.92 | 0.95 | 1.00 | Since pull request #18, the redact task reads a sentence written all in lower case or all in capitals a second time, from a title-cased copy with the same offsets, and likewise a surname in capitals before a comma and a given name (`WIERZBICKI, Tomasz`), the records convention. It joins pieces of one name that touch or meet at a hyphen, full stop or apostrophe before the threshold, scored by the most confident piece, so the initial of `A. Okonkwo` carries the surname; and of two readings of the same span it keeps the more confident. Since pull request #19, the uncased `dslim/bert-base-NER-uncased` reads the whole text once more, ignoring case, beside the cased model, which keeps what capitals tell in news and records: alone, the uncased model masked only 0.54 of the development set's records against 0.77 for the cased readings. Those choices were made on a separate development set, `tests/fixtures/ner-dev.json`, which before the uncased model was measured gained lower-case names among capitals, news sentences and sentences of common words that are also names (`pay the bill`, `a rose`, `Will the courier`); the uncased model invented no name in any of them. This sample stayed unseen until each final measurement; both sets are small and were written by the same author, so the figures describe these sentences, not informal text in general. -Against pull request #18's run on the same CPU type, exact recall at 0.85 rose from 0.47 to 0.60 in informal messages and from 0.92 to 0.94 in records, and news stayed at 1.00; the share of entities masked under any label, which is what a redaction hides, rose from 0.63 to 0.80 in informal messages and from 0.97 to 1.00 in records, and at 0.5 it is 1.00 in all three. Exact precision held at 0.94 in records and moved from 0.74 to 0.72 in informal messages. What still leaks at 0.85: a name after a sign-off (`Regards, olumide`), a handle (`@nadia`), some lower-case places and first names (`porto`, `galway`, `fatima`) and a capitalised company in an informal note (`Halvern`). A reading can still take a filler word into a name (`lol tomasz`) or give a name the wrong label, which still hides it, and the uncased model tags a domain name such as `example.com` as an organisation. +In the model workflow's run for pull request #19 on GitHub's EPYC runners, against pull request #18's run on the same CPU type, exact recall at 0.85 rose from 0.47 to 0.60 in informal messages and from 0.92 to 0.94 in records, and news stayed at 1.00; the share of entities masked under any label, which is what a redaction hides, rose from 0.63 to 0.80 in informal messages and from 0.97 to 1.00 in records, and at 0.5 it is 1.00 in all three. Exact precision held at 0.94 in records and moved from 0.74 to 0.72 in informal messages. The local table above differs from that run in informal messages, with 24 spans instead of 25 at 0.85 (exact precision 0.75) and 31 instead of 33 at 0.5, where 0.97 of the entities are masked, and in ORG, with 23 spans instead of 24 at 0.85 and 28 instead of 30 at 0.5. What still leaks at 0.85: a name after a sign-off (`Regards, olumide`), a handle (`@nadia`), some lower-case places and first names (`porto`, `galway`, `fatima`) and a capitalised company in an informal note (`Halvern`). A reading can still take a filler word into a name (`lol tomasz`) or give a name the wrong label, which still hides it, and the uncased model tags a domain name such as `example.com` as an organisation. int8 inference is not bit-identical across CPUs: a local run on an AMD Ryzen 9 9950X has differed from GitHub's EPYC runners by a span or two, so read such differences as noise. The redact tool's description quotes these figures; re-measure and update both together. To reproduce locally after acquiring the models: @@ -293,21 +293,21 @@ docker compose -f docker/compose.yaml --profile model down `NER_DATASET=wnut17 scripts/ner_measure.py` runs the WNUT-17 test split (Derczynski et al., emerging and rare entities in social media and forum posts) through the redact task, one sentence per job, at the same two thresholds as the invented sample. The split is CC BY 4.0 as its repository's README states, pinned to a commit and a SHA-256 in `scripts/public_data.py` and stored by `quality-fetcher`. Its person entities count as PER, corporation and group as ORG, and location as LOC; product and creative-work have no label among the task's three, so they are not gold spans and a span over one counts against precision. Rule spans are not counted. The table adds F1 to the invented sample's columns and replaces its registers with one row over all three labels. -Measured 2026-10-21 UTC, commit `e77310f`, runner: local, Windows 11 with Docker Desktop, CPU: AMD Ryzen 9 9950X 16-Core Processor. -Model `Xenova/bert-base-NER:int8+Xenova/bert-base-NER-uncased:int8`; WNUT-17 test split, `emerging.test.annotated` at SHA-256 `2aa79b764e56`, CC-BY-4.0: 1287 sentences with 810 entities mapped to labels, person to PER, corporation and group to ORG, location to LOC; product and creative-work have no label here, so a span over one counts against precision. Each sentence is its tokens joined by single spaces; one redact job per sentence and threshold, entities PER, ORG and LOC. Exact: same label and boundaries. Overlapping: same label and a shared character, one gold span per predicted span. Masked: gold spans that share a character with a predicted span of any label, one gold span per predicted span; a label's row counts only that label's predictions, so there it equals overlapping recall. +Measured 2026-10-26 UTC, commit `dd2cd9d`, runner: local, Windows 11 Pro with Docker Desktop 29.8.2, CPU: AMD Ryzen 9 9950X 16-Core Processor. +Model `Xenova/bert-base-NER:int8+Xenova/bert-base-NER-uncased:int8`; WNUT-17 test split, `emerging.test.annotated` at SHA-256 `2aa79b764e56`, CC-BY-4.0: 1287 sentences with 810 entities mapped to labels, person to PER, corporation and group to ORG, location to LOC; product and creative-work have no label here, so a span over one counts against precision. Each sentence is its tokens joined by single spaces; one redact job per sentence and threshold, entities PER, ORG and LOC. Exact: same label and boundaries. Overlapping: same label and a shared character, one gold span per predicted span. Masked: gold spans that share a character with a predicted span of any label, one gold span per predicted span; a label's row counts that label's gold spans hidden under any label, so it can exceed overlapping recall. | Threshold | Label | Gold | Predicted | Precision, exact | Recall, exact | F1, exact | Precision, overlapping | Recall, overlapping | F1, overlapping | Masked | | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | | 0.85 | all labels | 810 | 939 | 0.42 | 0.49 | 0.45 | 0.45 | 0.52 | 0.48 | 0.59 | -| 0.85 | PER | 429 | 310 | 0.75 | 0.54 | 0.63 | 0.78 | 0.56 | 0.65 | 0.56 | -| 0.85 | ORG | 231 | 487 | 0.17 | 0.36 | 0.23 | 0.19 | 0.40 | 0.26 | 0.40 | -| 0.85 | LOC | 150 | 142 | 0.57 | 0.54 | 0.55 | 0.62 | 0.59 | 0.60 | 0.59 | +| 0.85 | PER | 429 | 310 | 0.75 | 0.54 | 0.63 | 0.78 | 0.56 | 0.65 | 0.63 | +| 0.85 | ORG | 231 | 487 | 0.17 | 0.36 | 0.23 | 0.19 | 0.40 | 0.26 | 0.45 | +| 0.85 | LOC | 150 | 142 | 0.57 | 0.54 | 0.55 | 0.62 | 0.59 | 0.60 | 0.69 | | 0.5 | all labels | 810 | 1892 | 0.24 | 0.55 | 0.33 | 0.26 | 0.60 | 0.36 | 0.78 | -| 0.5 | PER | 429 | 395 | 0.63 | 0.58 | 0.61 | 0.67 | 0.62 | 0.65 | 0.62 | -| 0.5 | ORG | 231 | 1278 | 0.08 | 0.45 | 0.14 | 0.09 | 0.52 | 0.16 | 0.52 | -| 0.5 | LOC | 150 | 219 | 0.43 | 0.63 | 0.51 | 0.48 | 0.70 | 0.57 | 0.70 | +| 0.5 | PER | 429 | 395 | 0.63 | 0.58 | 0.61 | 0.67 | 0.62 | 0.65 | 0.85 | +| 0.5 | ORG | 231 | 1278 | 0.08 | 0.45 | 0.14 | 0.09 | 0.52 | 0.16 | 0.59 | +| 0.5 | LOC | 150 | 219 | 0.43 | 0.63 | 0.51 | 0.48 | 0.70 | 0.57 | 0.85 | -On real user-generated text the models find about half of the entities at the default threshold: exact F1 0.45 over all three labels, 0.63 for persons (precision 0.75, recall 0.54), 0.55 for places and 0.23 for organisations, where 487 predicted spans meet 231 gold ones. At 0.85, 0.59 of the entities share a character with a predicted span of some label (the Masked column, one entity per predicted span; an overlap, not a count of characters hidden), against 0.80 in the invented sample's informal messages; at 0.5 the share is 0.78, at the cost of 1278 ORG spans, nine in ten of them wrong. Part of the ORG excess may be WNUT-17's 269 products and creative works, which this mapping leaves without a label; how much was not counted. WNUT-17 was built from entities that are rare or new, which makes it harder than text naming well-known people and companies. +On real user-generated text the models find about half of the entities at the default threshold: exact F1 0.45 over all three labels, 0.63 for persons (precision 0.75, recall 0.54), 0.55 for places and 0.23 for organisations, where 487 predicted spans meet 231 gold ones. At 0.85, 0.59 of the entities share a character with a predicted span of some label (the Masked column, one entity per predicted span; an overlap, not a count of characters hidden; by label 0.63 of persons, 0.45 of organisations and 0.69 of places, some of them under another label), against 0.80 in the invented sample's informal messages; at 0.5 the share is 0.78, at the cost of 1278 ORG spans, nine in ten of them wrong. Part of the ORG excess may be WNUT-17's 269 products and creative works, which this mapping leaves without a label; how much was not counted. WNUT-17 was built from entities that are rare or new, which makes it harder than text naming well-known people and companies. These figures carry what the models do with this public, English, informal text, at character-level matching on tokens joined by single spaces, so `>` reads as `& gt ;` and punctuation stands apart from words, as in the file. They are not WNUT-17's own entity F1, which counts six types; and they are one run: int8 inference is not bit-identical across CPUs, as the section above notes. The redact tool's description still quotes the invented sample's figures. To reproduce, run `model-quality` as above, or the NER half alone: diff --git a/scripts/ner_measure.py b/scripts/ner_measure.py index 6e85f02..7b82d37 100644 --- a/scripts/ner_measure.py +++ b/scripts/ner_measure.py @@ -95,15 +95,15 @@ def predictions(spans: list) -> list: return [(span["start"], span["end"], span["label"]) for span in spans if span["source"].startswith("model:")] -def matched(gold: list, predicted: list, same_label: bool) -> int: +def matched(gold: list, predicted: list, same_label: bool) -> list: """Gold spans overlapped by a predicted span, each predicted span matching one gold span at most.""" - unused, count = list(predicted), 0 + unused, found = list(predicted), [] for start, end, label in gold: match = next((p for p in unused if (p[2] == label or not same_label) and p[0] < end and start < p[1]), None) if match is not None: unused.remove(match) - count += 1 - return count + found.append((start, end, label)) + return found def score(gold: list, predicted: list) -> dict: @@ -111,7 +111,7 @@ def score(gold: list, predicted: list) -> dict: span, so one span over two names finds one of them. Masked: a shared character under any label, which is what a redaction hides.""" return {"gold": len(gold), "predicted": len(predicted), "exact": len(set(gold) & set(predicted)), - "overlapping": matched(gold, predicted, True), "masked": matched(gold, predicted, False)} + "overlapping": len(matched(gold, predicted, True)), "masked": len(matched(gold, predicted, False))} def rate(part: int, whole: int) -> str: @@ -171,7 +171,8 @@ def add(total: dict, counts: dict) -> None: def tally(items: list, spans: dict) -> list: """Rows of (threshold, register or label, counts) from each threshold's spans per item: each register in the - sample's order, then each label, whose rows count only that label's gold and predicted spans.""" + sample's order, then each label, whose rows count only that label's gold and predicted spans, except Masked: + the label's gold spans hidden under a predicted span of any label, in the same matching as the whole.""" registers = list(dict.fromkeys(item["register"] for item in items)) rows = [] for threshold, found_per_item in spans.items(): @@ -179,8 +180,12 @@ def tally(items: list, spans: dict) -> list: for item, found in zip(items, found_per_item): gold, predicted = gold_spans(item), predictions(found) add(totals[item["register"]], score(gold, predicted)) + hidden = matched(gold, predicted, False) for label in LABELS: - add(totals[label], score([g for g in gold if g[2] == label], [p for p in predicted if p[2] == label])) + counts = score([g for g in gold if g[2] == label], [p for p in predicted if p[2] == label]) + # One matching for every row, so one span over two names masks one of them in the labels' rows too. + counts["masked"] = sum(1 for g in hidden if g[2] == label) + add(totals[label], counts) rows += [(threshold, name, counts) for name, counts in totals.items()] return rows @@ -207,13 +212,13 @@ def main() -> None: if wnut: pinned = public_data.DATASETS["wnut17"] print(f"""Measured {now} UTC, commit `{os.environ.get("BENCH_COMMIT", "unknown")}`, runner: {os.environ.get("BENCH_RUNNER", "unspecified")}, CPU: {cpu_model()}. -Model {", ".join(f"`{model}`" for model in sorted(models))}; WNUT-17 test split, `{pinned["url"].rsplit("/", 1)[1]}` at SHA-256 `{pinned["sha256"][:12]}`, {pinned["licence"]}: {len(items)} sentences with {sum(len(item["gold"]) for item in items)} entities mapped to labels, person to PER, corporation and group to ORG, location to LOC; product and creative-work have no label here, so a span over one counts against precision. Each sentence is its tokens joined by single spaces; one redact job per sentence and threshold, entities PER, ORG and LOC. Exact: same label and boundaries. Overlapping: same label and a shared character, one gold span per predicted span. Masked: gold spans that share a character with a predicted span of any label, one gold span per predicted span; a label's row counts only that label's predictions, so there it equals overlapping recall. +Model {", ".join(f"`{model}`" for model in sorted(models))}; WNUT-17 test split, `{pinned["url"].rsplit("/", 1)[1]}` at SHA-256 `{pinned["sha256"][:12]}`, {pinned["licence"]}: {len(items)} sentences with {sum(len(item["gold"]) for item in items)} entities mapped to labels, person to PER, corporation and group to ORG, location to LOC; product and creative-work have no label here, so a span over one counts against precision. Each sentence is its tokens joined by single spaces; one redact job per sentence and threshold, entities PER, ORG and LOC. Exact: same label and boundaries. Overlapping: same label and a shared character, one gold span per predicted span. Masked: gold spans that share a character with a predicted span of any label, one gold span per predicted span; a label's row counts that label's gold spans hidden under any label, so it can exceed overlapping recall. """) print(f1_table(tally(items, spans))) return entities = sum(len(item["entities"]) for item in items) print(f"""Measured {now} UTC, commit `{os.environ.get("BENCH_COMMIT", "unknown")}`, runner: {os.environ.get("BENCH_RUNNER", "unspecified")}, CPU: {cpu_model()}. -Model {", ".join(f"`{model}`" for model in sorted(models))}; `{sample.relative_to(REPO).as_posix()}`, {len(items)} invented sentences with {entities} labelled entities; one redact job per sentence and threshold, entities PER, ORG and LOC. Exact: same label and boundaries. Overlapping: same label and a shared character, one gold span per predicted span. Masked: a shared character under any label, the share of entities a redaction hides. +Model {", ".join(f"`{model}`" for model in sorted(models))}; `{sample.relative_to(REPO).as_posix()}`, {len(items)} invented sentences with {entities} labelled entities; one redact job per sentence and threshold, entities PER, ORG and LOC. Exact: same label and boundaries. Overlapping: same label and a shared character, one gold span per predicted span. Masked: a shared character under any label, one gold span per predicted span, the share of entities a redaction hides; a label's row counts that label's entities hidden under any label. | Threshold | Register or label | Gold | Predicted | Precision, exact | Recall, exact | Precision, overlapping | Recall, overlapping | Masked | | --- | --- | --- | --- | --- | --- | --- | --- | --- |""") diff --git a/tests/python/test_ner_measure.py b/tests/python/test_ner_measure.py index 886e30f..93345ce 100644 --- a/tests/python/test_ner_measure.py +++ b/tests/python/test_ner_measure.py @@ -123,10 +123,9 @@ def test_rows_add_up_by_register_and_by_label_for_each_threshold(self): self.assertEqual(counts[(0.85, "informal")], {"gold": 2, "predicted": 0, "exact": 0, "overlapping": 0, "masked": 0}) self.assertEqual(counts[(0.5, "informal")], {"gold": 2, "predicted": 1, "exact": 1, "overlapping": 1, "masked": 1}) - # Acme predicted as a place is an ORG missed and a LOC that is wrong, in each label's own row; in the - # register's row it is still a name masked. + # A wrong label misses ORG recall but still masks Acme in both the register and ORG rows. self.assertEqual(counts[(0.85, "news")], {"gold": 2, "predicted": 2, "exact": 1, "overlapping": 1, "masked": 2}) - self.assertEqual(counts[(0.85, "ORG")], {"gold": 1, "predicted": 0, "exact": 0, "overlapping": 0, "masked": 0}) + self.assertEqual(counts[(0.85, "ORG")], {"gold": 1, "predicted": 0, "exact": 0, "overlapping": 0, "masked": 1}) self.assertEqual(counts[(0.85, "LOC")], {"gold": 1, "predicted": 1, "exact": 0, "overlapping": 0, "masked": 0}) self.assertEqual(counts[(0.85, "PER")], {"gold": 2, "predicted": 1, "exact": 1, "overlapping": 1, "masked": 1}) @@ -196,11 +195,19 @@ def test_parsed_sentences_are_scored_like_the_sample(self): {"start": 12, "end": 16, "label": "ORG", "source": "model:ORG", "score": 0.9}] counts = {name: value for _, name, value in measure.tally(items, {0.85: [found]})} self.assertEqual(counts["all labels"], {"gold": 2, "predicted": 2, "exact": 1, "overlapping": 1, "masked": 2}) - # A label's row sees only that label's predictions, so the ORG span over Oslo masks nothing in LOC's row. - self.assertEqual(counts["LOC"], {"gold": 1, "predicted": 0, "exact": 0, "overlapping": 0, "masked": 0}) + # The ORG span over Oslo is no LOC prediction, but it hides Oslo, so LOC's row counts it masked. + self.assertEqual(counts["LOC"], {"gold": 1, "predicted": 0, "exact": 0, "overlapping": 0, "masked": 1}) self.assertEqual(counts["ORG"], {"gold": 0, "predicted": 1, "exact": 0, "overlapping": 0, "masked": 0}) self.assertEqual(counts["PER"], {"gold": 1, "predicted": 1, "exact": 1, "overlapping": 1, "masked": 1}) + def test_a_label_s_masked_entities_add_up_to_the_all_labels_row(self): + # One span over two names masks one of them, in the labels' rows as in the whole, so they sum to it. + items = measure.parse_conll("Ada\tB-person\nOslo\tB-location\n") + found = [{"start": 0, "end": 8, "label": "ORG", "source": "model:ORG", "score": 0.9}] + counts = {name: value for _, name, value in measure.tally(items, {0.85: [found]})} + self.assertEqual(counts["all labels"]["masked"], 1) + self.assertEqual([counts[label]["masked"] for label in ("PER", "ORG", "LOC")], [1, 0, 0]) + def test_f1_is_the_harmonic_mean_of_precision_and_recall(self): self.assertEqual(measure.f1(3, 4, 6), "0.60") # precision 0.5, recall 0.75 self.assertEqual(measure.f1(0, 4, 0), "0.00")