From ce129e985058fc1e7fa4cd3b3af8ac6f103d6baa Mon Sep 17 00:00:00 2001 From: Babak Naderi Date: Fri, 12 Jun 2026 16:26:27 +0200 Subject: [PATCH 001/111] Replace hardcoded math answers with dynamic hash-based verification - Add generate_math_questions.py to create math audio clips with Azure TTS, random UUID filenames, ASL normalization, and SHA-256 hashes - Add client-side math answer verification via SHA-256 hash in all 11 HTML templates (no raw answers exposed in source) - Move math answers from hardcoded cfg [math] sections to per-project CSV columns (math_ans, math_hash) flowing through general.csv - Update result_parser.py to use math_ans from input data with fallback to legacy config for backward compatibility - Add --general_assets argument to master_script.py for custom asset CSVs - Add general_assets_internal.csv template for internally generated clips - Pass math_ans and math_hash through create_input.py to per-project CSVs - Update documentation for new workflow and arguments Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/conf_master.md | 17 + docs/general_res.md | 7 +- docs/prep_acr.md | 1 + docs/prep_dcr_ccr.md | 1 + docs/prep_p804.md | 1 + docs/prep_p835.md | 1 + src/P808Template/ACR_template.html | 33 +- src/P808Template/CCR_template.html | 33 +- src/P808Template/DCR_template.html | 33 +- src/P808Template/P808_multi.html | 33 +- src/P808Template/P831_ACR_template.html | 33 +- src/P808Template/P831_DCR_template.html | 33 +- .../P835_personalized_template3.html | 33 +- src/P808Template/P835_template.html | 33 +- src/P808Template/P835_template_one_audio.html | 25 +- .../echo_impairment_test_fest_template.html | 33 +- .../echo_impairment_test_template.html | 33 +- .../acr_result_parser_template.cfg | 7 +- .../dcr_ccr_result_parser_template.cfg | 7 +- src/assets_master_script/general.csv | 38 +- .../p804_result_parser_template.cfg | 7 +- .../pp835_result_parser_template.cfg | 7 +- src/create_input.py | 28 + src/master_script.py | 11 +- src/result_parser.py | 53 +- src/utils/generate_math_questions.py | 600 ++++++++++++++++++ 26 files changed, 1068 insertions(+), 73 deletions(-) create mode 100644 src/utils/generate_math_questions.py diff --git a/docs/conf_master.md b/docs/conf_master.md index 404119d..92f5410 100644 --- a/docs/conf_master.md +++ b/docs/conf_master.md @@ -3,6 +3,23 @@ # Configure for `master_script.py` This describes the configuration for the `master_script.py`. A sample configuration file can be found in [`configurations\master.cfg`](.\src\configurations\master.cfg). + +## Command-line arguments + +* `--project`: Name of the project (required). +* `--cfg`: Configuration file path (required). See sections below. +* `--method`: Test method — `acr`, `dcr`, `ccr`, `p835`, `pp835`, `p804`, or `echo_impairment_test` (required). +* `--clips`: CSV with rating clip URLs in column `rating_clips`. +* `--gold_clips`: CSV with gold clip URLs and answers. +* `--training_clips`: CSV with training clip URLs. +* `--trapping_clips`: CSV with trapping clip URLs and answers. +* `--training_gold_clips`: CSV with gold training question details (P.804). +* `--general_assets`: Path to the general assets CSV. Defaults to `assets_master_script/general.csv`. + Use `assets_master_script/general_assets_internal.csv` for projects with internally generated + math clips (see `utils/generate_math_questions.py`). +* `--check_urls`: Validate that all links in the CSV files are accessible. +* `--create_local_test`: Generate a local preview HTML file after the project is created. +* `--p831_fest`: Use the question set of P.831 for FEST. ## `[create_input]` diff --git a/docs/general_res.md b/docs/general_res.md index 6817bff..005ccd5 100644 --- a/docs/general_res.md +++ b/docs/general_res.md @@ -11,8 +11,13 @@ files: - `src/P808Template/P835_template.html` - `src/P808Template/Qualification.html` -1. Upload the links in the `src/assets_master_script/general.csv`: +1. Upload the links in the `src/assets_master_script/general.csv` (or `general_assets_internal.csv` + for internal assets): - Column `math` should contain URLs of files you find here `src/P808Template/assets/clips/math/*`. + You can generate additional math clips with `src/utils/generate_math_questions.py`. + - Column `math_ans` should contain the correct answer (sum) for each math clip. + - Column `math_hash` should contain a SHA-256 hash of the clip URL and answer for client-side + verification. The `generate_math_questions.py` script computes these automatically. - Columns `pair_a`, `pair_b` should contain URLs of files you find here `src/P808Template/assets/clips/environment_test/*`. Use files starting by `40` in `pair_a`, and corresponding file starting by `50` in `pair_b`. \ No newline at end of file diff --git a/docs/prep_acr.md b/docs/prep_acr.md index 7cb5b30..741c9c7 100644 --- a/docs/prep_acr.md +++ b/docs/prep_acr.md @@ -81,6 +81,7 @@ a column named `trapping_clips` and expected answer to each clip in a column nam Optionally: - Add `--check_urls` to validate that all links in the CSV files are accessible before creating the project. - Add `--create_local_test` to generate a local preview HTML file for testing. See [preview_html](preview_html.md) for details. + - Add `--general_assets path/to/general.csv` to use a custom general assets CSV instead of the default `assets_master_script/general.csv`. Note: file paths are expected to be relative to the current working directory. diff --git a/docs/prep_dcr_ccr.md b/docs/prep_dcr_ccr.md index 29ca2a1..8974d31 100644 --- a/docs/prep_dcr_ccr.md +++ b/docs/prep_dcr_ccr.md @@ -47,6 +47,7 @@ column named `training_clips` and URLs to corresponding reference clips in colum Optionally: - Add `--check_urls` to validate that all links in the CSV files are accessible before creating the project. - Add `--create_local_test` to generate a local preview HTML file for testing. See [preview_html](preview_html.md) for details. + - Add `--general_assets path/to/general.csv` to use a custom general assets CSV instead of the default `assets_master_script/general.csv`. Note: file paths are expected to be relative to the current working directory. diff --git a/docs/prep_p804.md b/docs/prep_p804.md index ce7f2f9..9e7fc4f 100644 --- a/docs/prep_p804.md +++ b/docs/prep_p804.md @@ -86,6 +86,7 @@ a column named `trapping_clips` and expected answer to each clip in a column nam Optionally: - Add `--check_urls` to validate that all links in the CSV files are accessible before creating the project. - Add `--create_local_test` to generate a local preview HTML file for testing. See [preview_html](preview_html.md) for details. + - Add `--general_assets path/to/general.csv` to use a custom general assets CSV instead of the default `assets_master_script/general.csv`. Note: file paths are expected to be relative to the current working directory. diff --git a/docs/prep_p835.md b/docs/prep_p835.md index 267f9f6..6e009f4 100644 --- a/docs/prep_p835.md +++ b/docs/prep_p835.md @@ -86,6 +86,7 @@ a column named `trapping_clips` and expected answer to each clip in a column nam Optionally: - Add `--check_urls` to validate that all links in the CSV files are accessible before creating the project. - Add `--create_local_test` to generate a local preview HTML file for testing. See [preview_html](preview_html.md) for details. + - Add `--general_assets path/to/general.csv` to use a custom general assets CSV instead of the default `assets_master_script/general.csv`. Note: file paths are expected to be relative to the current working directory. diff --git a/src/P808Template/ACR_template.html b/src/P808Template/ACR_template.html index c3a7734..6988d22 100644 --- a/src/P808Template/ACR_template.html +++ b/src/P808Template/ACR_template.html @@ -119,6 +119,7 @@ // how many times a feedback was given to the workers, var n_cmp_feedbacks = 0; +var n_setup_feedback = 0; const Hide_HIT_REASON = Object.freeze({"MAX_ACHIVED":1, "QUALIFICATON":2}) /* @@ -542,6 +543,21 @@ }); } +async function verifyMathHash(userAnswer){ + var url = document.getElementById("math1").getAttribute("data-src"); + var mathHash = "${math_hash}"; + if (!mathHash || mathHash === "$" + "{math_hash}") { + return true; + } + var payload = url + ":" + userAnswer.trim(); + var data = new TextEncoder().encode(payload); + var buf = await crypto.subtle.digest("SHA-256", data); + var hex = Array.from(new Uint8Array(buf)).map(function(b) { + return b.toString(16).padStart(2, "0"); + }).join(""); + return hex === mathHash; +} + /* Validate the answers give to the cmps in the setup section and provide an adequate feedback */ @@ -558,8 +574,21 @@ if (!math_ans) return; - // 2. check math question - // do not check math question now + // 2. check math question via SHA-256 hash (client-side pre-check) + verifyMathHash(math_ans).then(function(mathCorrect) { + if (!mathCorrect) { + n_setup_feedback ++; + if (n_setup_feedback < config["cmp_max_n_feedback"]) { + alert("Please listen carefully and answer the math question correctly. Make sure both earbuds are working."); + return; + } + } + validateCMPSectionAfterMath(cmp_answers); + }); + return; +} + +function validateCMPSectionAfterMath(cmp_answers){ // 3. check cmps correct_ans = 0 ; for (i = 0; i < 4; i++) { diff --git a/src/P808Template/CCR_template.html b/src/P808Template/CCR_template.html index b4eb989..43794c2 100644 --- a/src/P808Template/CCR_template.html +++ b/src/P808Template/CCR_template.html @@ -125,6 +125,7 @@ var generalErrorLog= ""; // how many times a feedback was given to the workers, var n_cmp_feedbacks = 0; +var n_setup_feedback = 0; const Hide_HIT_REASON = Object.freeze({"MAX_ACHIVED":1, "QUALIFICATON":2}) /* @@ -499,6 +500,21 @@ }); } +async function verifyMathHash(userAnswer){ + var url = document.getElementById("math1").getAttribute("data-src"); + var mathHash = "${math_hash}"; + if (!mathHash || mathHash === "$" + "{math_hash}") { + return true; + } + var payload = url + ":" + userAnswer.trim(); + var data = new TextEncoder().encode(payload); + var buf = await crypto.subtle.digest("SHA-256", data); + var hex = Array.from(new Uint8Array(buf)).map(function(b) { + return b.toString(16).padStart(2, "0"); + }).join(""); + return hex === mathHash; +} + /* Validate the answers give to the cmps in the setup section and provide an adequate feedback */ @@ -515,8 +531,21 @@ if (!math_ans) return; - // 2. check math question - // do not check math question now + // 2. check math question via SHA-256 hash (client-side pre-check) + verifyMathHash(math_ans).then(function(mathCorrect) { + if (!mathCorrect) { + n_setup_feedback ++; + if (n_setup_feedback < config["cmp_max_n_feedback"]) { + alert("Please listen carefully and answer the math question correctly. Make sure both earbuds are working."); + return; + } + } + validateCMPSectionAfterMath(cmp_answers); + }); + return; +} + +function validateCMPSectionAfterMath(cmp_answers){ // 3. check cmps correct_ans = 0 ; for (i = 0; i < 4; i++) { diff --git a/src/P808Template/DCR_template.html b/src/P808Template/DCR_template.html index ce02a58..335ddf0 100644 --- a/src/P808Template/DCR_template.html +++ b/src/P808Template/DCR_template.html @@ -117,6 +117,7 @@ var generalErrorLog= ""; // how many times a feedback was given to the workers, var n_cmp_feedbacks = 0; +var n_setup_feedback = 0; const Hide_HIT_REASON = Object.freeze({"MAX_ACHIVED":1, "QUALIFICATON":2}) /* @@ -487,6 +488,21 @@ }); } +async function verifyMathHash(userAnswer){ + var url = document.getElementById("math1").getAttribute("data-src"); + var mathHash = "${math_hash}"; + if (!mathHash || mathHash === "$" + "{math_hash}") { + return true; + } + var payload = url + ":" + userAnswer.trim(); + var data = new TextEncoder().encode(payload); + var buf = await crypto.subtle.digest("SHA-256", data); + var hex = Array.from(new Uint8Array(buf)).map(function(b) { + return b.toString(16).padStart(2, "0"); + }).join(""); + return hex === mathHash; +} + /* Validate the answers give to the cmps in the setup section and provide an adequate feedback */ @@ -503,8 +519,21 @@ if (!math_ans) return; - // 2. check math question - // do not check math question now + // 2. check math question via SHA-256 hash (client-side pre-check) + verifyMathHash(math_ans).then(function(mathCorrect) { + if (!mathCorrect) { + n_setup_feedback ++; + if (n_setup_feedback < config["cmp_max_n_feedback"]) { + alert("Please listen carefully and answer the math question correctly. Make sure both earbuds are working."); + return; + } + } + validateCMPSectionAfterMath(cmp_answers); + }); + return; +} + +function validateCMPSectionAfterMath(cmp_answers){ // 3. check cmps correct_ans = 0 ; for (i = 0; i < 4; i++) { diff --git a/src/P808Template/P808_multi.html b/src/P808Template/P808_multi.html index 1bd86c6..e46a0eb 100644 --- a/src/P808Template/P808_multi.html +++ b/src/P808Template/P808_multi.html @@ -353,6 +353,7 @@ var generalErrorLog = ""; // how many times a feedback was given to the workers, var n_cmp_feedbacks = 0; + var n_setup_feedback = 0; const Hide_HIT_REASON = Object.freeze({ "MAX_ACHIVED": 1, "QUALIFICATON": 2, "GOLD_FAILED":3 }) /* Initializing the page: generate and hide sections depending to the cookies value @@ -886,6 +887,21 @@ }); } + async function verifyMathHash(userAnswer) { + var url = document.getElementById("math1").getAttribute("data-src"); + var mathHash = "${math_hash}"; + if (!mathHash || mathHash === "$" + "{math_hash}") { + return true; + } + var payload = url + ":" + userAnswer.trim(); + var data = new TextEncoder().encode(payload); + var buf = await crypto.subtle.digest("SHA-256", data); + var hex = Array.from(new Uint8Array(buf)).map(function(b) { + return b.toString(16).padStart(2, "0"); + }).join(""); + return hex === mathHash; + } + /* Validate the answers give to the cmps in the setup section and provide an adequate feedback */ @@ -902,8 +918,21 @@ if (!math_ans) return; - // 2. check math question - // do not check math question now + // 2. check math question via SHA-256 hash (client-side pre-check) + verifyMathHash(math_ans).then(function(mathCorrect) { + if (!mathCorrect) { + n_setup_feedback ++; + if (n_setup_feedback < config["cmp_max_n_feedback"]) { + alert("Please listen carefully and answer the math question correctly. Make sure both earbuds are working."); + return; + } + } + validateCMPSectionAfterMath(cmp_answers); + }); + return; + } + + function validateCMPSectionAfterMath(cmp_answers) { // 3. check cmps correct_ans = 0; for (i = 0; i < 4; i++) { diff --git a/src/P808Template/P831_ACR_template.html b/src/P808Template/P831_ACR_template.html index 083ab97..b763e1c 100644 --- a/src/P808Template/P831_ACR_template.html +++ b/src/P808Template/P831_ACR_template.html @@ -114,6 +114,7 @@ var generalErrorLog= ""; // how many times a feedback was given to the workers, var n_cmp_feedbacks = 0; +var n_setup_feedback = 0; const Hide_HIT_REASON = Object.freeze({"MAX_ACHIVED":1, "QUALIFICATON":2}) /* @@ -462,6 +463,21 @@ }); } +async function verifyMathHash(userAnswer){ + var url = document.getElementById("math1").getAttribute("data-src"); + var mathHash = "${math_hash}"; + if (!mathHash || mathHash === "$" + "{math_hash}") { + return true; + } + var payload = url + ":" + userAnswer.trim(); + var data = new TextEncoder().encode(payload); + var buf = await crypto.subtle.digest("SHA-256", data); + var hex = Array.from(new Uint8Array(buf)).map(function(b) { + return b.toString(16).padStart(2, "0"); + }).join(""); + return hex === mathHash; +} + /* Validate the answers give to the cmps in the setup section and provide an adequate feedback */ @@ -478,8 +494,21 @@ if (!math_ans) return; - // 2. check math question - // do not check math question now + // 2. check math question via SHA-256 hash (client-side pre-check) + verifyMathHash(math_ans).then(function(mathCorrect) { + if (!mathCorrect) { + n_setup_feedback ++; + if (n_setup_feedback < config["cmp_max_n_feedback"]) { + alert("Please listen carefully and answer the math question correctly. Make sure both earbuds are working."); + return; + } + } + validateCMPSectionAfterMath(cmp_answers); + }); + return; +} + +function validateCMPSectionAfterMath(cmp_answers){ // 3. check cmps correct_ans = 0 ; for (i = 0; i < 4; i++) { diff --git a/src/P808Template/P831_DCR_template.html b/src/P808Template/P831_DCR_template.html index 5157e06..ede7d61 100644 --- a/src/P808Template/P831_DCR_template.html +++ b/src/P808Template/P831_DCR_template.html @@ -114,6 +114,7 @@ var generalErrorLog= ""; // how many times a feedback was given to the workers, var n_cmp_feedbacks = 0; +var n_setup_feedback = 0; const Hide_HIT_REASON = Object.freeze({"MAX_ACHIVED":1, "QUALIFICATON":2}) /* Initializing the page: generate and hide sections depending to the cookies value @@ -422,6 +423,21 @@ }); } +async function verifyMathHash(userAnswer){ + var url = document.getElementById("math1").getAttribute("data-src"); + var mathHash = "${math_hash}"; + if (!mathHash || mathHash === "$" + "{math_hash}") { + return true; + } + var payload = url + ":" + userAnswer.trim(); + var data = new TextEncoder().encode(payload); + var buf = await crypto.subtle.digest("SHA-256", data); + var hex = Array.from(new Uint8Array(buf)).map(function(b) { + return b.toString(16).padStart(2, "0"); + }).join(""); + return hex === mathHash; +} + /* Validate the answers give to the cmps in the setup section and provide an adequate feedback */ @@ -438,8 +454,21 @@ if (!math_ans) return; - // 2. check math question - // do not check math question now + // 2. check math question via SHA-256 hash (client-side pre-check) + verifyMathHash(math_ans).then(function(mathCorrect) { + if (!mathCorrect) { + n_setup_feedback ++; + if (n_setup_feedback < config["cmp_max_n_feedback"]) { + alert("Please listen carefully and answer the math question correctly. Make sure both earbuds are working."); + return; + } + } + validateCMPSectionAfterMath(cmp_answers); + }); + return; +} + +function validateCMPSectionAfterMath(cmp_answers){ // 3. check cmps correct_ans = 0 ; for (i = 0; i < 4; i++) { diff --git a/src/P808Template/P835_personalized_template3.html b/src/P808Template/P835_personalized_template3.html index 3e12003..49bed2d 100644 --- a/src/P808Template/P835_personalized_template3.html +++ b/src/P808Template/P835_personalized_template3.html @@ -205,6 +205,7 @@ var generalErrorLog = ""; // how many times a feedback was given to the workers, var n_cmp_feedbacks = 0; + var n_setup_feedback = 0; const Hide_HIT_REASON = Object.freeze({ "MAX_ACHIVED": 1, "QUALIFICATON": 2, "GOLD_FAILED":3 }) /* Initializing the page: generate and hide sections depending to the cookies value @@ -696,6 +697,21 @@ }); } + async function verifyMathHash(userAnswer) { + var url = document.getElementById("math1").getAttribute("data-src"); + var mathHash = "${math_hash}"; + if (!mathHash || mathHash === "$" + "{math_hash}") { + return true; + } + var payload = url + ":" + userAnswer.trim(); + var data = new TextEncoder().encode(payload); + var buf = await crypto.subtle.digest("SHA-256", data); + var hex = Array.from(new Uint8Array(buf)).map(function(b) { + return b.toString(16).padStart(2, "0"); + }).join(""); + return hex === mathHash; + } + /* Validate the answers give to the cmps in the setup section and provide an adequate feedback */ @@ -712,8 +728,21 @@ if (!math_ans) return; - // 2. check math question - // do not check math question now + // 2. check math question via SHA-256 hash (client-side pre-check) + verifyMathHash(math_ans).then(function(mathCorrect) { + if (!mathCorrect) { + n_setup_feedback ++; + if (n_setup_feedback < config["cmp_max_n_feedback"]) { + alert("Please listen carefully and answer the math question correctly. Make sure both earbuds are working."); + return; + } + } + validateCMPSectionAfterMath(cmp_answers); + }); + return; + } + + function validateCMPSectionAfterMath(cmp_answers) { // 3. check cmps correct_ans = 0; for (i = 0; i < 4; i++) { diff --git a/src/P808Template/P835_template.html b/src/P808Template/P835_template.html index 4083969..2e2148c 100644 --- a/src/P808Template/P835_template.html +++ b/src/P808Template/P835_template.html @@ -128,6 +128,7 @@ var generalErrorLog= ""; // how many times a feedback was given to the workers, var n_cmp_feedbacks = 0; +var n_setup_feedback = 0; const Hide_HIT_REASON = Object.freeze({"MAX_ACHIVED":1, "QUALIFICATON":2}) /* Initializing the page: generate and hide sections depending to the cookies value @@ -538,6 +539,21 @@ }); } +async function verifyMathHash(userAnswer){ + var url = document.getElementById("math1").getAttribute("data-src"); + var mathHash = "${math_hash}"; + if (!mathHash || mathHash === "$" + "{math_hash}") { + return true; + } + var payload = url + ":" + userAnswer.trim(); + var data = new TextEncoder().encode(payload); + var buf = await crypto.subtle.digest("SHA-256", data); + var hex = Array.from(new Uint8Array(buf)).map(function(b) { + return b.toString(16).padStart(2, "0"); + }).join(""); + return hex === mathHash; +} + /* Validate the answers give to the cmps in the setup section and provide an adequate feedback */ @@ -554,8 +570,21 @@ if (!math_ans) return; - // 2. check math question - // do not check math question now + // 2. check math question via SHA-256 hash (client-side pre-check) + verifyMathHash(math_ans).then(function(mathCorrect) { + if (!mathCorrect) { + n_setup_feedback ++; + if (n_setup_feedback < config["cmp_max_n_feedback"]) { + alert("Please listen carefully and answer the math question correctly. Make sure both earbuds are working."); + return; + } + } + validateCMPSectionAfterMath(cmp_answers); + }); + return; +} + +function validateCMPSectionAfterMath(cmp_answers){ // 3. check cmps correct_ans = 0 ; for (i = 0; i < 4; i++) { diff --git a/src/P808Template/P835_template_one_audio.html b/src/P808Template/P835_template_one_audio.html index 574ea92..1645f0c 100644 --- a/src/P808Template/P835_template_one_audio.html +++ b/src/P808Template/P835_template_one_audio.html @@ -844,13 +844,34 @@ } } +async function verifyMathHash(userAnswer){ + var url = document.getElementById("math1").getAttribute("data-src"); + var mathHash = "${math_hash}"; + if (!mathHash || mathHash === "$" + "{math_hash}") { + return true; + } + var payload = url + ":" + userAnswer.trim(); + var data = new TextEncoder().encode(payload); + var buf = await crypto.subtle.digest("SHA-256", data); + var hex = Array.from(new Uint8Array(buf)).map(function(b) { + return b.toString(16).padStart(2, "0"); + }).join(""); + return hex === mathHash; +} + /* Called when user answers to one question from setup (i.e. q2. math question). Here the cookie will be added/updated */ function userAnsweringToSetupQuestions(){ - createCookie(config['cookieName']+"_setup","un_important",config['showSetupEveryMinutes']); - $('input[name="math"]').off('change'); + verifyMathHash($("#Math").val()).then(function(mathCorrect) { + if (!mathCorrect) { + alert("Please listen carefully and answer the math question correctly. Make sure both earbuds are working."); + return; + } + createCookie(config['cookieName']+"_setup","un_important",config['showSetupEveryMinutes']); + $('input[name="math"]').off('change'); + }); } function set_onclick_listeners(){ diff --git a/src/P808Template/echo_impairment_test_fest_template.html b/src/P808Template/echo_impairment_test_fest_template.html index 78804cc..338a14a 100644 --- a/src/P808Template/echo_impairment_test_fest_template.html +++ b/src/P808Template/echo_impairment_test_fest_template.html @@ -121,6 +121,7 @@ var generalErrorLog= ""; // how many times a feedback was given to the workers, var n_cmp_feedbacks = 0; +var n_setup_feedback = 0; const Hide_HIT_REASON = Object.freeze({"MAX_ACHIVED":1, "QUALIFICATON":2}) /* @@ -470,6 +471,21 @@ }); } +async function verifyMathHash(userAnswer){ + var url = document.getElementById("math1").getAttribute("data-src"); + var mathHash = "${math_hash}"; + if (!mathHash || mathHash === "$" + "{math_hash}") { + return true; + } + var payload = url + ":" + userAnswer.trim(); + var data = new TextEncoder().encode(payload); + var buf = await crypto.subtle.digest("SHA-256", data); + var hex = Array.from(new Uint8Array(buf)).map(function(b) { + return b.toString(16).padStart(2, "0"); + }).join(""); + return hex === mathHash; +} + /* Validate the answers give to the cmps in the setup section and provide an adequate feedback */ @@ -486,8 +502,21 @@ if (!math_ans) return; - // 2. check math question - // do not check math question now + // 2. check math question via SHA-256 hash (client-side pre-check) + verifyMathHash(math_ans).then(function(mathCorrect) { + if (!mathCorrect) { + n_setup_feedback ++; + if (n_setup_feedback < config["cmp_max_n_feedback"]) { + alert("Please listen carefully and answer the math question correctly. Make sure both earbuds are working."); + return; + } + } + validateCMPSectionAfterMath(cmp_answers); + }); + return; +} + +function validateCMPSectionAfterMath(cmp_answers){ // 3. check cmps correct_ans = 0 ; for (i = 0; i < 4; i++) { diff --git a/src/P808Template/echo_impairment_test_template.html b/src/P808Template/echo_impairment_test_template.html index 0cc9e07..8ae9a31 100644 --- a/src/P808Template/echo_impairment_test_template.html +++ b/src/P808Template/echo_impairment_test_template.html @@ -121,6 +121,7 @@ var generalErrorLog= ""; // how many times a feedback was given to the workers, var n_cmp_feedbacks = 0; +var n_setup_feedback = 0; const Hide_HIT_REASON = Object.freeze({"MAX_ACHIVED":1, "QUALIFICATON":2}) /* @@ -470,6 +471,21 @@ }); } +async function verifyMathHash(userAnswer){ + var url = document.getElementById("math1").getAttribute("data-src"); + var mathHash = "${math_hash}"; + if (!mathHash || mathHash === "$" + "{math_hash}") { + return true; + } + var payload = url + ":" + userAnswer.trim(); + var data = new TextEncoder().encode(payload); + var buf = await crypto.subtle.digest("SHA-256", data); + var hex = Array.from(new Uint8Array(buf)).map(function(b) { + return b.toString(16).padStart(2, "0"); + }).join(""); + return hex === mathHash; +} + /* Validate the answers give to the cmps in the setup section and provide an adequate feedback */ @@ -486,8 +502,21 @@ if (!math_ans) return; - // 2. check math question - // do not check math question now + // 2. check math question via SHA-256 hash (client-side pre-check) + verifyMathHash(math_ans).then(function(mathCorrect) { + if (!mathCorrect) { + n_setup_feedback ++; + if (n_setup_feedback < config["cmp_max_n_feedback"]) { + alert("Please listen carefully and answer the math question correctly. Make sure both earbuds are working."); + return; + } + } + validateCMPSectionAfterMath(cmp_answers); + }); + return; +} + +function validateCMPSectionAfterMath(cmp_answers){ // 3. check cmps correct_ans = 0 ; for (i = 0; i < 4; i++) { diff --git a/src/assets_master_script/acr_result_parser_template.cfg b/src/assets_master_script/acr_result_parser_template.cfg index e2ca537..c25abd3 100644 --- a/src/assets_master_script/acr_result_parser_template.cfg +++ b/src/assets_master_script/acr_result_parser_template.cfg @@ -12,10 +12,9 @@ expected_votes_per_file: 10 #condition_keys = {{cfg.condition_keys}} [math] -# correct answer to math questions -math1.wav = 3 -math2.wav = 7 -math3.wav = 6 +# Math answers are now provided via the math_ans column in the per-project CSV. +# The [math] section is kept for backward compatibility but can be left empty +# when using the new general.csv with math_ans and math_hash columns. [trapping] # question name that contains the url to of trapping question diff --git a/src/assets_master_script/dcr_ccr_result_parser_template.cfg b/src/assets_master_script/dcr_ccr_result_parser_template.cfg index aa8cbd6..bade351 100644 --- a/src/assets_master_script/dcr_ccr_result_parser_template.cfg +++ b/src/assets_master_script/dcr_ccr_result_parser_template.cfg @@ -12,10 +12,9 @@ expected_votes_per_file: 5 #condition_keys = {{cfg.condition_keys}} [math] -# correct answer to math questions -math1.wav = 3 -math2.wav = 7 -math3.wav = 6 +# Math answers are now provided via the math_ans column in the per-project CSV. +# The [math] section is kept for backward compatibility but can be left empty +# when using the new general.csv with math_ans and math_hash columns. [trapping] # question name that contains the url to of trapping question diff --git a/src/assets_master_script/general.csv b/src/assets_master_script/general.csv index b7e6699..da6e0a3 100644 --- a/src/assets_master_script/general.csv +++ b/src/assets_master_script/general.csv @@ -1,19 +1,19 @@ -math,pair_a,pair_b,hearing_test_url,hearing_test_ans -https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/math/math1.wav,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_jnd/40S_female1.wav,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_jnd/50S_female1.wav,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_hearing_test/s1.wav,246 -https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/math/math2.wav,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_jnd/40S_female2.wav,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_jnd/50S_female2.wav,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_hearing_test/s2.wav,626 -https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/math/math3.wav,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_jnd/40S_male1.wav,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_jnd/50S_male1.wav,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_hearing_test/s3.wav,802 -,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_jnd/40S_male2.wav,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_jnd/50S_male2.wav,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_hearing_test/s4.wav,913 -,,,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_hearing_test/s7.wav,135 -,,,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_hearing_test/s8.wav,156 -,,,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_hearing_test/s9.wav,282 -,,,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_hearing_test/s10.wav,286 -,,,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_hearing_test/s12.wav,340 -,,,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_hearing_test/s13.wav,359 -,,,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_hearing_test/s14.wav,401 -,,,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_hearing_test/s15.wav,468 -,,,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_hearing_test/s16.wav,534 -,,,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_hearing_test/s17.wav,591 -,,,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_hearing_test/s18.wav,628 -,,,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_hearing_test/s19.wav,680 -,,,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_hearing_test/s20.wav,815 -,,,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_hearing_test/s21.wav,962 +math,math_ans,math_hash,pair_a,pair_b,hearing_test_url,hearing_test_ans +https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/math/math1.wav,3,64e89493798b7fdab45235a9c52921b74e7ef071056dceb0f1e4b8d2c87ce1df,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_jnd/40S_female1.wav,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_jnd/50S_female1.wav,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_hearing_test/s1.wav,246 +https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/math/math2.wav,7,071343f6e8a48edc01ecfab7160d4da9501787007168e1cce2ee1345eb633426,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_jnd/40S_female2.wav,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_jnd/50S_female2.wav,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_hearing_test/s2.wav,626 +https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/math/math3.wav,6,d288c7e348c6b7ae478278f18d2ff94f0da0debddc89c7b10edcc216c1f6a576,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_jnd/40S_male1.wav,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_jnd/50S_male1.wav,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_hearing_test/s3.wav,802 +,,,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_jnd/40S_male2.wav,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_jnd/50S_male2.wav,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_hearing_test/s4.wav,913 +,,,,,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_hearing_test/s7.wav,135 +,,,,,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_hearing_test/s8.wav,156 +,,,,,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_hearing_test/s9.wav,282 +,,,,,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_hearing_test/s10.wav,286 +,,,,,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_hearing_test/s12.wav,340 +,,,,,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_hearing_test/s13.wav,359 +,,,,,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_hearing_test/s14.wav,401 +,,,,,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_hearing_test/s15.wav,468 +,,,,,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_hearing_test/s16.wav,534 +,,,,,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_hearing_test/s17.wav,591 +,,,,,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_hearing_test/s18.wav,628 +,,,,,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_hearing_test/s19.wav,680 +,,,,,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_hearing_test/s20.wav,815 +,,,,,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_hearing_test/s21.wav,962 diff --git a/src/assets_master_script/p804_result_parser_template.cfg b/src/assets_master_script/p804_result_parser_template.cfg index 0383955..abfe103 100644 --- a/src/assets_master_script/p804_result_parser_template.cfg +++ b/src/assets_master_script/p804_result_parser_template.cfg @@ -12,10 +12,9 @@ expected_votes_per_file: 10 #condition_keys = {{cfg.condition_keys}} [math] -# correct answer to math questions -math1.wav = 3 -math2.wav = 7 -math3.wav = 6 +# Math answers are now provided via the math_ans column in the per-project CSV. +# The [math] section is kept for backward compatibility but can be left empty +# when using the new general.csv with math_ans and math_hash columns. [trapping] # question name that contains the url to of trapping question diff --git a/src/assets_master_script/pp835_result_parser_template.cfg b/src/assets_master_script/pp835_result_parser_template.cfg index eaef842..21ae315 100644 --- a/src/assets_master_script/pp835_result_parser_template.cfg +++ b/src/assets_master_script/pp835_result_parser_template.cfg @@ -12,10 +12,9 @@ expected_votes_per_file: 10 #condition_keys = {{cfg.condition_keys}} [math] -# correct answer to math questions -math1.wav = 3 -math2.wav = 7 -math3.wav = 6 +# Math answers are now provided via the math_ans column in the per-project CSV. +# The [math] section is kept for backward compatibility but can be left empty +# when using the new general.csv with math_ans and math_hash columns. [trapping] # question name that contains the url to of trapping question diff --git a/src/create_input.py b/src/create_input.py index 4729dda..d36341b 100644 --- a/src/create_input.py +++ b/src/create_input.py @@ -273,6 +273,18 @@ def create_input_for_acr(cfg, df, output_path, method): # add math output_df['math'] = math_output + # add math_ans and math_hash if available (for client-side hash verification) + if 'math_ans' in df.columns: + math_ans_source = df['math_ans'].dropna() + math_ans_output = np.tile(math_ans_source.to_numpy(), + (n_sessions // math_ans_source.count()) + 1)[:n_sessions] + output_df['math_ans'] = math_ans_output + if 'math_hash' in df.columns: + math_hash_source = df['math_hash'].dropna() + math_hash_output = np.tile(math_hash_source.to_numpy(), + (n_sessions // math_hash_source.count()) + 1)[:n_sessions] + output_df['math_hash'] = math_hash_output + # CMPs: 4 pairs are needed for 1 session nPairs = 4 * n_sessions pair_a = df['pair_a'].dropna() @@ -458,6 +470,18 @@ def create_input_for_dcrccr(cfg, df, output_path): math_source = df['math'].dropna() math_output = np.tile(math_source.to_numpy(), (n_sessions // math_source.count()) + 1)[:n_sessions] + # add math_ans and math_hash if available (for client-side hash verification) + math_ans_output = None + math_hash_output = None + if 'math_ans' in df.columns: + math_ans_source = df['math_ans'].dropna() + math_ans_output = np.tile(math_ans_source.to_numpy(), + (n_sessions // math_ans_source.count()) + 1)[:n_sessions] + if 'math_hash' in df.columns: + math_hash_source = df['math_hash'].dropna() + math_hash_output = np.tile(math_hash_source.to_numpy(), + (n_sessions // math_hash_source.count()) + 1)[:n_sessions] + # CMPs: 4 pairs are needed for 1 session nPairs = 4 * n_sessions pair_a = df['pair_a'].dropna() @@ -482,6 +506,10 @@ def create_input_for_dcrccr(cfg, df, output_path): 'CMP4_A': new_4[:, 6], 'CMP4_B': new_4[:, 7]}) # add math output_df['math'] = math_output + if math_ans_output is not None: + output_df['math_ans'] = math_ans_output + if math_hash_output is not None: + output_df['math_hash'] = math_hash_output # rating_clips # repeat some clips to have a full design n_questions = int(cfg['number_of_clips_per_session']) diff --git a/src/master_script.py b/src/master_script.py index bb1fe08..b500326 100644 --- a/src/master_script.py +++ b/src/master_script.py @@ -803,7 +803,10 @@ def extend_general_cfg_bw(general, hitapp): async def main(cfg, test_method, args): # check assets - general_path = os.path.join(os.path.dirname(__file__), 'assets_master_script/general.csv') + if args.general_assets: + general_path = args.general_assets + else: + general_path = os.path.join(os.path.dirname(__file__), 'assets_master_script/general.csv') is_p831_fest = args.p831_fest assert os.path.exists(general_path), f"No csv file containing general infos in {general_path}" @@ -1011,6 +1014,12 @@ def check_urls_in_files_exist(csv_file_path, columns): "Default is False") parser.add_argument("--create_local_test", action='store_true', help="Generate a local preview HTML file after the project is created.") + parser.add_argument( + "--general_assets", + default=None, + help="Path to the general assets CSV (default: assets_master_script/general.csv). " + "Use assets_master_script/general_assets_internal.csv for internal assets." + ) # check input arguments args = parser.parse_args() diff --git a/src/result_parser.py b/src/result_parser.py index 9e025e4..1ab8fc0 100644 --- a/src/result_parser.py +++ b/src/result_parser.py @@ -623,17 +623,22 @@ def digitsum(x): return total -def check_math(input, output, audio_played): +def check_math(input, output, audio_played, expected_ans=None): """ - check if the math question is answered correctly - :param input: - :param output: - :param audio_played: - :return: + Check if the math question is answered correctly. + + When *expected_ans* is provided (from the ``input.math_ans`` column in the + per-project CSV), it is used directly. Otherwise falls back to the legacy + ``[math]`` config section that maps filenames to answers. + + :param input: The math clip URL assigned to the HIT (``input.math``). + :param output: The participant's typed answer (``answer.math``). + :param audio_played: Number of times the math audio was played. + :param expected_ans: Expected correct answer from the input data, or None. + :return: True if the answer is correct. """ if audio_played == 0: return False - keys = list(config["math"].keys()) try: ans = int(float(output)) except: @@ -641,12 +646,23 @@ def check_math(input, output, audio_played): # it could be a case that participant typed in the 2 or 3 numbers that they heard rather their sum. if ans > 9: ans = digitsum(ans) - try: - for key in keys: - if key in input and int(config['math'][key]) == ans: - return True - except: - return False + + # New path: use expected_ans from per-project CSV (math_ans column) + if expected_ans is not None: + try: + return int(float(expected_ans)) == ans + except: + pass + + # Legacy path: use [math] section in the config file + if config.has_section("math"): + try: + keys = list(config["math"].keys()) + for key in keys: + if key in input and int(config['math'][key]) == ans: + return True + except: + return False return False def check_qualification_answer(row): @@ -781,8 +797,17 @@ def data_cleaning(filename, method, wrong_vcodes): d['correct_math'] = None else: # step2. check math + expected_math_ans = row.get('input.math_ans', None) + if expected_math_ans is not None: + try: + # NaN check for pandas + if expected_math_ans != expected_math_ans: + expected_math_ans = None + except: + pass d['correct_math'] = 1 if check_math(row['input.math'], row['answer.math'], - row['answer.audio_n_play_math1']) else 0 + row['answer.audio_n_play_math1'], + expected_math_ans) else 0 # step3. check pair comparison for i in range(1, 5): if check_a_cmp(row[f'input.cmp{i}_a'], row[f'input.cmp{i}_b'], row[f'answer.cmp{i}'], diff --git a/src/utils/generate_math_questions.py b/src/utils/generate_math_questions.py new file mode 100644 index 0000000..c501e1a --- /dev/null +++ b/src/utils/generate_math_questions.py @@ -0,0 +1,600 @@ +""" +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +Generate math audio questions for P.808 headphone verification. + +Creates stereo WAV files where a spoken prompt ("Please add up the following +numbers") plays in both channels, followed by individual numbers panned to +either the left or right speaker. Each question uses at least one number in +each channel so participants must hear both sides to compute the correct sum. + +The script pre-renders TTS segments for the prompt and all required number +words, then assembles each question from these cached segments. A manifest +CSV is written alongside the WAV files with the correct answer for each +question. + +Requires the Azure Cognitive Services Speech SDK and Azure Identity: + + pip install azure-cognitiveservices-speech azure-identity + +See also ``trapping_clips_assets/messages/azure_tts_create_msgs.py`` for the +original Azure TTS example this script is based on. + +Usage: + python utils/generate_math_questions.py ^ + --output_dir output/math ^ + --count 10 ^ + --region eastus ^ + --resource_id +""" + +import argparse +import csv +import hashlib +import os +import random +import tempfile +import uuid +import wave + +import numpy as np + + +# --------------------------------------------------------------------------- +# Number-to-word conversion (1–99) +# --------------------------------------------------------------------------- + +_ONES = [ + "", "one", "two", "three", "four", "five", "six", "seven", "eight", "nine", + "ten", "eleven", "twelve", "thirteen", "fourteen", "fifteen", "sixteen", + "seventeen", "eighteen", "nineteen", +] +_TENS = [ + "", "", "twenty", "thirty", "forty", "fifty", + "sixty", "seventy", "eighty", "ninety", +] + +PROMPT_TEXT = "Please add up the following numbers" + + +def number_to_words(n): + """ + Convert an integer in the range 1–99 to its English word form. + + :param n: Integer between 1 and 99 inclusive. + :return: English word string (e.g., 1 → "one", 42 → "forty two"). + :raises ValueError: If *n* is outside the supported range. + """ + if not 1 <= n <= 99: + raise ValueError(f"Number {n} is outside the supported range (1-99).") + if n < 20: + return _ONES[n] + tens, ones = divmod(n, 10) + return _TENS[tens] + ("" if ones == 0 else " " + _ONES[ones]) + + +# --------------------------------------------------------------------------- +# Hash helpers +# --------------------------------------------------------------------------- + +def compute_math_hash(audio_url, answer): + """ + Compute a SHA-256 hash for client-side math answer verification. + + The hash is derived from the audio URL and the correct answer so that + the raw answer is never exposed in the HTML source. The client can + verify a user's input by computing the same hash and comparing. + + :param audio_url: Full URL of the math audio clip. + :param answer: Correct numeric answer (int or str). + :return: Hex-encoded SHA-256 digest string. + """ + payload = f"{audio_url}:{answer}" + return hashlib.sha256(payload.encode("utf-8")).hexdigest() + + +# --------------------------------------------------------------------------- +# Azure TTS helpers +# --------------------------------------------------------------------------- + +def configure_speech(region, resource_id): + """ + Create an Azure SpeechConfig authenticated via DefaultAzureCredential. + + The output format is set to 16 kHz 16-bit mono PCM to match the sample + rate of the existing math question clips shipped with the P.808 toolkit. + + :param region: Azure Speech service region (e.g. ``"eastus"``). + :param resource_id: Full Azure resource ID of the Speech resource. + :return: Configured ``speechsdk.SpeechConfig`` instance. + """ + import azure.cognitiveservices.speech as speechsdk + from azure.identity import DefaultAzureCredential + + credential = DefaultAzureCredential() + token = credential.get_token("https://cognitiveservices.azure.com/.default") + + speech_config = speechsdk.SpeechConfig(subscription="unused", region=region) + speech_config.authorization_token = "aad#" + resource_id + "#" + token.token + speech_config.set_speech_synthesis_output_format( + speechsdk.SpeechSynthesisOutputFormat.Riff16Khz16BitMonoPcm + ) + return speech_config + + +def synthesize_to_wav(speech_config, text, output_path, voice="en-US-AriaNeural"): + """ + Synthesize a text phrase to a WAV file using Azure Neural TTS. + + Uses SSML to select the voice and wraps the text in a ```` element + for natural sentence-level prosody. + + :param speech_config: Azure ``SpeechConfig`` instance. + :param text: Plain text to synthesize. + :param output_path: Destination path for the output WAV file. + :param voice: Azure TTS voice name (default: ``en-US-AriaNeural``). + :return: ``True`` on success, ``False`` on failure. + """ + import azure.cognitiveservices.speech as speechsdk + + audio_output = speechsdk.audio.AudioOutputConfig(filename=output_path) + synthesizer = speechsdk.SpeechSynthesizer( + speech_config=speech_config, audio_config=audio_output + ) + + ssml = ( + '' + f"{text}" + ) + result = synthesizer.speak_ssml_async(ssml).get() + + if result.reason == speechsdk.ResultReason.SynthesizingAudioCompleted: + return True + + cancellation = result.cancellation_details + print(f" TTS failed for '{text}': {cancellation.reason}") + if cancellation.error_details: + print(f" Error: {cancellation.error_details}") + return False + + +# --------------------------------------------------------------------------- +# ASL measurement and normalization (ITU-T P.56 Method B) +# --------------------------------------------------------------------------- + +def measure_asl(samples, sample_rate): + """ + Measure Active Speech Level per ITU-T P.56 Method B. + + Uses an iterative threshold refinement to distinguish active speech + frames from silence and returns the RMS level of active regions in + dBov (decibels relative to digital full-scale). + + :param samples: Audio samples as a float ndarray, values in [-1, 1]. + :param sample_rate: Sample rate in Hz. + :return: Active speech level in dBov, or ``-inf`` for silent input. + """ + x = samples.astype(np.float64) + sq = x ** 2 + + long_term_sq = np.mean(sq) + if long_term_sq < 1e-20: + return -np.inf + + # 30 ms frames for activity detection + frame_len = max(1, int(0.03 * sample_rate)) + n_frames = len(x) // frame_len + if n_frames == 0: + return 10 * np.log10(long_term_sq) + + frame_energies = np.mean( + sq[:n_frames * frame_len].reshape(n_frames, frame_len), axis=1 + ) + + # Iterative refinement: threshold at active_level - 15.9 dB + active_level_sq = long_term_sq + for _ in range(20): + threshold = active_level_sq * 10 ** (-15.9 / 10) + active_mask = frame_energies > threshold + + if not np.any(active_mask): + break + + new_active_level_sq = np.mean(frame_energies[active_mask]) + + if abs(10 * np.log10(new_active_level_sq / (active_level_sq + 1e-30))) < 0.05: + active_level_sq = new_active_level_sq + break + + active_level_sq = new_active_level_sq + + return 10 * np.log10(active_level_sq + 1e-30) + + +def normalize_segments_to_asl(segments, sample_rate, target_dbov=-26.0): + """ + Scale all pre-rendered TTS segments so their combined ASL matches the target. + + The prompt and every number word are concatenated into a single mono + signal for measurement. The resulting gain is applied uniformly to + every segment so relative levels are preserved. + + :param segments: Dict mapping ``"prompt"`` and integers to mono float32 + arrays (as returned by :func:`prerender_tts_segments`). + :param sample_rate: Sample rate in Hz. + :param target_dbov: Desired active speech level in dBov (default: -26). + :return: New segments dict with scaled arrays. + """ + all_audio = [segments["prompt"]] + for key in sorted(k for k in segments if isinstance(k, int)): + all_audio.append(segments[key]) + combined = np.concatenate(all_audio) + + current_asl = measure_asl(combined, sample_rate) + if np.isinf(current_asl): + print(f" Warning: could not measure ASL (silent input), skipping normalization.") + return segments + + gain_db = target_dbov - current_asl + gain = 10 ** (gain_db / 20) + + print(f" Current ASL: {current_asl:.1f} dBov -> target: {target_dbov:.1f} dBov " + f"(gain: {gain_db:+.1f} dB)") + + return {k: (v * gain).astype(np.float32) for k, v in segments.items()} + + +# --------------------------------------------------------------------------- +# Audio helpers +# --------------------------------------------------------------------------- + +def load_wav_mono(path): + """ + Load a WAV file and return its samples as a mono float32 array. + + Multi-channel files are down-mixed by averaging all channels. Supports + 16-bit and 32-bit integer PCM formats. + + :param path: Path to the WAV file. + :return: Tuple of (*samples* as float32 ndarray, *sample_rate* as int). + """ + with wave.open(path, "r") as w: + sample_rate = w.getframerate() + n_channels = w.getnchannels() + sampwidth = w.getsampwidth() + raw = w.readframes(w.getnframes()) + + if sampwidth == 2: + samples = np.frombuffer(raw, dtype=np.int16).astype(np.float32) / 32768.0 + elif sampwidth == 4: + samples = np.frombuffer(raw, dtype=np.int32).astype(np.float32) / 2147483648.0 + else: + raise ValueError(f"Unsupported sample width: {sampwidth}") + + if n_channels > 1: + samples = samples.reshape(-1, n_channels).mean(axis=1) + + return samples, sample_rate + + +def pan_to_stereo(mono, channel): + """ + Pan a mono audio signal to the left, right, or both stereo channels. + + :param mono: Mono audio samples as a float32 ndarray. + :param channel: Target channel — ``"left"``, ``"right"``, or ``"both"``. + :return: Stereo array with shape ``(N, 2)``. + """ + stereo = np.zeros((len(mono), 2), dtype=np.float32) + if channel == "left": + stereo[:, 0] = mono + elif channel == "right": + stereo[:, 1] = mono + else: + stereo[:, 0] = mono + stereo[:, 1] = mono + return stereo + + +def save_stereo_wav(path, stereo, sample_rate=16000): + """ + Save a stereo float32 array as a 16-bit PCM WAV file. + + Samples are clipped to [-1, 1] before conversion to 16-bit integers. + + :param path: Output file path. + :param stereo: Stereo audio array with shape ``(N, 2)``, values in [-1, 1]. + :param sample_rate: Sample rate in Hz (default: 16000). + """ + stereo = np.clip(stereo, -1.0, 1.0) + int_data = (stereo * 32767).astype(np.int16) + with wave.open(path, "w") as w: + w.setnchannels(2) + w.setsampwidth(2) + w.setframerate(sample_rate) + w.writeframes(int_data.tobytes()) + + +# --------------------------------------------------------------------------- +# Generation pipeline +# --------------------------------------------------------------------------- + +def prerender_tts_segments(speech_config, tmp_dir, min_number, max_number, + voice="en-US-AriaNeural"): + """ + Pre-render TTS audio for the prompt phrase and all required number words. + + Each segment is synthesized once and cached as a mono float32 array so + that multiple questions can be assembled without repeated TTS calls. + + :param speech_config: Azure ``SpeechConfig`` instance. + :param tmp_dir: Temporary directory for intermediate WAV files. + :param min_number: Smallest number to render. + :param max_number: Largest number to render. + :param voice: Azure TTS voice name. + :return: Dict mapping ``"prompt"`` and integers to mono float32 arrays. + :raises RuntimeError: If any TTS call fails. + """ + segments = {} + + prompt_path = os.path.join(tmp_dir, "prompt.wav") + print(" Rendering TTS: prompt") + if not synthesize_to_wav(speech_config, PROMPT_TEXT, prompt_path, voice): + raise RuntimeError("Failed to render TTS for the prompt.") + segments["prompt"], _ = load_wav_mono(prompt_path) + + for n in range(min_number, max_number + 1): + word = number_to_words(n) + num_path = os.path.join(tmp_dir, f"{n}.wav") + print(f" Rendering TTS: {word}") + if not synthesize_to_wav(speech_config, word, num_path, voice): + raise RuntimeError(f"Failed to render TTS for '{word}'.") + segments[n], _ = load_wav_mono(num_path) + + return segments + + +def build_stereo_clip(prompt, number_audios, channels, sample_rate, gap_sec=0.5): + """ + Assemble a stereo math question clip from pre-rendered segments. + + The prompt plays in both channels, followed by each number panned to its + assigned channel. A silence gap separates consecutive segments. + + :param prompt: Mono float32 array for the prompt phrase. + :param number_audios: List of mono float32 arrays, one per number. + :param channels: List of channel assignments (``"left"`` or ``"right"``). + :param sample_rate: Sample rate in Hz. + :param gap_sec: Silence gap between segments in seconds (default: 0.5). + :return: Stereo array with shape ``(N, 2)``. + """ + gap = np.zeros(int(gap_sec * sample_rate), dtype=np.float32) + + parts = [pan_to_stereo(prompt, "both"), pan_to_stereo(gap, "both")] + for audio, ch in zip(number_audios, channels): + parts.append(pan_to_stereo(audio, ch)) + parts.append(pan_to_stereo(gap, "both")) + + return np.concatenate(parts, axis=0) + + +def generate_math_questions(output_dir, count, speech_config, seed=42, + sample_rate=16000, numbers_per_question=3, + min_number=1, max_number=9, + voice="en-US-AriaNeural", start_index=1, + target_asl=-26.0, base_url=None): + """ + Generate N math question audio files with a manifest CSV. + + Each question is a stereo WAV containing a spoken prompt followed by + randomly chosen numbers panned to left or right channels. At least one + number is placed in each channel so the listener must hear both sides to + compute the correct answer. + + All TTS segments are normalized to the target active speech level (ASL) + per ITU-T P.56 Method B *before* stereo panning, so the per-ear level + matches the target regardless of channel assignment. + + Each clip is saved with a random UUID filename. When *base_url* is + provided, a ``general_assets_internal.csv`` is also written with + ``math``, ``math_ans``, and ``math_hash`` columns ready for use with + the P.808 master script. + + A ``math_questions.csv`` manifest is always written alongside the WAV + files with columns ``filename``, ``numbers``, ``channels``, ``answer``, + and ``math_hash``. + + :param output_dir: Directory for output WAV files and manifest CSV. + :param count: Number of questions to generate. + :param speech_config: Azure ``SpeechConfig`` instance. + :param seed: Random seed for reproducibility (default: 42). + :param sample_rate: Output sample rate in Hz (default: 16000). + :param numbers_per_question: How many numbers per question (default: 3). + :param min_number: Minimum number value (default: 1). + :param max_number: Maximum number value (default: 9). + :param voice: Azure TTS voice name (default: ``en-US-AriaNeural``). + :param start_index: Starting index for output filenames (default: 1). + :param target_asl: Target active speech level in dBov (default: -26). + :param base_url: Base URL where clips will be uploaded (e.g. + ``"https://host/container/clips/internal_assets/"``). When set, + a ``general_assets_internal.csv`` is generated with full URLs. + :return: List of dicts with ``filename``, ``numbers``, ``channels``, + ``answer``, and ``math_hash`` keys. + """ + random.seed(seed) + os.makedirs(output_dir, exist_ok=True) + + with tempfile.TemporaryDirectory() as tmp_dir: + print("Pre-rendering TTS segments...") + segments = prerender_tts_segments(speech_config, tmp_dir, min_number, + max_number, voice) + + print("Normalizing ASL...") + segments = normalize_segments_to_asl(segments, sample_rate, target_asl) + + print(f"\nGenerating {count} math question(s)...") + manifest = [] + general_rows = [] + + for i in range(count): + numbers = [random.randint(min_number, max_number) + for _ in range(numbers_per_question)] + + # Assign random channels, ensuring both L and R are used + channels = [random.choice(["left", "right"]) + for _ in range(numbers_per_question)] + if all(c == "left" for c in channels): + channels[random.randint(0, len(channels) - 1)] = "right" + elif all(c == "right" for c in channels): + channels[random.randint(0, len(channels) - 1)] = "left" + + answer = sum(numbers) + + # Random UUID filename + filename = f"{uuid.uuid4().hex}.wav" + filepath = os.path.join(output_dir, filename) + + number_audios = [segments[n] for n in numbers] + stereo = build_stereo_clip(segments["prompt"], number_audios, channels, + sample_rate) + save_stereo_wav(filepath, stereo, sample_rate) + + # Compute hash only when base_url is known + math_hash = "" + if base_url: + url = base_url.rstrip("/") + "/" + filename + math_hash = compute_math_hash(url, answer) + + label = " + ".join( + f"{n}({c[0].upper()})" for n, c in zip(numbers, channels) + ) + print(f" [{i + 1}/{count}] {filename}: {label} = {answer}") + + entry = { + "filename": filename, + "numbers": "+".join(str(n) for n in numbers), + "channels": ",".join(channels), + "answer": answer, + } + if base_url: + entry["math_hash"] = math_hash + manifest.append(entry) + + if base_url: + general_rows.append({ + "math": url, "math_ans": answer, "math_hash": math_hash, + }) + + # Write detailed manifest + fieldnames = ["filename", "numbers", "channels", "answer"] + if base_url: + fieldnames.append("math_hash") + manifest_path = os.path.join(output_dir, "math_questions.csv") + with open(manifest_path, "w", newline="") as f: + writer = csv.DictWriter(f, fieldnames=fieldnames) + writer.writeheader() + writer.writerows(manifest) + print(f"\nManifest saved to: {manifest_path}") + + # Write general_assets_internal.csv + if base_url: + general_path = os.path.join(output_dir, "general_assets_internal.csv") + with open(general_path, "w", newline="") as f: + writer = csv.DictWriter( + f, fieldnames=["math", "math_ans", "math_hash"] + ) + writer.writeheader() + writer.writerows(general_rows) + print(f"General assets CSV saved to: {general_path}") + + print(f"Generated {count} question(s) ({count} WAV files) in {output_dir}") + return manifest + + +# --------------------------------------------------------------------------- +# CLI entry point +# --------------------------------------------------------------------------- + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Generate math audio questions for P.808 headphone " + "verification. Creates stereo WAV files with numbers " + "panned to left/right speakers." + ) + parser.add_argument( + "--output_dir", "-o", required=True, + help="Directory for output WAV files and manifest CSV." + ) + parser.add_argument( + "--count", "-n", type=int, required=True, + help="Number of math questions to generate." + ) + parser.add_argument( + "--region", required=True, + help="Azure Speech service region (e.g. eastus)." + ) + parser.add_argument( + "--resource_id", required=True, + help="Azure Speech resource ID for AAD-based authentication. " + "Find it in Azure Portal > Speech resource > Properties." + ) + parser.add_argument( + "--voice", default="en-US-AriaNeural", + help="Azure TTS voice name (default: en-US-AriaNeural)." + ) + parser.add_argument( + "--seed", type=int, default=42, + help="Random seed for reproducibility (default: 42)." + ) + parser.add_argument( + "--numbers_per_question", type=int, default=3, + help="How many numbers per question (default: 3)." + ) + parser.add_argument( + "--min_number", type=int, default=1, + help="Minimum number value (default: 1)." + ) + parser.add_argument( + "--max_number", type=int, default=9, + help="Maximum number value (default: 9)." + ) + parser.add_argument( + "--target_asl", type=float, default=-26.0, + help="Target active speech level in dBov per ITU-T P.56 (default: -26)." + ) + parser.add_argument( + "--base_url", + default=None, + help="Base URL where clips will be uploaded. When provided, " + "general_assets_internal.csv and math_hash values are generated. " + "When omitted, math_hash is not computed." + ) + + args = parser.parse_args() + + assert args.count > 0, "Count must be positive" + assert 1 <= args.min_number <= args.max_number <= 99, ( + "Number range must satisfy 1 <= min_number <= max_number <= 99" + ) + assert args.numbers_per_question >= 2, ( + "At least 2 numbers per question are required to use both channels" + ) + + config = configure_speech(args.region, args.resource_id) + + generate_math_questions( + output_dir=args.output_dir, + count=args.count, + speech_config=config, + seed=args.seed, + numbers_per_question=args.numbers_per_question, + min_number=args.min_number, + max_number=args.max_number, + voice=args.voice, + target_asl=args.target_asl, + base_url=args.base_url, + ) From 9d28ef48e869f56dee906478e383ee8ba21f9715 Mon Sep 17 00:00:00 2001 From: Babak Naderi Date: Mon, 29 Jun 2026 20:55:14 +0200 Subject: [PATCH 002/111] Normalize gold source level and rework loudness degradation Normalize every gold source clip to -26 dBov active speech before applying degradations. Replace the relative +/-25 dB loudness change with absolute active-speech targets: about -10 dBov for the too-loud case and -45 dBov for the too-quiet case. Adds active_speech_level_dbov() and normalize_active_speech_level() helpers. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/create_gold_clips.py | 96 ++++++++++++++++++++++++++++++++++------ 1 file changed, 83 insertions(+), 13 deletions(-) diff --git a/src/create_gold_clips.py b/src/create_gold_clips.py index ed6d5fe..a895579 100644 --- a/src/create_gold_clips.py +++ b/src/create_gold_clips.py @@ -33,6 +33,16 @@ 'noise_ans': 5, 'reverb_ans': 5, 'sig_ans': 5, 'ovrl_ans': 5, } +# Target active-speech levels (dBov) for the loudness degradation. The "too loud" +# case is scaled so the active speech sits at about -10 dBov, the "too quiet" +# case at about -45 dBov. +LOUDNESS_TOO_LOUD_DBOV = -10.0 +LOUDNESS_TOO_QUIET_DBOV = -45.0 + +# Every gold source clip is normalized to this active speech level (dBov) before +# any degradation (or none, for the clean/score-5 case) is applied. +GOLD_SOURCE_TARGET_DBOV = -26.0 + GOLD_TYPES = { 'clean': { 'suffix': 'clean', @@ -229,23 +239,78 @@ def apply_coloration(signal, sr): return colored -def apply_loudness(signal, gain_db=None): +def active_speech_level_dbov(signal, sr, frame_ms=20.0, threshold_db=25.0): + """ + Estimate the active speech level of a signal in dBov. + + The level is measured over the active speech part only: the signal is split + into short frames and frames whose RMS is within ``threshold_db`` of the + loudest frame are treated as active (speech) frames. ``0 dBov`` corresponds + to a full-scale sine wave (RMS = 1/sqrt(2)). + + :param signal: Audio signal as a numpy array in the range [-1, 1]. + :param sr: Sample rate in Hz. + :param frame_ms: Frame length in milliseconds. + :param threshold_db: Frames within this many dB below the loudest frame count as active. + :return: Active speech level in dBov. + """ + eps = 1e-12 + full_scale_sine_rms = 1.0 / np.sqrt(2.0) + frame_len = max(1, int(sr * frame_ms / 1000.0)) + n_frames = len(signal) // frame_len + if n_frames == 0: + active_rms = np.sqrt(np.mean(signal ** 2) + eps) + return 20.0 * np.log10(active_rms / full_scale_sine_rms + eps) + frames = signal[:n_frames * frame_len].reshape(n_frames, frame_len) + frame_rms = np.sqrt(np.mean(frames ** 2, axis=1) + eps) + peak_rms = frame_rms.max() + threshold = peak_rms / (10.0 ** (threshold_db / 20.0)) + active = frame_rms >= threshold + if not np.any(active): + active = np.ones(n_frames, dtype=bool) + active_rms = np.sqrt(np.mean(frames[active] ** 2) + eps) + return 20.0 * np.log10(active_rms / full_scale_sine_rms + eps) + + +def normalize_active_speech_level(signal, sr, target_dbov): """ - Apply extreme loudness change by randomly making the signal too loud or too quiet. + Scale a signal so its active speech level matches a target level in dBov. :param signal: Audio signal as a numpy array. - :param gain_db: Gain in dB. If None, randomly picks +25 or -25 dB. + :param sr: Sample rate in Hz. + :param target_dbov: Target active speech level in dBov. + :return: Level-normalized signal, hard-clipped to [-1, 1]. + """ + current_dbov = active_speech_level_dbov(signal, sr) + gain_db = target_dbov - current_dbov + factor = 10.0 ** (gain_db / 20.0) + return np.clip(signal * factor, -1.0, 1.0) + + +def apply_loudness(signal, sr, target_dbov=None): + """ + Apply an extreme loudness change by scaling the active speech level to a target. + + The clip is randomly made too loud or too quiet by scaling so that the active + speech part reaches ``LOUDNESS_TOO_LOUD_DBOV`` or ``LOUDNESS_TOO_QUIET_DBOV``. + For the too-loud case peaks may exceed full scale, so the result is hard-clipped + to [-1, 1] (this preserves the loud level instead of rescaling it back down). + + :param signal: Audio signal as a numpy array. + :param sr: Sample rate in Hz. + :param target_dbov: Target active speech level in dBov. If None, randomly picks + the too-loud or too-quiet target. :return: Loudness-adjusted signal as a numpy array. """ - if gain_db is None: - gain_db = np.random.choice([25, -25]) + if target_dbov is None: + target_dbov = np.random.choice([LOUDNESS_TOO_LOUD_DBOV, LOUDNESS_TOO_QUIET_DBOV]) + current_dbov = active_speech_level_dbov(signal, sr) + gain_db = target_dbov - current_dbov factor = 10.0 ** (gain_db / 20.0) adjusted = signal * factor - # Prevent hard clipping for loud signals - peak = np.max(np.abs(adjusted)) - if peak > 1.0: - adjusted = adjusted / peak * 0.99 - return adjusted + # Preserve the target level (especially for the too-loud case) by hard-clipping + # rather than rescaling the peak back down. + return np.clip(adjusted, -1.0, 1.0) def _apply_random_post_processing(signal, sr, gold_type): @@ -356,11 +421,11 @@ def process_clip(signal, sr, gold_type, snr_db=-5.0, clip_threshold=0.005): result = apply_signal_distortion(signal, clip_threshold) result = add_background_noise(result, sr, snr_db) elif gold_type == 'loudness': - result = apply_loudness(signal) + result = apply_loudness(signal, sr) elif gold_type == 'loudness_distortion': - result = apply_loudness(apply_signal_distortion(signal, clip_threshold)) + result = apply_loudness(apply_signal_distortion(signal, clip_threshold), sr) elif gold_type == 'loudness_noise': - result = apply_loudness(add_background_noise(signal, sr, snr_db)) + result = apply_loudness(add_background_noise(signal, sr, snr_db), sr) else: raise ValueError(f"Unknown gold type: {gold_type}") @@ -403,6 +468,8 @@ def create_gold_clips(input_dir, output_dir, method, snr_db=-5.0, clip_threshold Generate gold clips from clean source audio files. For each source file, degraded versions are created based on the method. + Each source clip is first normalized to a fixed active speech level + (``GOLD_SOURCE_TARGET_DBOV``) before any degradation is applied. A CSV report mapping filenames to expected answers is written to output_dir. For P804, only dimensions with answer 1 are written; others are left empty. @@ -441,6 +508,9 @@ def create_gold_clips(input_dir, output_dir, method, snr_db=-5.0, clip_threshold for src_path in source_files: src_name = splitext(basename(src_path))[0] signal, sr = lr.load(src_path, sr=None) + # Normalize every source clip to a fixed active speech level before degrading, + # so degradations (or none, for the clean/score-5 case) start from -26 dBov. + signal = normalize_active_speech_level(signal, sr, GOLD_SOURCE_TARGET_DBOV) print(f" Processing: {basename(src_path)} ({len(signal)} samples, {sr} Hz)") for gold_type, type_info in applicable_types.items(): From 47d7b23834ecbd067812f7d4415eec31951f2942 Mon Sep 17 00:00:00 2001 From: Babak Naderi Date: Mon, 29 Jun 2026 20:55:36 +0200 Subject: [PATCH 003/111] Add Silero VAD silence pre-screen for rating clips New src/utils/detect_silence_vad.py computes per-clip active speech with Silero VAD and flags clips with little or no speech. Supports a 'prescreen' mode (report silent clips in a clip-list CSV) and a 'crosscheck' mode (compare VAD against the crowd is_silent_percentage). Wire an optional --check_silence flag into master_script.py, mirroring --check_urls, and add torch and silero-vad to requirements. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/master_script.py | 18 ++ src/requirements.txt | 2 + src/utils/detect_silence_vad.py | 331 ++++++++++++++++++++++++++++++++ 3 files changed, 351 insertions(+) create mode 100644 src/utils/detect_silence_vad.py diff --git a/src/master_script.py b/src/master_script.py index b500326..9fb6ce0 100644 --- a/src/master_script.py +++ b/src/master_script.py @@ -1012,6 +1012,9 @@ def check_urls_in_files_exist(csv_file_path, columns): # check links default to False parser.add_argument("--check_urls", action='store_true', help="Check if all links in the csv files are valid. " "Default is False") + parser.add_argument("--check_silence", action='store_true', + help="Run a Voice Activity Detector (Silero VAD) over the rating clips and report clips " + "with little or no speech. Optional, like --check_urls. Default is False.") parser.add_argument("--create_local_test", action='store_true', help="Generate a local preview HTML file after the project is created.") parser.add_argument( @@ -1088,6 +1091,21 @@ def check_urls_in_files_exist(csv_file_path, columns): check_urls_in_files_exist(args.trapping_clips, expected_columns_double_stimuli['trapping']) else: raise SystemExit(f"Error: No such a method supported for checking links: {test_method}") + + if args.check_silence: + # optional VAD pre-screen of rating clips for silent / no-speech content. + # Rating clips passed to the master script must always be publicly accessible. + print("Running VAD silence pre-screen over the rating clips ...") + if not args.clips: + raise SystemExit("Error: --check_silence requires a rating clips csv (--clips).") + from utils.detect_silence_vad import prescreen_clips + rating_column = ( + expected_columns_double_stimuli["clips"][0] + if test_method in ["dcr", "ccr"] + else expected_columns_single_stimuli["clips"][0] + ) + prescreen_clips(args.clips, column=rating_column) + asyncio.run(main(cfg, test_method, args)) if args.create_local_test: diff --git a/src/requirements.txt b/src/requirements.txt index 94ead51..2a7a630 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -11,3 +11,5 @@ scaleapi requests matplotlib statsmodels +torch +silero-vad diff --git a/src/utils/detect_silence_vad.py b/src/utils/detect_silence_vad.py new file mode 100644 index 0000000..e12bdbe --- /dev/null +++ b/src/utils/detect_silence_vad.py @@ -0,0 +1,331 @@ +""" +Detect silent / no-speech clips with a Voice Activity Detector (Silero VAD). + +This utility computes the amount of voiced speech in each clip and flags clips +that contain little or no speech. It supports two workflows: + +* ``prescreen`` - run VAD over a clip list (e.g. ``rating_clips.csv``) before + publishing a study and write a report flagging silent clips. The same logic is + reused by ``master_script.py`` via the optional ``--check_silence`` flag. +* ``crosscheck`` - compare the VAD result against the crowd ``is_silent_percentage`` + column produced by ``result_parser.py`` to validate silent votes and spot + broken clips or rater abuse. + +Audio is read with ``soundfile``/``librosa`` (already required by the toolkit), so +``torchaudio`` is not needed. ``torch`` and the ``silero-vad`` package are imported +lazily and are only required when VAD is actually run. +""" + +import argparse +import os +import sys +import tempfile +import urllib.request +from concurrent.futures import ThreadPoolExecutor, as_completed + +import librosa as lr +import numpy as np +import pandas as pd +import soundfile as sf + +VAD_SAMPLE_RATE = 16000 +DEFAULT_MIN_SPEECH_SEC = 0.30 +DEFAULT_MIN_SPEECH_RATIO = 0.02 +DEFAULT_CROWD_SILENT_THRESHOLD = 50.0 + +_VAD_MODEL = None +_GET_SPEECH_TS = None + + +def load_vad_model(): + """ + Load the Silero VAD model and the speech-timestamp helper (lazily, once). + + :return: Tuple of (model, get_speech_timestamps callable). + """ + global _VAD_MODEL, _GET_SPEECH_TS + if _VAD_MODEL is None: + try: + from silero_vad import load_silero_vad, get_speech_timestamps + except ImportError as err: + raise ImportError( + "Silero VAD is required for silence detection. Install it with " + "'pip install silero-vad torch' (see src/requirements.txt)." + ) from err + _VAD_MODEL = load_silero_vad() + _GET_SPEECH_TS = get_speech_timestamps + return _VAD_MODEL, _GET_SPEECH_TS + + +def load_audio_mono_16k(path): + """ + Read an audio file as a mono waveform resampled to 16 kHz. + + :param path: Path to a local audio file. + :return: 1-D float32 numpy array at 16 kHz. + """ + audio, sr = sf.read(path, dtype="float32") + if audio.ndim > 1: + audio = audio.mean(axis=1) + if sr != VAD_SAMPLE_RATE: + audio = lr.resample(audio.astype("float32"), orig_sr=sr, target_sr=VAD_SAMPLE_RATE) + return audio.astype("float32") + + +def speech_stats_for_audio(audio): + """ + Compute speech statistics for a 16 kHz mono waveform using Silero VAD. + + :param audio: 1-D float32 numpy array sampled at 16 kHz. + :return: Dict with total_sec, speech_sec, and speech_ratio. + """ + import torch + + model, get_speech_timestamps = load_vad_model() + total_sec = len(audio) / VAD_SAMPLE_RATE + if total_sec == 0: + return {"total_sec": 0.0, "speech_sec": 0.0, "speech_ratio": 0.0} + wav = torch.from_numpy(audio) + segments = get_speech_timestamps( + wav, model, sampling_rate=VAD_SAMPLE_RATE, return_seconds=True + ) + speech_sec = float(sum(seg["end"] - seg["start"] for seg in segments)) + return { + "total_sec": round(total_sec, 3), + "speech_sec": round(speech_sec, 3), + "speech_ratio": round(speech_sec / total_sec, 4) if total_sec else 0.0, + } + + +def _download_to_temp(url, sas_token=None): + """ + Download a remote clip to a temporary file. + + :param url: HTTP(S) URL of the clip. + :param sas_token: Optional Azure SAS token (without leading '?') for private storage. + :return: Path to the downloaded temporary file. + """ + full_url = url + if sas_token: + sep = "&" if "?" in url else "?" + full_url = f"{url}{sep}{sas_token.lstrip('?')}" + suffix = os.path.splitext(url.split("?")[0])[1] or ".wav" + fd, tmp_path = tempfile.mkstemp(suffix=suffix) + os.close(fd) + urllib.request.urlretrieve(full_url, tmp_path) + return tmp_path + + +def _prepare_local(url, sas_token=None): + """ + Resolve a clip URL to a local path, downloading it if it is remote. + + :param url: Clip URL or local file path. + :param sas_token: Optional Azure SAS token for private storage. + :return: Tuple of (local_path or None, is_remote, error_string). + """ + is_remote = url.lower().startswith("http") + try: + path = _download_to_temp(url, sas_token) if is_remote else url + return path, is_remote, "" + except Exception as err: # noqa: BLE001 + return None, is_remote, str(err) + + +def analyze_clip(url, sas_token=None, min_speech_sec=DEFAULT_MIN_SPEECH_SEC, + min_speech_ratio=DEFAULT_MIN_SPEECH_RATIO): + """ + Download (if remote) and analyze a single clip for speech presence. + + Note: VAD inference is not thread-safe, so callers must invoke this + sequentially (see ``prescreen_clips`` for the parallel-download pattern). + + :param url: Clip URL or local file path. + :param sas_token: Optional Azure SAS token for private storage. + :param min_speech_sec: Minimum voiced seconds to count the clip as non-silent. + :param min_speech_ratio: Minimum voiced ratio to count the clip as non-silent. + :return: Dict with the clip URL, speech stats, is_silent flag, and any error. + """ + path, is_remote, error = _prepare_local(url, sas_token) + result = {"file_url": url, "total_sec": None, "speech_sec": None, + "speech_ratio": None, "vad_is_silent": None, "error": error} + if error: + return result + try: + stats = speech_stats_for_audio(load_audio_mono_16k(path)) + result.update(stats) + result["vad_is_silent"] = bool( + stats["speech_sec"] < min_speech_sec or stats["speech_ratio"] < min_speech_ratio + ) + except Exception as err: # noqa: BLE001 + result["error"] = str(err) + finally: + if is_remote and path and os.path.exists(path): + os.remove(path) + return result + + +def prescreen_clips(csv_path, column="rating_clips", sas_token=None, + min_speech_sec=DEFAULT_MIN_SPEECH_SEC, + min_speech_ratio=DEFAULT_MIN_SPEECH_RATIO, + download_workers=8, report_path=None): + """ + Run VAD over every clip in a CSV column and report silent clips. + + Downloads run in parallel, but VAD inference runs sequentially in the calling + thread because the Silero model is not thread-safe. + + :param csv_path: Path to a CSV containing clip URLs. + :param column: Name of the column holding the clip URLs. + :param sas_token: Optional Azure SAS token for private storage. + :param min_speech_sec: Minimum voiced seconds to count the clip as non-silent. + :param min_speech_ratio: Minimum voiced ratio to count the clip as non-silent. + :param download_workers: Number of parallel download threads. + :param report_path: Where to write the report CSV (defaults next to the input). + :return: Tuple of (report DataFrame, path to the written report CSV). + """ + df = pd.read_csv(csv_path) + if column not in df.columns: + raise ValueError(f"Column '{column}' not found in {csv_path}") + urls = [u for u in df[column].tolist() if isinstance(u, str) and u.strip()] + urls = list(dict.fromkeys(urls)) + print(f" Checking speech presence (VAD) in {len(urls)} clips from {csv_path}") + + # warm up the model once before processing + load_vad_model() + + rows = [] + done = 0 + with ThreadPoolExecutor(max_workers=download_workers) as ex: + future_to_url = {ex.submit(_prepare_local, url, sas_token): url for url in urls} + for fut in as_completed(future_to_url): + url = future_to_url[fut] + path, is_remote, error = fut.result() + row = {"file_url": url, "total_sec": None, "speech_sec": None, + "speech_ratio": None, "vad_is_silent": None, "error": error} + if not error: + try: + stats = speech_stats_for_audio(load_audio_mono_16k(path)) + row.update(stats) + row["vad_is_silent"] = bool( + stats["speech_sec"] < min_speech_sec + or stats["speech_ratio"] < min_speech_ratio + ) + except Exception as err: # noqa: BLE001 + row["error"] = str(err) + finally: + if is_remote and path and os.path.exists(path): + os.remove(path) + rows.append(row) + done += 1 + if done % 100 == 0: + print(f" Analyzed: {done}/{len(urls)} clips") + + report = pd.DataFrame(rows) + n_silent = int(report["vad_is_silent"].fillna(False).sum()) + n_error = int((report["error"].astype(str).str.len() > 0).sum()) + if n_silent > 0: + print("\033[91m" + f" VAD flagged {n_silent}/{len(urls)} clips as silent " + f"(< {min_speech_sec}s speech). {n_error} clip(s) could not be read." + "\033[0m") + else: + print(f" VAD found speech in all {len(urls)} clips. {n_error} clip(s) could not be read.") + + if report_path is None: + report_path = os.path.splitext(csv_path)[0] + "_vad_silence_report.csv" + report.to_csv(report_path, index=False) + print(f" VAD silence report saved to: {report_path}") + return report, report_path + + +def crosscheck(vad_report_path, votes_per_clip_path, crowd_threshold=DEFAULT_CROWD_SILENT_THRESHOLD, + output_path=None): + """ + Compare VAD silence flags against the crowd is_silent_percentage column. + + :param vad_report_path: CSV produced by ``prescreen`` (has file_url, vad_is_silent). + :param votes_per_clip_path: A result_parser ``*_votes_per_clip*`` CSV with is_silent_percentage. + :param crowd_threshold: Percentage above which crowd votes mark a clip as silent. + :param output_path: Where to write the comparison CSV (defaults next to the VAD report). + :return: Path to the written comparison CSV. + """ + vad = pd.read_csv(vad_report_path) + votes = pd.read_csv(votes_per_clip_path) + if "is_silent_percentage" not in votes.columns: + raise ValueError( + f"'is_silent_percentage' not found in {votes_per_clip_path}. " + "Re-run result_parser.py with the updated version." + ) + merged = pd.merge( + vad[["file_url", "speech_sec", "speech_ratio", "vad_is_silent"]], + votes[["file_url", "is_silent_percentage"]], + on="file_url", how="inner", + ) + merged["crowd_is_silent"] = merged["is_silent_percentage"] >= crowd_threshold + merged["agreement"] = merged["vad_is_silent"] == merged["crowd_is_silent"] + merged["disagreement_type"] = "" + merged.loc[merged["vad_is_silent"] & ~merged["crowd_is_silent"], "disagreement_type"] = \ + "vad_silent_crowd_rated" + merged.loc[~merged["vad_is_silent"] & merged["crowd_is_silent"], "disagreement_type"] = \ + "crowd_silent_vad_speech" + + n = len(merged) + n_disagree = int((~merged["agreement"]).sum()) + print(f" Cross-checked {n} clips; {n_disagree} disagreement(s) between VAD and crowd.") + if output_path is None: + output_path = os.path.splitext(vad_report_path)[0] + "_vs_crowd.csv" + merged.to_csv(output_path, index=False) + print(f" Cross-check report saved to: {output_path}") + return output_path + + +def _build_arg_parser(): + """ + Build the command-line argument parser. + + :return: Configured argparse.ArgumentParser. + """ + parser = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + sub = parser.add_subparsers(dest="command", required=True) + + pre = sub.add_parser("prescreen", help="Flag silent clips in a clip-list CSV using VAD.") + pre.add_argument("--input", "-i", required=True, help="CSV file containing clip URLs.") + pre.add_argument("--column", "-c", default="rating_clips", help="Column with clip URLs.") + pre.add_argument("--sas_token", default=None, help="Azure SAS token for private storage.") + pre.add_argument("--min_speech_sec", type=float, default=DEFAULT_MIN_SPEECH_SEC, + help="Minimum voiced seconds to consider a clip non-silent.") + pre.add_argument("--min_speech_ratio", type=float, default=DEFAULT_MIN_SPEECH_RATIO, + help="Minimum voiced ratio to consider a clip non-silent.") + pre.add_argument("--workers", type=int, default=8, help="Parallel download workers.") + pre.add_argument("--report", default=None, help="Path to the output report CSV.") + + cc = sub.add_parser("crosscheck", help="Compare VAD report against crowd is_silent_percentage.") + cc.add_argument("--vad_report", required=True, help="VAD report CSV from 'prescreen'.") + cc.add_argument("--votes_per_clip", required=True, + help="result_parser *_votes_per_clip*.csv with is_silent_percentage.") + cc.add_argument("--crowd_threshold", type=float, default=DEFAULT_CROWD_SILENT_THRESHOLD, + help="Percentage above which crowd votes mark a clip as silent.") + cc.add_argument("--output", default=None, help="Path to the comparison CSV.") + return parser + + +def main(): + """ + Command-line entry point for VAD-based silence detection. + + :return: None. + """ + args = _build_arg_parser().parse_args() + if args.command == "prescreen": + prescreen_clips( + args.input, column=args.column, sas_token=args.sas_token, + min_speech_sec=args.min_speech_sec, min_speech_ratio=args.min_speech_ratio, + download_workers=args.workers, report_path=args.report, + ) + elif args.command == "crosscheck": + crosscheck(args.vad_report, args.votes_per_clip, + crowd_threshold=args.crowd_threshold, output_path=args.output) + + +if __name__ == "__main__": + sys.exit(main()) From b0c6eab16b6e3992c27cd877d8b1f5843504c7ab Mon Sep 17 00:00:00 2001 From: Babak Naderi Date: Mon, 29 Jun 2026 20:55:56 +0200 Subject: [PATCH 004/111] Ignore internal general assets CSV Keep src/assets_master_script/general_assets_internal.csv out of version control; it is for internal usage only. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 0b3f9ae..ad6c95e 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,6 @@ desktop.ini .idea/ .env .vscode + +# internal-only math assets (keep local, do not commit) +src/assets_master_script/general_assets_internal.csv \ No newline at end of file From 616d5e390a98f9c739c98a37911678657d6e89c6 Mon Sep 17 00:00:00 2001 From: Babak Naderi Date: Mon, 29 Jun 2026 21:06:43 +0200 Subject: [PATCH 005/111] Add silent / no-speech option for P.804 ratings Add a 'The clip is silent - no voice to rate' checkbox to each trial in both the training and rating sections of P808_multi.html. When ticked (after the clip has played once) it sets all seven scales to 1 via the real radios, records an always-submitted is_silent_q{n}/is_silent_t{n} flag, and lets the trial complete. A detailed note is added to the Detailed Instructions section warning that misuse leads to rejection. result_parser.py drops silent cases (identified by the is_silent flag) before computing the MOS, guards clips left with no valid votes, and reports per-clip n_silent and is_silent_percentage columns. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/P808Template/P808_multi.html | 58 ++++++++++++++++++++++++++++++++ src/result_parser.py | 42 +++++++++++++++++++---- 2 files changed, 93 insertions(+), 7 deletions(-) diff --git a/src/P808Template/P808_multi.html b/src/P808Template/P808_multi.html index e46a0eb..2165628 100644 --- a/src/P808Template/P808_multi.html +++ b/src/P808Template/P808_multi.html @@ -1845,6 +1845,7 @@

Listen to the following audio sample

 

+


`; @@ -2242,6 +2243,8 @@ //$('input[name=' + tmp_name + ']').click({ trial_num: i, num: j, isTraining: true, id:tmp_name }, onRating); $('input[name=' + tmp_name + ']').click({trial_num: i, isTraining: true, id:tmp_name }, onRating); } + // "silent / nothing to rate" escape hatch for the training trial + $('input[name=silent_t' + (i + 1) + ']').change({ trial_num: i, isTraining: true }, onSilent); //elementsTraining[i] = el; // initialize the GUI //hideAllElementsOnlyNumT(i, 0); @@ -2299,6 +2302,8 @@ //set the listeners to change to next question when vote is given $('input[name=' + tmp_name + ']').click({trial_num: i, isTraining: false , id:tmp_name}, onRating); } + // "silent / nothing to rate" escape hatch: zeroes all 7 scales for this trial + $('input[name=silent_q' + (i + 1) + ']').change({ trial_num: i }, onSilent); //elements[i] = el; // initialize the GUI //hideAllElementsOnlyNumR(i, 0); @@ -2583,6 +2588,55 @@ } } + /* + Handle the per-trial "audio is silent / nothing to rate" checkbox. + When checked, all 7 scales for that trial are set to 0, marked as + answered, and disabled; when unchecked, the scales are restored. + */ + function onSilent(event) { + var n = event.data.trial_num + 1; + var isTraining = event.data.isTraining === true; + var p = isTraining ? 't' : 'q'; + var unanswered = isTraining ? unansweredQuestionsTraining : unansweredQuestions; + var checked = $('#silent_' + p + n).is(':checked'); + var dims = ['noise', 'disc', 'col', 'loud', 'reverb', 'sig', 'ovrl']; + if (checked) { + var finished = $('input[name=audio_n_finish_' + p + n + '_audio]').val(); + if (finished == 0) { + alert('Please listen to the audio sample once before marking it as silent.'); + $('#silent_' + p + n).prop('checked', false); + return; + } + // record the silent state in a hidden field that is always submitted, + // then set every scale to 1 using the real radios and lock the others. + $('#is_silent_' + p + n).val('1'); + for (var k = 0; k < dims.length; k++) { + var nm = p + n + '_' + dims[k]; + var group = $('input[name=' + nm + ']'); + var one = group.filter('[value="1"]'); + group.prop('checked', false); + one.prop('checked', true).prop('disabled', false); + group.not(one).prop('disabled', true); + unanswered.delete(nm); + } + if (isTraining) { + if (unansweredQuestionsTraining.size == 0) + trainingIsFinished(); + } else { + check_submit_button(); + } + } else { + $('#is_silent_' + p + n).val('0'); + for (var k = 0; k < dims.length; k++) { + var nm = p + n + '_' + dims[k]; + $('input[name=' + nm + ']').prop('disabled', false).prop('checked', false); + unanswered.add(nm); + } + if (!isTraining) + $('#submitButton').prop('disabled', true); + } + } + function trialEnded(isTraining) { if (isTraining) showNextQuestionInTrainingSection(); @@ -3405,6 +3459,10 @@

Scales and examples:

7. Overall Quality:

This refers to how well the sample you heard is suitable for purpose of everday speech communicaions considering all impairments.

+ +

Silent clips:

+

A few clips may be completely silent and contain no voice at all. For such a clip, tick the checkbox "The clip is silent — no voice to rate" shown under the audio player instead of rating the seven scales. Only use it when you hear no speech whatsoever; using it on a clip that does contain speech is a misuse and will lead to rejection.

+
-

NOTE: New Instruction (April, 2023). This HIT is newly designed with updated instruction. Please carefully read and follow

Please adjust the volume level of your headset to a comfortable level so that you hear the following audio sample very well. It is a very important step for this task and will directly influence your judgment. From eb57865dea65e6750640407148cc903a8fffd49e Mon Sep 17 00:00:00 2001 From: Babak Naderi Date: Mon, 29 Jun 2026 22:10:46 +0200 Subject: [PATCH 007/111] Decouple qualification into a separate, toggleable section Add a show_qualification option to [hit_app_html] (default true), injected by master_script into P808_multi.html as showQualification. When false, the main HIT skips the qualification section entirely - useful when qualification is run as a separate study. Update Qualification.html into a renderable template: the hearing-test number clips become {{cfg.num1_url}}..{{cfg.num5_url}} and the P.804 bandwidth/quality-discrimination test is added. Add a --qualification_only master_script mode (create_qualification_only) that renders the standalone qualification page from the general assets, skipping clip/session generation. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/P808Template/P808_multi.html | 6 +++ src/P808Template/Qualification.html | 64 ++++++++++++++++++++++++++--- src/configurations/master.cfg | 3 ++ src/master_script.py | 48 ++++++++++++++++++++++ 4 files changed, 115 insertions(+), 6 deletions(-) diff --git a/src/P808Template/P808_multi.html b/src/P808Template/P808_multi.html index bb6be30..0431ea0 100644 --- a/src/P808Template/P808_multi.html +++ b/src/P808Template/P808_multi.html @@ -286,6 +286,7 @@ cookieName: "{{cfg.cookie_name}}", qualificationCookieName: "{{cfg.qual_cookie_name}}", qualificationValidFor: 43200, + showQualification: {{cfg.show_qualification}}, forceRetrainingInHours: 3, showSetupEveryMinutes: 120, debug: "true", @@ -397,6 +398,11 @@ cookie_debug_status = ""; // given cookie availability disable the qualification qual_passed = readCookie(config['qualificationCookieName']); + // when the qualification section is disabled (e.g. it is run as a + // separate Prolific study), treat it as already passed and skip it. + if (config['showQualification'] === false) { + qual_passed = "true"; + } //------------- is_test = false; //------------ diff --git a/src/P808Template/Qualification.html b/src/P808Template/Qualification.html index bae9fd1..04c31cf 100644 --- a/src/P808Template/Qualification.html +++ b/src/P808Template/Qualification.html @@ -451,39 +451,91 @@

-   +  
-   +  
-   +  
-   +  
-   +  
- + +
+ +
+
+
+   +
+
+
+
+
+
+
+
+   +
+
+
+
+
+
+
+
+   +
+
+
+
+
+
+
+
+   +
+
+
+
+
+
+
+
+   +
+
+
+
+
+
+
+ +
+

Thank you for your participation. The qualifications will be assigned to a selected group of participants within the next 3 days.
diff --git a/src/configurations/master.cfg b/src/configurations/master.cfg index b93a794..aee2539 100644 --- a/src/configurations/master.cfg +++ b/src/configurations/master.cfg @@ -37,6 +37,9 @@ quantity_bonus: 0.1 quality_top_percentage: 20 quality_bonus: 0.15 contact_email:ic3ai@outlook.com +# show the in-HIT qualification section (true/false). Set to false when the +# qualification is run as a separate study (e.g. a Prolific screener). Default: true. +show_qualification: true # Deprecated use [hit_app_html] [acr_html] diff --git a/src/master_script.py b/src/master_script.py index 9fb6ce0..d3c401c 100644 --- a/src/master_script.py +++ b/src/master_script.py @@ -438,6 +438,13 @@ async def create_hit_app_pp835_p804( config["contact_email"] = ( cfg["contact_email"] if "contact_email" in cfg else "ic3ai@outlook.com" ) + # whether the in-HIT qualification section is shown. Default: shown. + config["show_qualification"] = ( + "false" + if str(cfg.get("show_qualification", "true")).strip().lower() + in ("false", "0", "no", "off") + else "true" + ) config["hit_base_payment"] = cfg["hit_base_payment"] config["quantity_hits_more_than"] = cfg["quantity_hits_more_than"] @@ -709,6 +716,37 @@ def prepare_basic_cfg(df): return config +def create_qualification_only(args): + """ + Generate the standalone qualification page (Qualification.html) for a project. + + Renders the qualification template with the general assets (the hearing-test + number clips) so it can be published as a separate qualification study, e.g. a + Prolific screener. Skips the rating/clip/session generation entirely. + + :param args: Parsed command-line arguments (uses project and general_assets). + :return: Path to the generated qualification HTML file. + """ + general_path = args.general_assets or os.path.join( + os.path.dirname(__file__), "assets_master_script/general.csv" + ) + assert os.path.exists(general_path), f"No general assets csv in {general_path}" + df_general = pd.read_csv(general_path) + config = prepare_basic_cfg(df_general) + + template_path = os.path.join(os.path.dirname(__file__), "P808Template/Qualification.html") + with open(template_path, "r", encoding="utf-8") as file: + content = file.read() + html = Template(content).render(cfg=config) + + os.makedirs(args.project, exist_ok=True) + out_path = os.path.join(args.project, f"{args.project}_qualification.html") + with open(out_path, "w", encoding="utf-8") as file: + file.write(html) + print(f" [{out_path}] is created") + return out_path + + def get_path(test_method, is_p831_fest): """ check all the preequsites and see if all resources are available @@ -1023,6 +1061,11 @@ def check_urls_in_files_exist(csv_file_path, columns): help="Path to the general assets CSV (default: assets_master_script/general.csv). " "Use assets_master_script/general_assets_internal.csv for internal assets." ) + parser.add_argument( + "--qualification_only", action='store_true', + help="Only generate the standalone qualification page (Qualification.html) for this " + "project, e.g. to publish a separate qualification screener. Skips clip/session generation." + ) # check input arguments args = parser.parse_args() @@ -1037,6 +1080,11 @@ def check_urls_in_files_exist(csv_file_path, columns): if args.p831_fest: assert test_method in p831_methods, f"This method is not supported with p831, please choose one of {p831_methods}" + if args.qualification_only: + # generate only the standalone qualification page and stop + create_qualification_only(args) + raise SystemExit(0) + assert os.path.exists(args.cfg), f"No config file in {args.cfg}" if args.training_clips: assert os.path.exists( From 04e4ce7476c1c5a7cd71f5391ace12e959e30a7d Mon Sep 17 00:00:00 2001 From: Babak Naderi Date: Tue, 30 Jun 2026 18:23:07 +0200 Subject: [PATCH 008/111] Disable front-end gold/trapping QC and stop embedding answers The rating-section gold/trapping correctness check ran in the browser and could block the HIT (the 'QA system recognized... answers known to us' message). Its correct answers were recoverable client-side - gold answers were base64 (reversible via atob), trapping answers were plain text - so the check could not be made tamper-proof for a 1-5 answer space. Blank the embedded gold/trapping answers and clip identifiers (goldClip/goldClip2 url+answers, knownQuestionUrl, knownQuestionAns), remove the answer example from the config comment, and neutralize the gold-failure warning/block at init. Gold/trapping/math validation is already performed server-side in result_parser.py. Training per-dimension feedback (training_gold_clips, knownQuestionInTraining*) is unchanged. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/P808Template/P808_multi.html | 76 ++++++++++++-------------------- 1 file changed, 28 insertions(+), 48 deletions(-) diff --git a/src/P808Template/P808_multi.html b/src/P808Template/P808_multi.html index 0431ea0..7b7cbe6 100644 --- a/src/P808Template/P808_multi.html +++ b/src/P808Template/P808_multi.html @@ -263,23 +263,6 @@ - randomizeRatingQuestions: boolean whether the questions in the rating section should be randomized on loading time. - allowedMaxHITsInProject: How many HITs each worker is allowed to answer in this project - allowedMaxContinuesSessionDurationInMinutes: how log is the maximum continue session. A break of 10min will be forced - - questionUrls: {{cfg.rating_urls}}, - trainingUrls: {{cfg.training_urls}}, - goldClip: { - url: "${gold_url_2}", - sig_ans:"${gold_sig_ans_2}", - noise_ans:"${gold_noise_ans_2}", - disc_ans:"${gold_disc_ans_2}", - col_ans:"${gold_col_ans_2}", - loud_ans:"${gold_loud_ans_2}", - reverb_ans:"${gold_reverb_ans_2}", - ovrl_ans: "${gold_ovrl_ans_2}", - n_fail_show_warning: 3, - n_fail_block: 6 - } - - */ var config = { @@ -295,29 +278,36 @@ training_gold_clips: {{ cfg.training_gold_clips }}, knownQuestionInTrainingUrl: "{{cfg.training_trap_urls}}", knownQuestionInTrainingAns: "{{cfg.training_trap_ans}}", - knownQuestionUrl: "${TP}", - knownQuestionAns: "${TP_ANS}", + // Front-end gold/trapping quality control is DISABLED: the correct answers + // are intentionally NOT embedded in the page. (They could be trivially + // recovered client-side - gold answers were base64, trapping answers plain + // text - so a front-end check cannot be made tamper-proof for a 1-5 answer + // space.) Gold/trapping validation is performed server-side in + // result_parser.py. Training feedback (config.training_gold_clips and + // knownQuestionInTraining*) is unaffected and still works. + knownQuestionUrl: "", + knownQuestionAns: "", goldClip: { - url: "${gold_url}", - sig_ans:"${gold_sig_ans}", - noise_ans:"${gold_noise_ans}", - disc_ans:"${gold_disc_ans}", - col_ans:"${gold_col_ans}", - loud_ans:"${gold_loud_ans}", - reverb_ans:"${gold_reverb_ans}", - ovrl_ans: "${gold_ovrl_ans}", + url: "", + sig_ans:"", + noise_ans:"", + disc_ans:"", + col_ans:"", + loud_ans:"", + reverb_ans:"", + ovrl_ans: "", n_fail_show_warning: 4, n_fail_block: 6 }, goldClip2: { - url: "${gold_url_2}", - sig_ans:"${gold_sig_ans_2}", - noise_ans:"${gold_noise_ans_2}", - disc_ans:"${gold_disc_ans_2}", - col_ans:"${gold_col_ans_2}", - loud_ans:"${gold_loud_ans_2}", - reverb_ans:"${gold_reverb_ans_2}", - ovrl_ans: "${gold_ovrl_ans_2}", + url: "", + sig_ans:"", + noise_ans:"", + disc_ans:"", + col_ans:"", + loud_ans:"", + reverb_ans:"", + ovrl_ans: "", dummy:"dummy" }, randomizeTrainingQuestions: "true", @@ -420,20 +410,10 @@ cookie_debug_status = cookie_debug_status + "ACR_LISTENER Exist*"; } - let gold_stat = getGoldQuestionStat(); - console.log("gold_stat:"+ gold_stat['failed'] +"warning:" + gold_stat['warning_showed'] ); + // Front-end gold QC disabled: never block or warn based on gold + // "failures" (validation is server-side). Keep force_re_training so the + // later training/cookie logic still works. let force_re_training = false; - if (gold_stat['failed']>= config['goldClip']['n_fail_block']) { - disableTheHIT(Hide_HIT_REASON.GOLD_FAILED); - }else if (gold_stat['failed']>= config['goldClip']['n_fail_show_warning'] && gold_stat['warning_showed'] == 0) { - console.log("show warning"); - // show warning - $("#gold_warning").show(); - // force training - force_re_training = true; - //gold_stat['warning_showed'] = 1; - //saveGoldQuestionStat(gold_stat); - } // detailed instruction if (is_test || readCookie(config['cookieName'] + "_instra")) { From 63fa94fca007a64a06fb7c34109629619a4a3a40 Mon Sep 17 00:00:00 2001 From: Babak Naderi Date: Tue, 30 Jun 2026 18:23:25 +0200 Subject: [PATCH 009/111] Improve standalone qualification page and answers export Make the qualification page generic (not study-specific), add the participant-group / 30-day expiry wording, switch the mother-tongue question to 10 multi-select language checkboxes, and fix the broken loudspeaker device image (use the working blob asset). In create_qualification_only: render the hearing-test clips as per-row placeholders, generate N qualification instances (--n_samples) in a single answers CSV (q_*/ans_* columns, plus the Q3-8 criteria) for server-side validation, and support --create_local_test. --method and --cfg are no longer required with --qualification_only. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/P808Template/Qualification.html | 83 +++++++++++---------- src/master_script.py | 108 ++++++++++++++++++++++++---- 2 files changed, 137 insertions(+), 54 deletions(-) diff --git a/src/P808Template/Qualification.html b/src/P808Template/Qualification.html index 04c31cf..c5cf919 100644 --- a/src/P808Template/Qualification.html +++ b/src/P808Template/Qualification.html @@ -56,13 +56,15 @@ } function makeCheckboxBeRequired(){ - var requiredCheckboxes = $('.4_lds :checkbox[required]'); - requiredCheckboxes.change(function(){ - if(requiredCheckboxes.is(':checked')) { - requiredCheckboxes.removeAttr('required'); - } else { - requiredCheckboxes.attr('required', 'required'); - } + $('.4_lds, .mts_group').each(function(){ + var group = $(this).find(':checkbox'); + group.on('change', function(){ + if(group.is(':checked')) { + group.removeAttr('required'); + } else { + group.attr('required', 'required'); + } + }); }); } function forceNoSpaceInInput(){ @@ -286,33 +288,27 @@
-

Instructions for speech quality assessment (Part 1 - Qualification)

+

Speech quality assessment — Qualification

Introduction -

We are looking for crowdworkers who are willing to participate in a speech quality assessment experiment. During that test you will listen to 11 audio files, each 6-8 seconds long (two sentences), via your listening device and you will be asked to indicate your opinion quality of each on the following scale:

+

Welcome! This is a generic qualification task for the speech quality assessment studies run with this account. By completing it once, you become eligible to take part in the speech quality assessment studies (for example ITU-T P.804 / P.808) published under this account.

- scale - -

Each of those HITs can be completed in about 5 minutes. There will be a total of 60 HITs available for each crowdworker. It results to $ 48 payout including bonuses. Bonuses will be granted based on 1)number of tasks you perform, 2)quality of your work.

- - Procedure: +

In those studies you listen to short speech samples (a few seconds each) through your listening device and give your opinion about the quality of the speech you hear, on one or more rating scales.

+ + How it works:
    -
  1. To get access to the above mentioned rating job, you should first complete this qualification job.
    2. -
  2. Selected group of crowdworkers will be invited to perform the training job (2 minutes) in which you will listen to 6 sample audio files.
    3. -
  3. Then, they get access to the rating job and can perform up to 57 tasks.
    4. +
  4. Complete this qualification task. If you pass, you will be added to a participant group that gets access to our further speech quality assessment studies.
    5. +
  5. Your eligibility expires automatically after 30 days. To renew it, simply take this qualification task again.
- - - scale - - Conditions: + + Conditions:
    -
  • You must perform the task in a quiet environment like at home.
    • -
  • You must use headphones. Note that, loudspeakers are not acceptable.
    • +
  • You must use a headset (headphones). Loudspeakers are not acceptable.
    • +
  • You must perform the task in a quiet environment, for example at home.
- -

Thank you for your help in this experiment.

+ +

Please answer the following questions carefully. Thank you for your participation.

@@ -349,9 +345,18 @@

-
-
- +
+
+
+
+
+
+
+
+
+
+
+
@@ -362,10 +367,10 @@

- - - - + + + + @@ -451,32 +456,32 @@

-   +  
-   +  
-   +  
-   +  
-   +  
@@ -537,7 +542,7 @@

-

Thank you for your participation. The qualifications will be assigned to a selected group of participants within the next 3 days.
+
Thank you for your participation. If you pass, you will be assigned to a participant group that gets access to further studies. Your eligibility expires automatically after 30 days, after which you need to take this qualification task again to renew it.
diff --git a/src/master_script.py b/src/master_script.py index d3c401c..dfb1494 100644 --- a/src/master_script.py +++ b/src/master_script.py @@ -720,30 +720,100 @@ def create_qualification_only(args): """ Generate the standalone qualification page (Qualification.html) for a project. - Renders the qualification template with the general assets (the hearing-test - number clips) so it can be published as a separate qualification study, e.g. a - Prolific screener. Skips the rating/clip/session generation entirely. - - :param args: Parsed command-line arguments (uses project and general_assets). + Writes the qualification HTML (the hearing-test number clips are ${q_numN} + placeholders, filled per row by the HIT app server) plus an answers CSV with + one row per qualification instance (args.n_samples rows). Each row holds the + question audio URLs (q_*) and correct answers (ans_*) for server-side + validation; no answers are embedded in the HTML. Optionally generates a local + preview from the first row when args.create_local_test is set. + + :param args: Parsed command-line arguments (project, general_assets, n_samples, + create_local_test). :return: Path to the generated qualification HTML file. """ + # Static bandwidth/quality-discrimination clips and their correct answers. + # These must match the comb_bw* clips in P808Template/Qualification.html. + bw_base = "https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/bw-test" + bandwidth_tests = [ + ("comb_bw1", f"{bw_base}/d_g1_cmb.wav", "dq"), + ("comb_bw2", f"{bw_base}/d_g2_cmb.wav", "dq"), + ("comb_bw3", f"{bw_base}/d_g3_cmb.wav", "dq"), + ("comb_bw4", f"{bw_base}/d_g4_cmb.wav", "sq"), + ("comb_bw5", f"{bw_base}/d_g5_cmb.wav", "sq"), + ] + # Non-audio criteria questions 3-8 and their expected answer(s) ("|"-separated + # when several are acceptable). Questions 5 and 6 are recency screeners with no + # fixed correct answer. + criteria = { + "ans_3_mother_tongue": "english|en", + "ans_4_ld": "in-ear|over-ear", + "ans_5_last_subjective": "", + "ans_6_audio_test": "", + "ans_7_working_area": "N", + "ans_8_hearing": "normal", + } + + n_samples = max(1, int(getattr(args, "n_samples", 1) or 1)) general_path = args.general_assets or os.path.join( os.path.dirname(__file__), "assets_master_script/general.csv" ) assert os.path.exists(general_path), f"No general assets csv in {general_path}" df_general = pd.read_csv(general_path) - config = prepare_basic_cfg(df_general) + # write the qualification HTML (placeholders are filled by the HIT app server) template_path = os.path.join(os.path.dirname(__file__), "P808Template/Qualification.html") with open(template_path, "r", encoding="utf-8") as file: - content = file.read() - html = Template(content).render(cfg=config) + html = file.read() os.makedirs(args.project, exist_ok=True) out_path = os.path.join(args.project, f"{args.project}_qualification.html") with open(out_path, "w", encoding="utf-8") as file: file.write(html) print(f" [{out_path}] is created") + + # answers CSV: one row per instance, q_* (audio URL) and ans_* (correct answer) + columns = [] + for i in range(1, 6): + columns += [f"q_num{i}", f"ans_num{i}"] + for name, _url, _ans in bandwidth_tests: + columns += [f"q_{name}", f"ans_{name}"] + columns += list(criteria.keys()) + + rows = [] + for _ in range(n_samples): + # a fresh random sampling of the hearing-test number clips per instance + sample_cfg = prepare_basic_cfg(df_general) + row = {} + for i in range(1, 6): + encoded = sample_cfg.get(f"num{i}_ans") + row[f"q_num{i}"] = sample_cfg.get(f"num{i}_url", "") + row[f"ans_num{i}"] = base64.b64decode(encoded).decode("ascii") if encoded else "" + for name, url, answer in bandwidth_tests: + row[f"q_{name}"] = url + row[f"ans_{name}"] = answer + row.update(criteria) + rows.append(row) + + answers_df = pd.DataFrame(rows, columns=columns) + answers_path = os.path.join(args.project, f"{args.project}_qualification_answers.csv") + answers_df.to_csv(answers_path, index=False) + print(f" [{answers_path}] ({n_samples} instance(s)) is created") + + # optional local preview built from the first instance + if getattr(args, "create_local_test", False): + from utils.preview_html import ( + replace_placeholders, + replace_with_public_urls, + _disable_fetch_for_local, + ) + preview_html = _disable_fetch_for_local( + replace_placeholders(replace_with_public_urls(html), answers_df.iloc[0]) + ) + preview_path = os.path.join(args.project, f"{args.project}_qualification_row-1.html") + with open(preview_path, "w", encoding="utf-8") as file: + file.write(preview_html) + print(f" [{preview_path}] is created") + return out_path @@ -1029,8 +1099,8 @@ def check_urls_in_files_exist(csv_file_path, columns): print("Welcome to the Master script for P808 Toolkit.") parser = argparse.ArgumentParser(description='Master script to prepare the P.808 subjective test') parser.add_argument("--project", help="Name of the project", required=True) - parser.add_argument("--cfg", help="Configuration file, see master.cfg", required=True) - parser.add_argument("--method", required=True, + parser.add_argument("--cfg", help="Configuration file, see master.cfg", required=False) + parser.add_argument("--method", required=False, help=f"one of the test methods: 'acr', 'dcr', 'ccr', 'p835','{p835_personalized}', p804, or 'echo_impairment_test'") parser.add_argument("--p831_fest", action='store_true', help="Use the question set of P.831 for FEST") parser.add_argument("--clips", help="A csv containing urls of all clips to be rated in column 'rating_clips', in " @@ -1066,10 +1136,22 @@ def check_urls_in_files_exist(csv_file_path, columns): help="Only generate the standalone qualification page (Qualification.html) for this " "project, e.g. to publish a separate qualification screener. Skips clip/session generation." ) + parser.add_argument( + "--n_samples", type=int, default=1, + help="Number of qualification instances (rows) to generate in the answers CSV " + "when using --qualification_only. Default: 1." + ) # check input arguments args = parser.parse_args() + if args.qualification_only: + # only generate the standalone qualification page; this mode needs just + # --project (and optionally --general_assets), not --method or --cfg. + create_qualification_only(args) + raise SystemExit(0) + + assert args.method, "--method is required (except with --qualification_only)" methods = ["acr", "dcr", "ccr", "p835", "echo_impairment_test", p835_personalized, 'p804'] test_method = args.method.lower() assert ( @@ -1080,11 +1162,7 @@ def check_urls_in_files_exist(csv_file_path, columns): if args.p831_fest: assert test_method in p831_methods, f"This method is not supported with p831, please choose one of {p831_methods}" - if args.qualification_only: - # generate only the standalone qualification page and stop - create_qualification_only(args) - raise SystemExit(0) - + assert args.cfg, "--cfg is required (except with --qualification_only)" assert os.path.exists(args.cfg), f"No config file in {args.cfg}" if args.training_clips: assert os.path.exists( From 9e6e35b12d67171def2710250d240794aca7248a Mon Sep 17 00:00:00 2001 From: Babak Naderi Date: Tue, 30 Jun 2026 18:56:17 +0200 Subject: [PATCH 010/111] Disable front-end gold/trapping QC across remaining HIT templates Apply the same fix as P808_multi.html (commit 04e4ce7) to the other HIT templates: stop embedding the correct gold/trapping answers and clip identifiers in the served page, and neutralize the gold-failure warning/block where present. The embedded answers were recoverable client-side (gold base64, trapping plain text), so the front-end check could not be made tamper-proof; gold/trapping validation is done server-side in result_parser.py. Per template: blank knownQuestionUrl and the gold answer/identity keys (goldClipURL/goldClipAns or the goldClip/goldClip2 url+*_ans fields). DCR/CCR/P831_DCR keep their structural constant knownQuestionAns (5/0, not a secret) but blank knownQuestionUrl. bw_check.html and P835_personalized_template3.html also had the init warning/block neutralized and comment-block placeholders cleaned. Training feedback (knownQuestionInTraining*) is unchanged. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/P808Template/ACR_template.html | 8 ++--- src/P808Template/CCR_template.html | 4 +-- src/P808Template/DCR_template.html | 2 +- src/P808Template/P831_ACR_template.html | 8 ++--- src/P808Template/P831_DCR_template.html | 2 +- .../P835_personalized_template3.html | 35 ++++++------------ src/P808Template/P835_template.html | 8 ++--- src/P808Template/P835_template_one_audio.html | 8 ++--- src/P808Template/bw_check.html | 36 +++++++------------ .../echo_impairment_test_fest_template.html | 8 ++--- .../echo_impairment_test_template.html | 8 ++--- 11 files changed, 51 insertions(+), 76 deletions(-) diff --git a/src/P808Template/ACR_template.html b/src/P808Template/ACR_template.html index 6988d22..0f25776 100644 --- a/src/P808Template/ACR_template.html +++ b/src/P808Template/ACR_template.html @@ -84,10 +84,10 @@ knownQuestionInTrainingUrl:"{{cfg.training_trap_urls}}", knownQuestionInTrainingAns: "{{cfg.training_trap_ans}}", - knownQuestionUrl:"${TP}", - knownQuestionAns: "${TP_ANS}", - goldClipURL:"${gold_clips}", - goldClipAns:"${gold_clips_ans}", + knownQuestionUrl:"", + knownQuestionAns: "", + goldClipURL:"", + goldClipAns:"", randomizeTrainingQuestions:"true", randomizeRatingQuestions:"true", allowedMaxHITsInProject:{{cfg.allowed_max_hit_in_project}}, diff --git a/src/P808Template/CCR_template.html b/src/P808Template/CCR_template.html index 43794c2..36659d3 100644 --- a/src/P808Template/CCR_template.html +++ b/src/P808Template/CCR_template.html @@ -92,7 +92,7 @@ knownQuestionInTrainingUrl:"{{cfg.training_trap_urls}}", knownQuestionInTrainingAns: "0", - knownQuestionUrl:"${TP}", + knownQuestionUrl:"", knownQuestionAns: "0", randomizeTrainingQuestions:"true", @@ -582,7 +582,7 @@ } /* -Remove the answers for all pair-comparison questions in the setup section. +Remove the answers for all pair-comparison questions in the setup section. */ function makeAllCMPsUnChecked(){ // remove the listeners diff --git a/src/P808Template/DCR_template.html b/src/P808Template/DCR_template.html index 335ddf0..47cff95 100644 --- a/src/P808Template/DCR_template.html +++ b/src/P808Template/DCR_template.html @@ -84,7 +84,7 @@ knownQuestionInTrainingUrl:"{{cfg.training_trap_urls}}", knownQuestionInTrainingAns: "5", - knownQuestionUrl:"${TP}", + knownQuestionUrl:"", knownQuestionAns: "5", randomizeTrainingQuestions:"true", diff --git a/src/P808Template/P831_ACR_template.html b/src/P808Template/P831_ACR_template.html index b763e1c..c258453 100644 --- a/src/P808Template/P831_ACR_template.html +++ b/src/P808Template/P831_ACR_template.html @@ -83,10 +83,10 @@ knownQuestionInTrainingUrl:"{{cfg.training_trap_urls}}", knownQuestionInTrainingAns: "{{cfg.training_trap_ans}}", - knownQuestionUrl:"${TP}", - knownQuestionAns: "${TP_ANS}", - goldClipURL:"${gold_clips}", - goldClipAns:"${gold_clips_ans}", + knownQuestionUrl:"", + knownQuestionAns: "", + goldClipURL:"", + goldClipAns:"", randomizeTrainingQuestions:"true", randomizeRatingQuestions:"true", allowedMaxHITsInProject:{{cfg.allowed_max_hit_in_project}}, diff --git a/src/P808Template/P831_DCR_template.html b/src/P808Template/P831_DCR_template.html index ede7d61..b892521 100644 --- a/src/P808Template/P831_DCR_template.html +++ b/src/P808Template/P831_DCR_template.html @@ -84,7 +84,7 @@ knownQuestionInTrainingUrl:"{{cfg.training_trap_urls}}", knownQuestionInTrainingAns: "5", - knownQuestionUrl:"${TP}", + knownQuestionUrl:"", knownQuestionAns: "5", randomizeTrainingQuestions:"true", diff --git a/src/P808Template/P835_personalized_template3.html b/src/P808Template/P835_personalized_template3.html index 49bed2d..28b27b3 100644 --- a/src/P808Template/P835_personalized_template3.html +++ b/src/P808Template/P835_personalized_template3.html @@ -133,10 +133,10 @@ questionUrls: {{cfg.rating_urls}}, trainingUrls: {{cfg.training_urls}}, goldClip: { - url: "${gold_url_2}", - sig_ans:"${gold_sig_ans_2}", - bak_ans:"${gold_bak_ans_2}", - ovrl_ans: "${gold_ovrl_ans_2}", + url: "", + sig_ans:"", + bak_ans:"", + ovrl_ans: "", n_fail_show_warning: 3, n_fail_block: 5 } @@ -155,13 +155,13 @@ training_gold_clips: {{ cfg.training_gold_clips }}, knownQuestionInTrainingUrl: "{{cfg.training_trap_urls}}", knownQuestionInTrainingAns: "{{cfg.training_trap_ans}}", - knownQuestionUrl: "${TP}", - knownQuestionAns: "${TP_ANS}", + knownQuestionUrl: "", + knownQuestionAns: "", goldClip: { - url: "${gold_url}", - sig_ans:"${gold_sig_ans}", - bak_ans:"${gold_bak_ans}", - ovrl_ans: "${gold_ovrl_ans}", + url: "", + sig_ans:"", + bak_ans:"", + ovrl_ans: "", n_fail_show_warning: 3, n_fail_block: 5 }, @@ -248,21 +248,8 @@ cookie_debug_status = cookie_debug_status + "ACR_LISTENER Exist*"; } - let gold_stat = getGoldQuestionStat(); - console.log("gold_stat:"+ gold_stat['failed'] +"warning:" + gold_stat['warning_showed'] ); - cookie_debug_status = cookie_debug_status + "gold_stat"+ gold_stat['failed'] +",warning:" + gold_stat['warning_showed'] + "*"; + // Front-end gold QC disabled: never block or warn based on gold failures. let force_re_training = false; - if (gold_stat['failed']>= config['goldClip']['n_fail_block']) { - disableTheHIT(Hide_HIT_REASON.GOLD_FAILED); - }else if (gold_stat['failed']>= config['goldClip']['n_fail_show_warning'] && gold_stat['warning_showed'] == 0) { - console.log("show warning"); - // show warning - $("#gold_warning").show(); - // force training - force_re_training = true; - gold_stat['warning_showed'] = 1; - saveGoldQuestionStat(gold_stat); - } // given cookie availability disable the training if (readCookie(config['cookieName'] + "_training") && !force_re_training) { $('.trainingFieldset').prop("disabled", true); diff --git a/src/P808Template/P835_template.html b/src/P808Template/P835_template.html index 2e2148c..1fe85ae 100644 --- a/src/P808Template/P835_template.html +++ b/src/P808Template/P835_template.html @@ -94,10 +94,10 @@ knownQuestionInTrainingUrl:"{{cfg.training_trap_urls}}", knownQuestionInTrainingAns: "{{cfg.training_trap_ans}}", - knownQuestionUrl:"${TP}", - knownQuestionAns: "${TP_ANS}", - goldClipURL:"${gold_clips}", - goldClipAns:"${gold_clips_ans}", + knownQuestionUrl:"", + knownQuestionAns: "", + goldClipURL:"", + goldClipAns:"", randomizeTrainingQuestions:"true", randomizeRatingQuestions:"true", allowedMaxHITsInProject:{{cfg.allowed_max_hit_in_project}}, diff --git a/src/P808Template/P835_template_one_audio.html b/src/P808Template/P835_template_one_audio.html index 1645f0c..417c8e9 100644 --- a/src/P808Template/P835_template_one_audio.html +++ b/src/P808Template/P835_template_one_audio.html @@ -90,10 +90,10 @@ knownQuestionInTrainingUrl:"{{cfg.training_trap_urls}}", knownQuestionInTrainingAns: "{{cfg.training_trap_ans}}", - knownQuestionUrl:"${TP}", - knownQuestionAns: "${TP_ANS}", - goldClipURL:"${gold_clips}", - goldClipAns:"${gold_clips_ans}", + knownQuestionUrl:"", + knownQuestionAns: "", + goldClipURL:"", + goldClipAns:"", randomizeTrainingQuestions:"true", randomizeRatingQuestions:"true", allowedMaxHITsInProject:{{cfg.allowed_max_hit_in_project}}, diff --git a/src/P808Template/bw_check.html b/src/P808Template/bw_check.html index 5611399..3bf0faa 100644 --- a/src/P808Template/bw_check.html +++ b/src/P808Template/bw_check.html @@ -149,13 +149,13 @@ training_gold_clips: {{ cfg.training_gold_clips }}, knownQuestionInTrainingUrl: "{{cfg.training_trap_urls}}", knownQuestionInTrainingAns: "{{cfg.training_trap_ans}}", - knownQuestionUrl: "${TP}", - knownQuestionAns: "${TP_ANS}", + knownQuestionUrl: "", + knownQuestionAns: "", goldClip: { - url: "${gold_url}", - sig_ans:"${gold_sig_ans}", - bak_ans:"${gold_bak_ans}", - ovrl_ans: "${gold_ovrl_ans}", + url: "", + sig_ans:"", + bak_ans:"", + ovrl_ans: "", n_fail_show_warning: 3, n_fail_block: 5 }, @@ -217,13 +217,13 @@ "ovrl":{"ans": "aHR0cHM6Ly9wODM1LXJlZi1sYW5nLnMzLmFtYXpvbmF3cy5jb20vdHBzL2kwMV9mMV8xX3Nob3J0Lndhdl81", msg:"ovrl_is_wrong", var:1}} ], - knownQuestionUrl: "https://s3-us-west-1.amazonaws.com/itutest.qu.tuberlin.de/401/D401_c01_ef02_s002.wav", - knownQuestionAns: "TP_ANS", + knownQuestionUrl: "", + knownQuestionAns: "", // default variance is 1 if a scale is not relevant, remove it from list goldClip: { - "url":'https://mlvideoprem.blob.core.windows.net/mlcodec-gan-prem/tmp/1sec.wav', - 'sig_ans':'aHR0cHM6Ly9tbHZpZGVvcHJlbS5ibG9iLmNvcmUud2luZG93cy5uZXQvbWxjb2RlYy1nYW4tcHJlbS90bXAvMXNlYy53YXZfMg==', - 'ovrl_ans':'aHR0cHM6Ly9tbHZpZGVvcHJlbS5ibG9iLmNvcmUud2luZG93cy5uZXQvbWxjb2RlYy1nYW4tcHJlbS90bXAvMXNlYy53YXZfMg==', + "url":"", + 'sig_ans':"", + 'ovrl_ans':"", 'n_fail_show_warning': 2, 'n_fail_block': 6 }, @@ -307,20 +307,8 @@ cookie_debug_status = cookie_debug_status + "ACR_LISTENER Exist*"; } - let gold_stat = getGoldQuestionStat(); - console.log("gold_stat:"+ gold_stat['failed'] +"warning:" + gold_stat['warning_showed'] ); + // Front-end gold QC disabled: never block or warn based on gold failures. let force_re_training = false; - if (gold_stat['failed']>= config['goldClip']['n_fail_block']) { - disableTheHIT(Hide_HIT_REASON.GOLD_FAILED); - }else if (gold_stat['failed']>= config['goldClip']['n_fail_show_warning'] && gold_stat['warning_showed'] == 0) { - console.log("show warning"); - // show warning - $("#gold_warning").show(); - // force training - force_re_training = true; - gold_stat['warning_showed'] = 1; - saveGoldQuestionStat(gold_stat); - } // given cookie availability disable the training //if (readCookie(config['cookieName'] + "_training") && !force_re_training) { $('.trainingFieldset').prop("disabled", true); diff --git a/src/P808Template/echo_impairment_test_fest_template.html b/src/P808Template/echo_impairment_test_fest_template.html index 338a14a..6857091 100644 --- a/src/P808Template/echo_impairment_test_fest_template.html +++ b/src/P808Template/echo_impairment_test_fest_template.html @@ -90,10 +90,10 @@ knownQuestionInTrainingUrl:"{{cfg.training_trap_urls}}", knownQuestionInTrainingAns: "{{cfg.training_trap_ans}}", - knownQuestionUrl:"${TP}", - knownQuestionAns: "${TP_ANS}", - goldClipURL:"${gold_clips}", - goldClipAns:"${gold_clips_ans}", + knownQuestionUrl:"", + knownQuestionAns: "", + goldClipURL:"", + goldClipAns:"", randomizeTrainingQuestions:"true", randomizeRatingQuestions:"true", allowedMaxHITsInProject:{{cfg.allowed_max_hit_in_project}}, diff --git a/src/P808Template/echo_impairment_test_template.html b/src/P808Template/echo_impairment_test_template.html index 8ae9a31..3aa79c3 100644 --- a/src/P808Template/echo_impairment_test_template.html +++ b/src/P808Template/echo_impairment_test_template.html @@ -90,10 +90,10 @@ knownQuestionInTrainingUrl:"{{cfg.training_trap_urls}}", knownQuestionInTrainingAns: "{{cfg.training_trap_ans}}", - knownQuestionUrl:"${TP}", - knownQuestionAns: "${TP_ANS}", - goldClipURL:"${gold_clips}", - goldClipAns:"${gold_clips_ans}", + knownQuestionUrl:"", + knownQuestionAns: "", + goldClipURL:"", + goldClipAns:"", randomizeTrainingQuestions:"true", randomizeRatingQuestions:"true", allowedMaxHITsInProject:{{cfg.allowed_max_hit_in_project}}, From 38423611f8b2bf453a3404a79dc60d3fd7082c78 Mon Sep 17 00:00:00 2001 From: Babak Naderi Date: Tue, 30 Jun 2026 19:00:44 +0200 Subject: [PATCH 011/111] Highlight the headset rule in red across all HIT templates Make the 'use a headset, not the loudspeaker' rule red in the instructions/rules of every HIT template and the standalone qualification page, to reduce loudspeaker use and rejections. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/P808Template/ACR_template.html | 2 +- src/P808Template/CCR_template.html | 2 +- src/P808Template/DCR_template.html | 2 +- src/P808Template/P808_multi.html | 2 +- src/P808Template/P831_ACR_template.html | 2 +- src/P808Template/P831_DCR_template.html | 2 +- src/P808Template/P835_personalized_template3.html | 2 +- src/P808Template/P835_template.html | 2 +- src/P808Template/P835_template_one_audio.html | 2 +- src/P808Template/Qualification.html | 2 +- src/P808Template/bw_check.html | 2 +- src/P808Template/echo_impairment_test_fest_template.html | 2 +- src/P808Template/echo_impairment_test_template.html | 2 +- 13 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/P808Template/ACR_template.html b/src/P808Template/ACR_template.html index 0f25776..0c09ec2 100644 --- a/src/P808Template/ACR_template.html +++ b/src/P808Template/ACR_template.html @@ -1233,7 +1233,7 @@

You should follow the below mentioned rules, otherwise your answers will be invalid.

Rules:

    -
  • You must use a headset, not the loudspeaker: otherwise your response will be rejected
    • +
  • You must use a headset, not the loudspeaker: otherwise your response will be rejected
  • You must perform the task in a quiet environment
  • Do not change the volume after modifying it in the Setup section.
diff --git a/src/P808Template/CCR_template.html b/src/P808Template/CCR_template.html index 36659d3..f540d8e 100644 --- a/src/P808Template/CCR_template.html +++ b/src/P808Template/CCR_template.html @@ -1442,7 +1442,7 @@

You should follow the below mentioned rules, otherwise your answers will be invalid.

Rules:

    -
  • Use a headset, not the loudspeaker: otherwise your response will be rejected
    • +
  • Use a headset, not the loudspeaker: otherwise your response will be rejected
  • Perform the task in a quite environment
  • Do not change the volume after modifying it in the Setup section.
diff --git a/src/P808Template/DCR_template.html b/src/P808Template/DCR_template.html index 47cff95..6ccff62 100644 --- a/src/P808Template/DCR_template.html +++ b/src/P808Template/DCR_template.html @@ -1149,7 +1149,7 @@

You should follow the below mentioned rules, otherwise your answers will be invalid.

Rules:

    -
  • Use a headset, not the loudspeaker: otherwise your response will be rejected
    • +
  • Use a headset, not the loudspeaker: otherwise your response will be rejected
  • Perform the task in a quite environment
  • Do not change the volume after modifying it in the Setup section.
diff --git a/src/P808Template/P808_multi.html b/src/P808Template/P808_multi.html index 7b7cbe6..746836a 100644 --- a/src/P808Template/P808_multi.html +++ b/src/P808Template/P808_multi.html @@ -2765,7 +2765,7 @@

You should follow the below mentioned rules, otherwise your answers will be invalid.

Rules:

    -
  • You must use a headset, not the loudspeaker: otherwise your response will be rejected +
  • You must use a headset, not the loudspeaker: otherwise your response will be rejected
  • You must perform the task in a quiet environment
  • Do not change the volume after modifying it in the Setup section.
    • diff --git a/src/P808Template/P831_ACR_template.html b/src/P808Template/P831_ACR_template.html index c258453..5eb14b4 100644 --- a/src/P808Template/P831_ACR_template.html +++ b/src/P808Template/P831_ACR_template.html @@ -1151,7 +1151,7 @@

    You should follow the below mentioned rules, otherwise your answers will be invalid.

    Rules:

      -
    • You must use a headset, not the loudspeaker: otherwise your response will be rejected
      • +
    • You must use a headset, not the loudspeaker: otherwise your response will be rejected
    • You must perform the task in a quiet environment
    • Do not change the volume after modifying it in the Setup section.
    diff --git a/src/P808Template/P831_DCR_template.html b/src/P808Template/P831_DCR_template.html index b892521..db0a788 100644 --- a/src/P808Template/P831_DCR_template.html +++ b/src/P808Template/P831_DCR_template.html @@ -1084,7 +1084,7 @@

    You should follow the below mentioned rules, otherwise your answers will be invalid.

    Rules:

      -
    • Use a headset, not the loudspeaker: otherwise your response will be rejected
      • +
    • Use a headset, not the loudspeaker: otherwise your response will be rejected
    • Perform the task in a quite environment
    • Do not change the volume after modifying it in the Setup section.
    diff --git a/src/P808Template/P835_personalized_template3.html b/src/P808Template/P835_personalized_template3.html index 28b27b3..fb6a0a0 100644 --- a/src/P808Template/P835_personalized_template3.html +++ b/src/P808Template/P835_personalized_template3.html @@ -1845,7 +1845,7 @@

    You should follow the below mentioned rules, otherwise your answers will be invalid.

    Rules:

      -
    • You must use a headset, not the loudspeaker: otherwise your response will be rejected +
    • You must use a headset, not the loudspeaker: otherwise your response will be rejected
    • You must perform the task in a quiet environment
    • Do not change the volume after modifying it in the Setup section.
      • diff --git a/src/P808Template/P835_template.html b/src/P808Template/P835_template.html index 1fe85ae..d2f13f4 100644 --- a/src/P808Template/P835_template.html +++ b/src/P808Template/P835_template.html @@ -1490,7 +1490,7 @@

      You should follow the below mentioned rules, otherwise your answers will be invalid.

      Rules:

        -
      • You must use a headset, not the loudspeaker: otherwise your response will be rejected
        • +
      • You must use a headset, not the loudspeaker: otherwise your response will be rejected
      • You must perform the task in a quiet environment
      • Do not change the volume after modifying it in the Setup section.
      diff --git a/src/P808Template/P835_template_one_audio.html b/src/P808Template/P835_template_one_audio.html index 417c8e9..36ac977 100644 --- a/src/P808Template/P835_template_one_audio.html +++ b/src/P808Template/P835_template_one_audio.html @@ -1293,7 +1293,7 @@

      You should follow the below mentioned rules, otherwise your answers will be invalid.

      Rules:

        -
      • You must use a headset, not the loudspeaker: otherwise your response will be rejected
        • +
      • You must use a headset, not the loudspeaker: otherwise your response will be rejected
      • You must perform the task in a quiet environment
      • Do not change the volume after modifying it in the Setup section.
      diff --git a/src/P808Template/Qualification.html b/src/P808Template/Qualification.html index c5cf919..bfb1c8d 100644 --- a/src/P808Template/Qualification.html +++ b/src/P808Template/Qualification.html @@ -304,7 +304,7 @@ Conditions:
        -
      • You must use a headset (headphones). Loudspeakers are not acceptable.
        • +
      • You must use a headset (headphones). Loudspeakers are not acceptable.
      • You must perform the task in a quiet environment, for example at home.
      diff --git a/src/P808Template/bw_check.html b/src/P808Template/bw_check.html index 3bf0faa..6821e0e 100644 --- a/src/P808Template/bw_check.html +++ b/src/P808Template/bw_check.html @@ -2125,7 +2125,7 @@

      You should follow the below mentioned rules, otherwise your answers will be invalid.

      Rules:

        -
      • You must use a headset, not the loudspeaker: otherwise your response will be rejected +
      • You must use a headset, not the loudspeaker: otherwise your response will be rejected
      • You must perform the task in a quiet environment
      • Do not change the volume after modifying it in the Setup section.
        • diff --git a/src/P808Template/echo_impairment_test_fest_template.html b/src/P808Template/echo_impairment_test_fest_template.html index 6857091..1ce0f7f 100644 --- a/src/P808Template/echo_impairment_test_fest_template.html +++ b/src/P808Template/echo_impairment_test_fest_template.html @@ -1388,7 +1388,7 @@

        You should follow the below mentioned rules, otherwise your answers will be invalid.

        Rules:

          -
        • You must use a headset, not the loudspeaker: otherwise your response will be rejected
          • +
        • You must use a headset, not the loudspeaker: otherwise your response will be rejected
        • You must perform the task in a quiet environment
        • Do not change the volume after modifying it in the Setup section.
        diff --git a/src/P808Template/echo_impairment_test_template.html b/src/P808Template/echo_impairment_test_template.html index 3aa79c3..11d81c7 100644 --- a/src/P808Template/echo_impairment_test_template.html +++ b/src/P808Template/echo_impairment_test_template.html @@ -1396,7 +1396,7 @@

        You should follow the below mentioned rules, otherwise your answers will be invalid.

        Rules:

          -
        • You must use a headset, not the loudspeaker: otherwise your response will be rejected
          • +
        • You must use a headset, not the loudspeaker: otherwise your response will be rejected
        • You must perform the task in a quiet environment
        • Do not change the volume after modifying it in the Setup section.
        From 4d763e3bbc77c6116b239d1ada93d2095efc80c5 Mon Sep 17 00:00:00 2001 From: Babak Naderi Date: Wed, 1 Jul 2026 16:05:52 +0200 Subject: [PATCH 012/111] Add bandwidth-check clip generator and docs Add src/utils/create_bandwidth_check_clips.py to generate the P.80x qualification bandwidth-discrimination clips from clean references. Each reference yields five 'A | beep | B' clips: q1-q3 add band-limited noise (3.5/8.5/15-22 kHz at +13 dB over active speech level) to the second half (answer dq), q4/q5 carry no audible change (answer sq). Both halves come from the same source; every segment gets an independent inaudible dither so halves and the two 'same' clips are never bit-identical. Output names are anonymized UUIDs with a bandwidth_check_clips.csv manifest (ref_clip, q1-q5, ans_q1-q5). Add docs/bandwidth_check_clips.md and link it from docs/preparation.md. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/bandwidth_check_clips.md | 128 ++++++ docs/preparation.md | 1 + src/utils/create_bandwidth_check_clips.py | 471 ++++++++++++++++++++++ 3 files changed, 600 insertions(+) create mode 100644 docs/bandwidth_check_clips.md create mode 100644 src/utils/create_bandwidth_check_clips.py diff --git a/docs/bandwidth_check_clips.md b/docs/bandwidth_check_clips.md new file mode 100644 index 0000000..d814e95 --- /dev/null +++ b/docs/bandwidth_check_clips.md @@ -0,0 +1,128 @@ +> **⚠️ Note: This document is AI-generated and has not been reviewed yet. After review, remove this comment and below LLM targeted message.** + + + +[Home](../README.md) > [Preparation](preparation.md) > Bandwidth Check Clips + +# Bandwidth Check Clips + +Bandwidth-check clips are the "same vs. different quality" discrimination items used in the +P.80x qualification test. Each clip plays two speech segments joined by a short beep +(`segment A | beep | segment B`) and the participant is asked whether the audio *after* the +beep sounds different from the audio *before* it. + +They screen the participant's playback chain and hearing: high-frequency band-limited noise is +added to the second half of some clips. A participant only perceives the difference if their +equipment and hearing reproduce that band, which lets the test discriminate wideband (WB), +super-wideband (SWB), and full-band (FB) capable setups. + +`create_bandwidth_check_clips.py` generates these clips from a set of clean, full-band +reference recordings. + +## Overview + +Both halves of every clip always come from the **same** source reference. The five cases per +reference are: + +| Case | Second half | Correct answer | +|------|-------------|----------------| +| q1 | reference + noise (3.5–22 kHz) | `dq` (different) | +| q2 | reference + noise (8.5–22 kHz) | `dq` (different) | +| q3 | reference + noise (15–22 kHz) | `dq` (different) | +| q4 | reference (no audible change) | `sq` (same) | +| q5 | reference (no audible change) | `sq` (same) | + +The band-limited noise is set about **+13 dB** above the reference active speech level (ITU-T +P.56), so it is clearly audible *within its band*. q1 uses the widest band and is the obvious +attention case; q3 (15–22 kHz) only reveals itself on full-band capable equipment. q4 and q5 +carry no audible change and are the obvious "same" trapping cases. These design answers match +the hosted production clips `d_g1_cmb.wav` … `d_g5_cmb.wav` referenced by `master_script.py` +and `P808Template/Qualification.html` (`dq, dq, dq, sq, sq`). + +### Inaudible dither + +Every segment also receives an independent, **inaudible** dither (around −75 dBov, far below +the speech but above the 16-bit LSB). This keeps the two halves — and the two "same" clips +(q4/q5) — from ever being bit-identical, so exact-match and deduplication detection cannot flag +them, while humans still perceive the two halves as identical. + +### Anonymized names + +Output clip file names are random UUIDs so the hosted names do not reveal the source reference. +The manifest CSV keeps the source-to-clip mapping. + +## Generating the clips + +```bash +cd src +python utils/create_bandwidth_check_clips.py ^ + --input_dir C:/datasets/p501 ^ + --output_dir output/bw_test ^ + --base_url https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/bw-test +``` + +### Arguments + +| Argument | Required | Default | Description | +|----------|----------|---------|-------------| +| `--input_dir`, `-i` | Yes | — | Directory containing clean, full-band reference WAV files. | +| `--output_dir`, `-o` | Yes | — | Directory for the generated clips and the manifest CSV. | +| `--base_url` | No | — | Base URL where clips will be hosted. When set, the `q1`…`q5` CSV columns hold full URLs instead of bare file names. | +| `--noise_gain_db` | No | 13.0 | Band-noise level relative to the reference active speech level, in dB. | +| `--beep_freq` | No | 440.0 | Beep tone frequency in Hz. | +| `--beep_sec` | No | 1.0 | Beep tone duration in seconds. | +| `--gap_sec` | No | 0.2 | Silence gap on each side of the beep, in seconds. | +| `--seed` | No | — | Integer seed for reproducible noise. | +| `--limit` | No | — | Cap on the number of references processed. | + +### Output + +The script writes, to `--output_dir`: + +- Five WAV clips per reference, named with random UUIDs (source sample rate and subtype preserved). +- `bandwidth_check_clips.csv`, one row per reference, with columns `ref_clip`, `q1`…`q5` + (clip file name, or full URL when `--base_url` is given) and `ans_q1`…`ans_q5` + (the correct answer `dq`/`sq` for each case). + +References sampled below twice the lowest band edge (3.5 kHz) cannot carry the high-frequency +noise and are skipped with a warning; use full-band (48 kHz) sources. + +## Source clip recommendations + +- Use clean, full-band speech at 48 kHz (for example the ITU-T P.501 references in + `C:/datasets/p501`). +- Include a variety of speakers (male and female). +- Four to six references are enough to build a diverse qualification pool. + +## Using the clips in the qualification test + +The P.80x qualification page (`P808Template/Qualification.html`) references five bandwidth +clips as `comb_bw1` … `comb_bw5`, and `master_script.py` (`create_qualification_only`) lists +their correct answers. To use freshly generated clips: + +1. Upload the generated WAV files to the bandwidth-test container + (`p808-assets/clips/bw-test`, or your own public location). +2. Map `q1`…`q5` from `bandwidth_check_clips.csv` to `comb_bw1`…`comb_bw5`, keeping the + `ans_q1`…`ans_q5` answers (`dq, dq, dq, sq, sq`). +3. Validation is performed server-side against these answers; no correct answers are embedded + in the HTML. + +## Reproducibility + +- **Model:** Claude Opus 4.8 (model ID `claude-opus-4.8`) +- **Generated:** 2026-07-01 16:03 (UTC+02:00) +- **Generation parameters:** managed by the GitHub Copilot CLI and not exposed to the + assistant (no explicit temperature or max-token values were set by the author). +- **Context:** Authored alongside `src/utils/create_bandwidth_check_clips.py`, based on the + script's implementation, the legacy `speech_impairment_utility.py` design, and the + `comb_bw*` usage in `master_script.py` and `P808Template/Qualification.html`. +- **Regeneration prompt:** "Write `docs/bandwidth_check_clips.md` documenting + `src/utils/create_bandwidth_check_clips.py`: explain the bandwidth-check qualification clips + (`A | beep | B`), the five cases and their `dq`/`sq` answers, the band-limited noise + (3.5/8.5/15–22 kHz at +13 dB over active speech level), the same-source halves with inaudible + dither, the UUID-anonymized names, the CLI arguments, the manifest CSV columns, and how to + wire `q1`…`q5` to `comb_bw1`…`comb_bw5`. Follow the repo doc style (breadcrumb header, + argument table) and include the AI-generated disclaimer banner, hidden LLM watermark, and this + reproducibility section." diff --git a/docs/preparation.md b/docs/preparation.md index 604c513..4c9f516 100644 --- a/docs/preparation.md +++ b/docs/preparation.md @@ -35,4 +35,5 @@ URLs associated to them as described in [General Resources](general_res.md) ## Utility Scripts - [Gold Standard Clips](gold_clips.md) — Generate gold clips for quality control. +- [Bandwidth Check Clips](bandwidth_check_clips.md) — Generate the bandwidth-discrimination clips for the P.80x qualification test. - [Upload Clips to Storage](upload_clips.md) — Upload local clips or copy from private to public Azure storage. diff --git a/src/utils/create_bandwidth_check_clips.py b/src/utils/create_bandwidth_check_clips.py new file mode 100644 index 0000000..7a74137 --- /dev/null +++ b/src/utils/create_bandwidth_check_clips.py @@ -0,0 +1,471 @@ +""" +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +Generate the bandwidth-check ("same vs. different quality") clips used in the +P.80x qualification test. + +For every clean reference speech file in an input directory this script builds +five combined clips. Each combined clip is two speech segments joined by a +short beep tone (``segment A | beep | segment B``); the listener is asked +whether the audio *after* the beep sounds different from the audio *before* it. + +The five cases per reference are (both halves always come from the *same* +source clip): + + q1 reference | beep | reference + noise(3.5-22 kHz) -> different (dq) + q2 reference | beep | reference + noise(8.5-22 kHz) -> different (dq) + q3 reference | beep | reference + noise(15-22 kHz) -> different (dq) + q4 reference | beep | reference -> same (sq) + q5 reference | beep | reference -> same (sq) + +The added noise is band-limited to a high-frequency band and made clearly +audible *within that band*. A participant only hears the difference if their +playback chain and hearing reproduce that band, so q1 (widest band) is the +obvious/attention case while q3 (15-22 kHz) discriminates full-band capable +equipment. q4 and q5 carry no audible change and are the obvious "same" +(trapping) cases. These design answers match the hosted production clips +(``d_g1_cmb.wav`` .. ``d_g5_cmb.wav``) referenced by ``master_script`` and +``P808Template/Qualification.html`` (dq, dq, dq, sq, sq). + +Every segment additionally receives an independent *inaudible* dither (far +below the speech, but above the 16-bit LSB). This keeps the two halves - and +the two "same" clips - from ever being bit-identical, so exact-match / dedup +detection cannot flag them, while humans still perceive them as identical. + +Output clip names are anonymized random UUIDs so the hosted file names do not +reveal the source reference; the manifest CSV keeps the source-to-clip mapping. + +The references must be full-band (48 kHz recommended); a reference sampled below +twice the lowest band edge cannot carry the high-frequency noise and is +skipped. + +Usage: + python utils/create_bandwidth_check_clips.py ^ + --input_dir C:/datasets/p501 ^ + --output_dir output/bw_test ^ + --base_url https://host/container/clips/bw-test +""" + +import argparse +import csv +import os +import uuid + +import numpy as np +import soundfile as sf +from scipy.signal import butter, sosfilt + + +# --------------------------------------------------------------------------- +# Design constants +# --------------------------------------------------------------------------- + +# Low edge of the band-limited noise for q1/q2/q3 (high edge is the reference +# bandwidth, clamped below Nyquist). Lower edges are progressively higher so the +# added noise sits in an ever-higher band that needs better equipment to hear. +NOISE_BANDS_HZ = [ + (3500, 22000), # q1: WB+ band, audible on most decent equipment + (8500, 22000), # q2: SWB band + (15000, 22000), # q3: FB band, only high-end equipment reproduces it +] + +# Correct answer for each of the five cases ("dq" = different, "sq" = same). +CASE_ANSWERS = ["dq", "dq", "dq", "sq", "sq"] + +# Band-limited noise level, in dB relative to the reference active speech level. +# The production clips use roughly +13 dB, which keeps the noise clearly audible +# within its band while the reference stays undistorted. +DEFAULT_NOISE_GAIN_DB = 13.0 + +# Level of the inaudible dither added to every segment, in dBov. It is well below +# the speech (so humans cannot hear it) yet several 16-bit LSBs high, so it +# survives quantization and keeps the two halves - and the two "same" clips - +# from ever being bit-identical (defeating exact-match / dedup detection). +INAUDIBLE_DITHER_DBOV = -75.0 + +FULL_SCALE_SINE_RMS = 1.0 / np.sqrt(2.0) + + +# --------------------------------------------------------------------------- +# Level helpers +# --------------------------------------------------------------------------- + +def rms(signal): + """ + Compute the root-mean-square level of a signal. + + :param signal: Audio signal as a numpy array. + :return: RMS value as a float (with a small epsilon floor). + """ + return float(np.sqrt(np.mean(np.square(signal)) + 1e-12)) + + +def active_speech_level_dbov(signal, sr, frame_ms=20.0, threshold_db=25.0): + """ + Estimate the active speech level of a signal in dBov. + + The level is measured over the active speech part only: the signal is split + into short frames and frames whose RMS is within ``threshold_db`` of the + loudest frame are treated as active (speech) frames. ``0 dBov`` corresponds + to a full-scale sine wave (RMS = 1/sqrt(2)). + + :param signal: Audio signal as a numpy array in the range [-1, 1]. + :param sr: Sample rate in Hz. + :param frame_ms: Frame length in milliseconds. + :param threshold_db: Frames within this many dB below the loudest frame count as active. + :return: Active speech level in dBov. + """ + eps = 1e-12 + frame_len = max(1, int(sr * frame_ms / 1000.0)) + n_frames = len(signal) // frame_len + if n_frames == 0: + active_rms = np.sqrt(np.mean(signal ** 2) + eps) + return 20.0 * np.log10(active_rms / FULL_SCALE_SINE_RMS + eps) + frames = signal[:n_frames * frame_len].reshape(n_frames, frame_len) + frame_rms = np.sqrt(np.mean(frames ** 2, axis=1) + eps) + peak_rms = frame_rms.max() + threshold = peak_rms / (10.0 ** (threshold_db / 20.0)) + active = frame_rms >= threshold + if not np.any(active): + active = np.ones(n_frames, dtype=bool) + active_rms = np.sqrt(np.mean(frames[active] ** 2) + eps) + return 20.0 * np.log10(active_rms / FULL_SCALE_SINE_RMS + eps) + + +def scale_to_dbov(signal, target_dbov): + """ + Scale a stationary signal so its overall RMS reaches a target level in dBov. + + :param signal: Audio signal as a numpy array. + :param target_dbov: Target RMS level in dBov (0 dBov = full-scale sine). + :return: Level-scaled signal as a numpy array. + """ + target_rms = FULL_SCALE_SINE_RMS * (10.0 ** (target_dbov / 20.0)) + return signal * (target_rms / rms(signal)) + + +# --------------------------------------------------------------------------- +# Signal building blocks +# --------------------------------------------------------------------------- + +def bandpass(signal, low_hz, high_hz, fs, order=5): + """ + Band-pass filter a signal, clamping the band edges to the valid range. + + The high edge is clamped just below Nyquist and the low edge is kept above + zero so the filter stays stable for any input sample rate. + + :param signal: Audio signal as a numpy array. + :param low_hz: Lower cut-off frequency in Hz. + :param high_hz: Upper cut-off frequency in Hz. + :param fs: Sample rate in Hz. + :param order: Butterworth filter order (default: 5). + :return: Band-pass filtered signal as a numpy array. + """ + nyq = 0.5 * fs + low = max(1.0, low_hz) / nyq + high = min(high_hz, 0.99 * nyq) / nyq + sos = butter(order, [low, high], btype="bandpass", output="sos") + return sosfilt(sos, signal) + + +def band_limited_noise(num_samples, low_hz, high_hz, fs, target_dbov, + order=5, rng=None): + """ + Create band-limited white noise scaled to a target level. + + White Gaussian noise is band-pass filtered to ``[low_hz, high_hz]`` and then + scaled so its RMS reaches ``target_dbov``. + + :param num_samples: Length of the noise in samples. + :param low_hz: Lower cut-off frequency in Hz. + :param high_hz: Upper cut-off frequency in Hz. + :param fs: Sample rate in Hz. + :param target_dbov: Target RMS level of the band-limited noise in dBov. + :param order: Butterworth filter order (default: 5). + :param rng: Optional numpy random generator for reproducibility. + :return: Band-limited noise as a numpy array. + """ + if rng is None: + rng = np.random.default_rng() + noise = rng.standard_normal(num_samples) + noise = bandpass(noise, low_hz, high_hz, fs, order=order) + return scale_to_dbov(noise, target_dbov) + + +def add_inaudible_dither(signal, rng, level_dbov=INAUDIBLE_DITHER_DBOV): + """ + Add inaudible low-level white noise to a segment. + + The noise sits far below the speech so it cannot be heard, but it is high + enough to survive 16-bit quantization. A fresh realization is drawn on every + call so no two segments end up bit-identical, which defeats exact-match and + dedup detection of the "same" trapping clips without affecting how the audio + sounds. + + :param signal: Audio signal as a numpy array. + :param rng: Numpy random generator used to draw the noise. + :param level_dbov: Noise RMS level in dBov (default: ``INAUDIBLE_DITHER_DBOV``). + :return: Signal with the inaudible dither added. + """ + noise = scale_to_dbov(rng.standard_normal(len(signal)), level_dbov) + return signal + noise + + +def make_beep(fs, freq_hz=440.0, duration_sec=1.0, amplitude=0.2, fade_ms=10.0): + """ + Create a short beep tone with fade-in/out to avoid clicks. + + :param fs: Sample rate in Hz. + :param freq_hz: Tone frequency in Hz (default: 440). + :param duration_sec: Tone duration in seconds (default: 1.0). + :param amplitude: Peak amplitude in [0, 1] (default: 0.2). + :param fade_ms: Fade-in/out length in milliseconds (default: 10). + :return: Beep tone as a numpy array. + """ + n = int(duration_sec * fs) + t = np.arange(n) / fs + tone = amplitude * np.sin(2.0 * np.pi * freq_hz * t) + fade = min(int(fade_ms / 1000.0 * fs), n // 2) + if fade > 0: + ramp = np.linspace(0.0, 1.0, fade) + tone[:fade] *= ramp + tone[-fade:] *= ramp[::-1] + return tone + + +def assemble_pair(part_a, part_b, beep, gap): + """ + Join two speech segments with a beep separator and silence gaps. + + The result is ``part_a | gap | beep | gap | part_b``. + + :param part_a: First speech segment as a numpy array. + :param part_b: Second speech segment as a numpy array. + :param beep: Beep tone as a numpy array. + :param gap: Silence gap as a numpy array (placed on both sides of the beep). + :return: Concatenated clip as a numpy array, hard-clipped to [-1, 1]. + """ + clip = np.concatenate([part_a, gap, beep, gap, part_b]) + return np.clip(clip, -1.0, 1.0) + + +# --------------------------------------------------------------------------- +# Per-reference generation +# --------------------------------------------------------------------------- + +def load_mono(path): + """ + Read a WAV file as a mono float signal. + + Multi-channel files are down-mixed to mono by averaging the channels. + + :param path: Path to the WAV file. + :return: Tuple of (mono signal as a numpy array, sample rate in Hz, subtype + string such as ``"PCM_16"``). + """ + signal, fs = sf.read(path) + if signal.ndim > 1: + signal = signal.mean(axis=1) + return signal.astype(np.float64), fs, sf.info(path).subtype + + +def generate_for_reference(ref_path, output_dir, noise_gain_db=DEFAULT_NOISE_GAIN_DB, + beep_freq=440.0, beep_sec=1.0, gap_sec=0.2, seed=None): + """ + Build the five bandwidth-check clips for a single reference file. + + Both halves of every clip come from the *same* reference source. q1-q3 add + audible band-limited noise to the second half (answer "different"); q4 and q5 + add no audible change and are the "same" trapping cases. To keep the two + halves - and the two "same" clips - from being bit-identical, every segment + also receives an independent inaudible dither, so exact-match / dedup + detection cannot flag them while humans still hear "same". + + Output file names are anonymized random UUIDs so the hosted clip names do not + reveal the source reference; the manifest CSV keeps the mapping. + + :param ref_path: Path to the clean reference WAV file. + :param output_dir: Directory where the five output clips are written. + :param noise_gain_db: Band-noise level relative to the reference active + speech level, in dB (default: +13). + :param beep_freq: Beep tone frequency in Hz (default: 440). + :param beep_sec: Beep tone duration in seconds (default: 1.0). + :param gap_sec: Silence gap on each side of the beep, in seconds (default: 0.2). + :param seed: Optional integer seed for reproducible noise. + :return: List of the five output clip file names (anonymized basenames), + ordered q1..q5, or ``None`` if the reference sample rate is too low for + the noise bands. + """ + ref, fs, subtype = load_mono(ref_path) + nyq = 0.5 * fs + if nyq <= min(low for low, _ in NOISE_BANDS_HZ): + print(f" Skipping {os.path.basename(ref_path)}: sample rate {fs} Hz too " + f"low for the bandwidth bands.") + return None + + rng = np.random.default_rng(seed) + ref_asl = active_speech_level_dbov(ref, fs) + noise_dbov = ref_asl + noise_gain_db + + beep = make_beep(fs, freq_hz=beep_freq, duration_sec=beep_sec) + gap = np.zeros(int(gap_sec * fs)) + + filenames = [] + + for i in range(5): + # First half: the reference with only inaudible dither. + part_a = add_inaudible_dither(ref, rng) + if i < len(NOISE_BANDS_HZ): + # q1-q3: high-frequency band-limited noise on the second half. + low_hz, high_hz = NOISE_BANDS_HZ[i] + noise = band_limited_noise(len(ref), low_hz, high_hz, fs, + noise_dbov, rng=rng) + part_b = add_inaudible_dither(ref + noise, rng) + else: + # q4/q5: same source, no audible change (only inaudible dither). + part_b = add_inaudible_dither(ref, rng) + + clip = assemble_pair(part_a, part_b, beep, gap) + out_name = f"{uuid.uuid4().hex}.wav" + sf.write(os.path.join(output_dir, out_name), clip, fs, subtype=subtype) + filenames.append(out_name) + + return filenames + + +def generate_bandwidth_check_clips(input_dir, output_dir, base_url=None, + noise_gain_db=DEFAULT_NOISE_GAIN_DB, + beep_freq=440.0, beep_sec=1.0, gap_sec=0.2, + seed=None, limit=None): + """ + Generate bandwidth-check clips for every reference WAV in a directory. + + For each reference, five clips (q1..q5) are written to ``output_dir`` and a + ``bandwidth_check_clips.csv`` manifest is produced with one row per + reference. The manifest columns are ``ref_clip``, ``q1``..``q5`` (output + clip name, or full URL when ``base_url`` is given) and ``ans_q1``..``ans_q5`` + (the correct answer ``dq``/``sq`` for each case). + + :param input_dir: Directory containing clean reference WAV files. + :param output_dir: Directory for the generated clips and the manifest CSV. + :param base_url: Optional base URL; when set the q1..q5 columns hold full + URLs (``base_url`` + file name) instead of bare file names. + :param noise_gain_db: Band-noise level relative to the reference active + speech level, in dB (default: +13). + :param beep_freq: Beep tone frequency in Hz (default: 440). + :param beep_sec: Beep tone duration in seconds (default: 1.0). + :param gap_sec: Silence gap on each side of the beep, in seconds (default: 0.2). + :param seed: Optional integer seed for reproducible noise. + :param limit: Optional cap on the number of references processed. + :return: Path to the generated manifest CSV. + """ + os.makedirs(output_dir, exist_ok=True) + references = sorted(f for f in os.listdir(input_dir) + if f.lower().endswith(".wav")) + if limit is not None: + references = references[:limit] + assert references, f"No .wav reference files found in {input_dir}" + + manifest = [] + for idx, ref_name in enumerate(references): + ref_path = os.path.join(input_dir, ref_name) + print(f"[{idx + 1}/{len(references)}] {ref_name}") + # Derive a per-reference seed so runs are reproducible yet references differ. + ref_seed = None if seed is None else seed + idx + filenames = generate_for_reference( + ref_path, output_dir, noise_gain_db=noise_gain_db, + beep_freq=beep_freq, beep_sec=beep_sec, gap_sec=gap_sec, seed=ref_seed, + ) + if filenames is None: + continue + + row = {"ref_clip": ref_name} + for i, name in enumerate(filenames): + value = base_url.rstrip("/") + "/" + name if base_url else name + row[f"q{i + 1}"] = value + row[f"ans_q{i + 1}"] = CASE_ANSWERS[i] + manifest.append(row) + + columns = ["ref_clip"] + for i in range(1, 6): + columns.append(f"q{i}") + for i in range(1, 6): + columns.append(f"ans_q{i}") + + manifest_path = os.path.join(output_dir, "bandwidth_check_clips.csv") + with open(manifest_path, "w", newline="") as f: + writer = csv.DictWriter(f, fieldnames=columns) + writer.writeheader() + writer.writerows(manifest) + + print(f"\nGenerated {len(manifest)} reference set(s) " + f"({len(manifest) * 5} clips) in {output_dir}") + print(f"Manifest saved to: {manifest_path}") + return manifest_path + + +# --------------------------------------------------------------------------- +# CLI entry point +# --------------------------------------------------------------------------- + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Generate the five bandwidth-check clips per reference for " + "the P.80x qualification test." + ) + parser.add_argument( + "--input_dir", "-i", required=True, + help="Directory containing clean, full-band reference WAV files." + ) + parser.add_argument( + "--output_dir", "-o", required=True, + help="Directory for the generated clips and manifest CSV." + ) + parser.add_argument( + "--base_url", default=None, + help="Base URL where clips will be hosted. When set, the q1..q5 CSV " + "columns hold full URLs instead of file names." + ) + parser.add_argument( + "--noise_gain_db", type=float, default=DEFAULT_NOISE_GAIN_DB, + help="Band-noise level relative to the reference active speech level, " + f"in dB (default: {DEFAULT_NOISE_GAIN_DB})." + ) + parser.add_argument( + "--beep_freq", type=float, default=440.0, + help="Beep tone frequency in Hz (default: 440)." + ) + parser.add_argument( + "--beep_sec", type=float, default=1.0, + help="Beep tone duration in seconds (default: 1.0)." + ) + parser.add_argument( + "--gap_sec", type=float, default=0.2, + help="Silence gap on each side of the beep, in seconds (default: 0.2)." + ) + parser.add_argument( + "--seed", type=int, default=None, + help="Optional integer seed for reproducible noise." + ) + parser.add_argument( + "--limit", type=int, default=None, + help="Optional cap on the number of references processed." + ) + + args = parser.parse_args() + + generate_bandwidth_check_clips( + input_dir=args.input_dir, + output_dir=args.output_dir, + base_url=args.base_url, + noise_gain_db=args.noise_gain_db, + beep_freq=args.beep_freq, + beep_sec=args.beep_sec, + gap_sec=args.gap_sec, + seed=args.seed, + limit=args.limit, + ) From 566f9a3a143018723f04aa7d1b5a2d5dc0ced6c1 Mon Sep 17 00:00:00 2001 From: Babak Naderi Date: Thu, 2 Jul 2026 13:04:06 +0200 Subject: [PATCH 013/111] Add --no_anonymize option to bandwidth-check generator Add a --no_anonymize flag to create_bandwidth_check_clips.py to emit descriptive _q{n}.wav names for listening review; the default remains random UUID names. Document the option in docs/bandwidth_check_clips.md. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/bandwidth_check_clips.md | 6 +++-- src/utils/create_bandwidth_check_clips.py | 28 +++++++++++++++++------ 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/docs/bandwidth_check_clips.md b/docs/bandwidth_check_clips.md index d814e95..5598dfd 100644 --- a/docs/bandwidth_check_clips.md +++ b/docs/bandwidth_check_clips.md @@ -76,12 +76,14 @@ python utils/create_bandwidth_check_clips.py ^ | `--gap_sec` | No | 0.2 | Silence gap on each side of the beep, in seconds. | | `--seed` | No | — | Integer seed for reproducible noise. | | `--limit` | No | — | Cap on the number of references processed. | +| `--no_anonymize` | No | False | Use descriptive `_q{n}.wav` names instead of random UUIDs (useful for listening review). | ### Output The script writes, to `--output_dir`: -- Five WAV clips per reference, named with random UUIDs (source sample rate and subtype preserved). +- Five WAV clips per reference, named with random UUIDs (source sample rate and subtype + preserved). Pass `--no_anonymize` for descriptive `_q{n}.wav` names when reviewing. - `bandwidth_check_clips.csv`, one row per reference, with columns `ref_clip`, `q1`…`q5` (clip file name, or full URL when `--base_url` is given) and `ans_q1`…`ans_q5` (the correct answer `dq`/`sq` for each case). @@ -112,7 +114,7 @@ their correct answers. To use freshly generated clips: ## Reproducibility - **Model:** Claude Opus 4.8 (model ID `claude-opus-4.8`) -- **Generated:** 2026-07-01 16:03 (UTC+02:00) +- **Generated:** 2026-07-01, updated 2026-07-02 (UTC+02:00) - **Generation parameters:** managed by the GitHub Copilot CLI and not exposed to the assistant (no explicit temperature or max-token values were set by the author). - **Context:** Authored alongside `src/utils/create_bandwidth_check_clips.py`, based on the diff --git a/src/utils/create_bandwidth_check_clips.py b/src/utils/create_bandwidth_check_clips.py index 7a74137..4cd9bd9 100644 --- a/src/utils/create_bandwidth_check_clips.py +++ b/src/utils/create_bandwidth_check_clips.py @@ -274,7 +274,8 @@ def load_mono(path): def generate_for_reference(ref_path, output_dir, noise_gain_db=DEFAULT_NOISE_GAIN_DB, - beep_freq=440.0, beep_sec=1.0, gap_sec=0.2, seed=None): + beep_freq=440.0, beep_sec=1.0, gap_sec=0.2, seed=None, + anonymize=True): """ Build the five bandwidth-check clips for a single reference file. @@ -286,7 +287,9 @@ def generate_for_reference(ref_path, output_dir, noise_gain_db=DEFAULT_NOISE_GAI detection cannot flag them while humans still hear "same". Output file names are anonymized random UUIDs so the hosted clip names do not - reveal the source reference; the manifest CSV keeps the mapping. + reveal the source reference; the manifest CSV keeps the mapping. Set + ``anonymize`` to False to use descriptive ``_q{n}.wav`` names for + listening review. :param ref_path: Path to the clean reference WAV file. :param output_dir: Directory where the five output clips are written. @@ -296,9 +299,10 @@ def generate_for_reference(ref_path, output_dir, noise_gain_db=DEFAULT_NOISE_GAI :param beep_sec: Beep tone duration in seconds (default: 1.0). :param gap_sec: Silence gap on each side of the beep, in seconds (default: 0.2). :param seed: Optional integer seed for reproducible noise. - :return: List of the five output clip file names (anonymized basenames), - ordered q1..q5, or ``None`` if the reference sample rate is too low for - the noise bands. + :param anonymize: When True (default) use random UUID file names; when False + use descriptive ``_q{n}.wav`` names. + :return: List of the five output clip file names (basenames), ordered q1..q5, + or ``None`` if the reference sample rate is too low for the noise bands. """ ref, fs, subtype = load_mono(ref_path) nyq = 0.5 * fs @@ -314,6 +318,7 @@ def generate_for_reference(ref_path, output_dir, noise_gain_db=DEFAULT_NOISE_GAI beep = make_beep(fs, freq_hz=beep_freq, duration_sec=beep_sec) gap = np.zeros(int(gap_sec * fs)) + stem = os.path.splitext(os.path.basename(ref_path))[0] filenames = [] for i in range(5): @@ -330,7 +335,7 @@ def generate_for_reference(ref_path, output_dir, noise_gain_db=DEFAULT_NOISE_GAI part_b = add_inaudible_dither(ref, rng) clip = assemble_pair(part_a, part_b, beep, gap) - out_name = f"{uuid.uuid4().hex}.wav" + out_name = f"{uuid.uuid4().hex}.wav" if anonymize else f"{stem}_q{i + 1}.wav" sf.write(os.path.join(output_dir, out_name), clip, fs, subtype=subtype) filenames.append(out_name) @@ -340,7 +345,7 @@ def generate_for_reference(ref_path, output_dir, noise_gain_db=DEFAULT_NOISE_GAI def generate_bandwidth_check_clips(input_dir, output_dir, base_url=None, noise_gain_db=DEFAULT_NOISE_GAIN_DB, beep_freq=440.0, beep_sec=1.0, gap_sec=0.2, - seed=None, limit=None): + seed=None, limit=None, anonymize=True): """ Generate bandwidth-check clips for every reference WAV in a directory. @@ -361,6 +366,8 @@ def generate_bandwidth_check_clips(input_dir, output_dir, base_url=None, :param gap_sec: Silence gap on each side of the beep, in seconds (default: 0.2). :param seed: Optional integer seed for reproducible noise. :param limit: Optional cap on the number of references processed. + :param anonymize: When True (default) use random UUID clip names; when False + use descriptive ``_q{n}.wav`` names for listening review. :return: Path to the generated manifest CSV. """ os.makedirs(output_dir, exist_ok=True) @@ -379,6 +386,7 @@ def generate_bandwidth_check_clips(input_dir, output_dir, base_url=None, filenames = generate_for_reference( ref_path, output_dir, noise_gain_db=noise_gain_db, beep_freq=beep_freq, beep_sec=beep_sec, gap_sec=gap_sec, seed=ref_seed, + anonymize=anonymize, ) if filenames is None: continue @@ -455,6 +463,11 @@ def generate_bandwidth_check_clips(input_dir, output_dir, base_url=None, "--limit", type=int, default=None, help="Optional cap on the number of references processed." ) + parser.add_argument( + "--no_anonymize", action="store_true", + help="Use descriptive _q{n}.wav names instead of random UUIDs " + "(useful for listening review)." + ) args = parser.parse_args() @@ -468,4 +481,5 @@ def generate_bandwidth_check_clips(input_dir, output_dir, base_url=None, gap_sec=args.gap_sec, seed=args.seed, limit=args.limit, + anonymize=not args.no_anonymize, ) From d550a7a303349a53dc0df59a5e386cd1eba5c3ab Mon Sep 17 00:00:00 2001 From: Babak Naderi Date: Fri, 3 Jul 2026 13:50:19 +0200 Subject: [PATCH 014/111] Have study agents write a re-runnable .bat Update the create-study and analyze-results agent runbooks to always emit a batch file so the requester can re-run without re-deriving the command: - create-study writes regenerate_study.bat (re-runs master_script.py) - analyze-results writes rerun_result_parser.bat (re-runs result_parser.py) Both .bat files use %BASE% for input paths and an absolute repo path for the script. Update each agent's description accordingly and add the .bat to the create-study handoff checklist. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/agents/analyze-results.agent.md | 34 ++++++++++++++++- .github/agents/create-study.agent.md | 50 ++++++++++++++++++++++--- 2 files changed, 78 insertions(+), 6 deletions(-) diff --git a/.github/agents/analyze-results.agent.md b/.github/agents/analyze-results.agent.md index c993fa0..841a5bd 100644 --- a/.github/agents/analyze-results.agent.md +++ b/.github/agents/analyze-results.agent.md @@ -1,6 +1,6 @@ --- name: analyze-results -description: Analyzes crowdsourced subjective test results — runs result_parser.py for data cleaning, quality checks, and per-clip/per-worker MOS aggregation. +description: Analyzes crowdsourced subjective test results — runs result_parser.py for data cleaning, quality checks, and per-clip/per-worker MOS aggregation, and writes a re-runnable rerun_result_parser.bat that re-runs result_parser.py. --- # Analyze subjective test results @@ -109,6 +109,38 @@ python REPO_ROOT\src\result_parser.py ` - The working directory should be the project directory so output files are written there. +### 3b. Write a re-run batch file (rerun_result_parser.bat) + +Always write a `rerun_result_parser.bat` in the results directory so the requester can +re-run the analysis later (e.g. after a config change or with an updated answers export) +without re-deriving the command. It must reproduce the exact `result_parser.py` invocation +from step 3, using `%BASE%` (the folder the `.bat` lives in) for the input files and the +absolute repo path for the script. Include the `--prolific_answers` line only if Prolific +was used. + +```bat +@echo off +REM Re-run the result parser for PROJECT (METHOD). +REM Rebuilds the data-cleaning report and per-clip / per-worker MOS outputs from the +REM batch answers CSV using the existing result-parser config. + +setlocal +set "BASE=%~dp0" +set "REPO=REPO_ROOT" + +cd /d "%BASE%" + +python "%REPO%\src\result_parser.py" ^ + --cfg "%BASE%RESULT_PARSER_CFG" ^ + --method METHOD ^ + --answers "%BASE%Batch_XXX.csv" ^ + --prolific_answers "%BASE%prolific_demographic_export_XXX.csv" + +endlocal +``` + +Save as `rerun_result_parser.bat` in the results directory. + ### 4. Analyze the output and summarize After the parser completes, provide a summary covering: diff --git a/.github/agents/create-study.agent.md b/.github/agents/create-study.agent.md index e1810af..7a89acb 100644 --- a/.github/agents/create-study.agent.md +++ b/.github/agents/create-study.agent.md @@ -1,6 +1,6 @@ --- name: create-study -description: Creates subjective speech quality tests using the P.808 toolkit — handles study setup, gold/trapping clip generation, storage upload, and project building for crowdsourcing platforms. +description: Creates subjective speech quality tests using the P.808 toolkit — handles study setup, gold/trapping clip generation, storage upload, project building for crowdsourcing platforms, and writes a re-runnable regenerate_study.bat that re-runs master_script.py. --- # Create subjective test instructions @@ -653,6 +653,45 @@ needed — training clips are embedded in the training gold CSV. - If `quantity_hits_more_than` triggers a warning, update the config file with the suggested value and re-run. +### 8b. Write a re-run batch file (regenerate_study.bat) + +Always write a `regenerate_study.bat` next to the input CSVs so the requester can rebuild +the study later (e.g. after a config tweak) without re-deriving the command. It must +reproduce the exact `master_script.py` invocation from step 8, using `%BASE%` (the folder +the `.bat` lives in) for all input paths and the absolute repo path for the script. + +Include only the flags that were actually used: `--training_gold_clips` (P.804/pp835) or +`--training_clips` (other methods), and `--general_assets` if an internal assets CSV was +passed. Omit `--check_urls` (URLs were already validated on first run); keep +`--create_local_test` so a preview is regenerated. + +```bat +@echo off +REM Regenerate the METHOD study PROJECT_NAME from existing input CSVs. +REM Clips (rating, gold, trapping) are already uploaded to public storage; this +REM only rebuilds the HIT app, publish batch, result-parser cfg, and preview. + +setlocal +set "BASE=%~dp0" +set "REPO=REPO_ROOT" +set "PROJECT=PROJECT_NAME" + +cd /d "%BASE%" + +python "%REPO%\src\master_script.py" ^ + --project %PROJECT% ^ + --method METHOD ^ + --cfg "%BASE%PROJECT_CONFIG.cfg" ^ + --clips "%BASE%rating_clips.csv" ^ + --gold_clips "%BASE%gold_clips.csv" ^ + --trapping_clips "%BASE%trapping_clips.csv" ^ + --create_local_test + +endlocal +``` + +Save as `regenerate_study.bat` next to the input CSVs and confirm it runs. + ### 9. Verify the generated project artifacts The output project directory (`PROJECT_NAME\`) should contain: @@ -703,10 +742,11 @@ remind the requester to run the azcopy commands before publishing the study. 1. The project directory with all three artifacts. 2. The config file used (saved next to input CSVs for future re-runs). -3. The azcopy commands for uploading generated clips (if applicable). -4. The method and scale used. -5. Any warnings or deviations from the documented flow. -6. Instructions for the requester to publish on their chosen platform: +3. The `regenerate_study.bat` (next to the input CSVs) for one-command re-runs. +4. The azcopy commands for uploading generated clips (if applicable). +5. The method and scale used. +6. Any warnings or deviations from the documented flow. +7. Instructions for the requester to publish on their chosen platform: - **Prolific**: follow the team's Prolific workflow or `docs\running_test_prolific.md`. - **AMT**: follow `docs\running_test_mturk.md`. From d94c626a55a871acfafebfbdc2deac5d485238a4 Mon Sep 17 00:00:00 2001 From: Babak Naderi Date: Fri, 3 Jul 2026 15:01:50 +0200 Subject: [PATCH 015/111] Add --payment_per_session to result_parser and wire it into the agent result_parser.py: - Add optional --payment_per_session (reward per session/HIT, e.g. 2.10) so payment-per-hour stats can be computed for Prolific studies, which do not include the reward in their export. Takes precedence over the pre-existing (now deprecated-alias) --rewards. - Guard calc_stats against missing Answer.* columns (Answer.2_birth_year, Answer.Math, Answer.t1_ovrl, Answer.t1) so it no longer raises KeyError on Prolific batches that omit in-form demographics/sections. analyze-results agent: collect payment_per_session when the study was run on Prolific and pass --payment_per_session in the run command and the generated rerun_result_parser.bat. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/agents/analyze-results.agent.md | 21 +++++++++++++++++---- src/result_parser.py | 21 ++++++++++++++++++--- 2 files changed, 35 insertions(+), 7 deletions(-) diff --git a/.github/agents/analyze-results.agent.md b/.github/agents/analyze-results.agent.md index 841a5bd..cbcaed7 100644 --- a/.github/agents/analyze-results.agent.md +++ b/.github/agents/analyze-results.agent.md @@ -45,6 +45,12 @@ Do not guess these values if they are missing: (AMT) or HIT App server. Contains worker responses. 4. **Prolific demographic CSV** (optional): `prolific_demographic_export_*.csv` — only needed if the study was run on Prolific via HIT App server. +5. **Payment per session** (Prolific only): the reward paid to a participant per + session/HIT (e.g. `2.10`). Prolific does not include the reward in its export, + so the parser cannot compute payment-per-hour statistics without it. Ask the + user for this value whenever the study was run on Prolific; pass it to the + parser as `--payment_per_session`. Not needed for AMT (the reward is already a + column in the AMT batch). ## Execution workflow @@ -54,6 +60,8 @@ Do not guess these values if they are missing: - The path to the project directory (where the `*_result_parser.cfg` is). - The test method used. - Whether they used Prolific or AMT. +- **If Prolific**: the payment per session (reward per HIT, e.g. `2.10`) — this is + not in the Prolific export and is required for payment-per-hour statistics. Then instruct: "Please download the answers file (`Batch_XXX.csv`) and, if using Prolific, the demographic export (`prolific_demographic_export_*.csv`) and place @@ -94,13 +102,17 @@ python REPO_ROOT\src\result_parser.py ` **With Prolific demographic data:** +Also pass `--payment_per_session` (the reward per HIT, e.g. `2.10`) so payment-per-hour +statistics are computed — Prolific does not include the reward in its export. + ```powershell Set-Location PROJECT_DIR python REPO_ROOT\src\result_parser.py ` --cfg RESULT_PARSER_CFG ` --method METHOD ` --answers Batch_XXX.csv ` - --prolific_answers prolific_demographic_export_XXX.csv + --prolific_answers prolific_demographic_export_XXX.csv ` + --payment_per_session 2.10 ``` **Notes:** @@ -115,8 +127,8 @@ Always write a `rerun_result_parser.bat` in the results directory so the request re-run the analysis later (e.g. after a config change or with an updated answers export) without re-deriving the command. It must reproduce the exact `result_parser.py` invocation from step 3, using `%BASE%` (the folder the `.bat` lives in) for the input files and the -absolute repo path for the script. Include the `--prolific_answers` line only if Prolific -was used. +absolute repo path for the script. Include the `--prolific_answers` and +`--payment_per_session` lines only if Prolific was used. ```bat @echo off @@ -134,7 +146,8 @@ python "%REPO%\src\result_parser.py" ^ --cfg "%BASE%RESULT_PARSER_CFG" ^ --method METHOD ^ --answers "%BASE%Batch_XXX.csv" ^ - --prolific_answers "%BASE%prolific_demographic_export_XXX.csv" + --prolific_answers "%BASE%prolific_demographic_export_XXX.csv" ^ + --payment_per_session 2.10 endlocal ``` diff --git a/src/result_parser.py b/src/result_parser.py index d2b6bc3..869cad8 100644 --- a/src/result_parser.py +++ b/src/result_parser.py @@ -1677,8 +1677,12 @@ def calc_payment_stat(df): # return 0 if Reward not if column if 'Reward' not in df.columns: - if args.rewards is not None: - df['Reward'] = "$"+args.rewards + # Prolific does not include the reward in its export; use the per-session + # payment provided on the command line (--payment_per_session, or the + # deprecated --rewards alias) when available. + payment_per_session = getattr(args, 'payment_per_session', None) or args.rewards + if payment_per_session is not None: + df['Reward'] = "$" + str(payment_per_session) else: df['Reward'] = '$0.00' # from Prolific we doing get the rewards in the csv file word_duration_col = "work_duration_sec" if 'WorkTimeInSeconds' not in df.columns else 'WorkTimeInSeconds' @@ -1712,6 +1716,13 @@ def calc_stats(input_file): """ df = pd.read_csv(input_file, low_memory=False) + # Some studies (e.g. run on Prolific) do not collect in-form demographics or + # every setup section, so these answer columns may be absent from the batch. + # Treat missing columns as all-NaN so the payment breakdown degrades + # gracefully instead of raising a KeyError. + for col in ['Answer.2_birth_year', 'Answer.Math', 'Answer.t1_ovrl', 'Answer.t1']: + if col not in df.columns: + df[col] = np.nan df_full = df.copy() overall_time, overall_pay = calc_payment_stat(df) @@ -2146,7 +2157,11 @@ def analyze_results(config, test_method, answer_path,prolific_ans_path, list_of_ parser.add_argument('--quality_bonus', help="Quality bonus will be calculated. Just use it with your final download" " of answers and when the project is completed", action="store_true") - parser.add_argument('--rewards', help="If using Prolific the amount of rewards are not included in CSV file, specifiz it here like 2.10" + parser.add_argument('--rewards', help="Deprecated alias of --payment_per_session. If using Prolific the amount of rewards are not included in CSV file, specifiz it here like 2.10" + , required=False, default=None) + parser.add_argument('--payment_per_session', help="Payment (reward) paid to a participant per session/HIT, e.g. 2.10. " + "Prolific does not include the reward in its export, so provide it here to " + "compute payment-per-hour statistics. Optional; takes precedence over --rewards." , required=False, default=None) #parser.add_argument('--adc' , help="name of Advance Data Cleaning script. If set, the answers will be filtered by that as well", default=None) From efb5f86b42c14aaa411faf6f41bcebf703ce5536 Mon Sep 17 00:00:00 2001 From: Babak Naderi Date: Fri, 3 Jul 2026 15:33:01 +0200 Subject: [PATCH 016/111] Add target platform to result-parser config; skip bonus report on Prolific result_parser.py: read an optional [general] platform (mturk/prolific) from the config; when absent, infer it (prolific if a Prolific export is passed, else mturk). Skip quantity/quality bonus report generation when the platform is Prolific, since bonuses are handled on Prolific rather than via this report. master_script.py + templates: render a platform entry into every generated *_result_parser.cfg, sourced from [create_input] platform (default prolific). Agents: create-study records platform in the study cfg; analyze-results notes the bonus report is MTurk-only. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/agents/analyze-results.agent.md | 2 +- .github/agents/create-study.agent.md | 5 ++ .../acr_result_parser_template.cfg | 4 +- .../dcr_ccr_result_parser_template.cfg | 4 +- .../p804_result_parser_template.cfg | 2 + .../pp835_result_parser_template.cfg | 2 + src/master_script.py | 6 +- src/result_parser.py | 58 ++++++++++++------- 8 files changed, 57 insertions(+), 26 deletions(-) diff --git a/.github/agents/analyze-results.agent.md b/.github/agents/analyze-results.agent.md index cbcaed7..67b2bd5 100644 --- a/.github/agents/analyze-results.agent.md +++ b/.github/agents/analyze-results.agent.md @@ -212,7 +212,7 @@ After analysis, direct the user to the key output files: | `Batch_XXX_all_votes_per_clip.csv` | All individual votes per clip (key: `all_votes` in name) | | `Batch_XXX_data_cleaning_report.csv` | Detailed per-submission data cleaning report | | `detailed_gold_question_performance.csv` | Per-gold-clip acceptance/rejection statistics | -| `Batch_XXX_quantity_bonus_report.csv` | Quantity bonus calculations | +| `Batch_XXX_quantity_bonus_report.csv` | Quantity bonus calculations (MTurk only; not generated when the platform is Prolific) | **Scale suffixes by method:** diff --git a/.github/agents/create-study.agent.md b/.github/agents/create-study.agent.md index 7a89acb..e037b66 100644 --- a/.github/agents/create-study.agent.md +++ b/.github/agents/create-study.agent.md @@ -581,6 +581,7 @@ number_of_clips_per_session:10 number_of_trapping_per_session:1 number_of_gold_clips_per_session:GOLD_PER_SESSION clip_packing_strategy: random +platform: PLATFORM [hit_app_html] allowed_max_hit_in_project:COMPUTED_VALUE @@ -600,6 +601,10 @@ contact_email:USER_PROVIDED_EMAIL - `contact_email` = user-provided. Never hardcode. - `allowed_max_hit_in_project` = `BEST_PRACTICE_ALLOWED_MAX_HITS`. - `quantity_hits_more_than` ≈ `floor(total_sessions / 2)`, at least 2. +- `platform` = the crowd platform (`prolific` or `mturk`), from the requester's answer + (defaults to `prolific`). It is written into the generated `*_result_parser.cfg` so the + result parser knows the platform; for `prolific` no bonus report is generated (bonuses are + handled on Prolific). ### 8. Run the master script diff --git a/src/assets_master_script/acr_result_parser_template.cfg b/src/assets_master_script/acr_result_parser_template.cfg index c25abd3..835de00 100644 --- a/src/assets_master_script/acr_result_parser_template.cfg +++ b/src/assets_master_script/acr_result_parser_template.cfg @@ -1,6 +1,8 @@ # Configuration for ´result_parser.py´ script (acr method) [general] +# target crowdsourcing platform: "mturk" or "prolific". For prolific, bonus reports are not generated. +platform: {{cfg.platform}} number_of_questions_in_rating: {{cfg.q_num}} expected_votes_per_file: 10 # "condition_pattern" specifies a regex to extract the condition name from the file name. @@ -31,7 +33,7 @@ variance: 1 [acceptance_criteria] all_audio_played_equal: 1 -# bandwidth control range: "NB-WB", "SWB", "FB" +# bandwidth control range: "NB-WB", "SWB", "FB" bw_min: {{cfg.bw_min}}, bw_max: {{cfg.bw_max}}, # number of correct answers to the math questions should be bigger and equal to diff --git a/src/assets_master_script/dcr_ccr_result_parser_template.cfg b/src/assets_master_script/dcr_ccr_result_parser_template.cfg index bade351..b81558f 100644 --- a/src/assets_master_script/dcr_ccr_result_parser_template.cfg +++ b/src/assets_master_script/dcr_ccr_result_parser_template.cfg @@ -1,6 +1,8 @@ # Configuration for ´result_parser.py´ script (ccr/dcr method) [general] +# target crowdsourcing platform: "mturk" or "prolific". For prolific, bonus reports are not generated. +platform: {{cfg.platform}} number_of_questions_in_rating: {{cfg.q_num}} expected_votes_per_file: 5 # "condition_pattern" specifies a regex to extract the condition name from the file name. @@ -23,7 +25,7 @@ url_found_in: input.tp [acceptance_criteria] all_audio_played_equal: 1 -# bandwidth control range: "NB-WB", "SWB", "FB" +# bandwidth control range: "NB-WB", "SWB", "FB" bw_min: {{cfg.bw_min}}, bw_max: {{cfg.bw_max}}, # number of correct answers to the math questions should be bigger and equal to diff --git a/src/assets_master_script/p804_result_parser_template.cfg b/src/assets_master_script/p804_result_parser_template.cfg index abfe103..e699b32 100644 --- a/src/assets_master_script/p804_result_parser_template.cfg +++ b/src/assets_master_script/p804_result_parser_template.cfg @@ -1,6 +1,8 @@ # Configuration for ´result_parser.py´ script (acr method) [general] +# target crowdsourcing platform: "mturk" or "prolific". For prolific, bonus reports are not generated. +platform: {{cfg.platform}} number_of_questions_in_rating: {{cfg.q_num}} expected_votes_per_file: 10 # "condition_pattern" specifies a regex to extract the condition name from the file name. diff --git a/src/assets_master_script/pp835_result_parser_template.cfg b/src/assets_master_script/pp835_result_parser_template.cfg index 21ae315..2fd44b9 100644 --- a/src/assets_master_script/pp835_result_parser_template.cfg +++ b/src/assets_master_script/pp835_result_parser_template.cfg @@ -1,6 +1,8 @@ # Configuration for ´result_parser.py´ script (acr method) [general] +# target crowdsourcing platform: "mturk" or "prolific". For prolific, bonus reports are not generated. +platform: {{cfg.platform}} number_of_questions_in_rating: {{cfg.q_num}} expected_votes_per_file: 10 # "condition_pattern" specifies a regex to extract the condition name from the file name. diff --git a/src/master_script.py b/src/master_script.py index dfb1494..ff10fd8 100644 --- a/src/master_script.py +++ b/src/master_script.py @@ -107,6 +107,8 @@ def create_analyzer_cfg_general(cfg, cfg_section, template_path, out_path, gener default_keys = 'condition_num' config['condition_pattern'] = cfg['create_input'].get("condition_pattern", default_condition) config['condition_keys'] = cfg['create_input'].get("condition_keys", default_keys) + # target crowdsourcing platform (mturk or prolific); Prolific skips bonus reports + config['platform'] = cfg['create_input'].get("platform", "prolific") # BW check config['bw_min'] = general_cfg['bw_min'] @@ -148,6 +150,8 @@ def create_analyzer_cfg_dcr_ccr(cfg, template_path, out_path, general_cfg, n_HIT default_keys = 'condition_num' config['condition_pattern'] = cfg['create_input'].get("condition_pattern", default_condition) config['condition_keys'] = cfg['create_input'].get("condition_keys", default_keys) + # target crowdsourcing platform (mturk or prolific); Prolific skips bonus reports + config['platform'] = cfg['create_input'].get("platform", "prolific") # BW check config['bw_min'] = general_cfg['bw_min'] @@ -194,7 +198,7 @@ async def create_hit_app_ccr_dcr(cfg, template_path, out_path, training_path, cf rating_urls = [] n_clips = int(cfg_g['number_of_clips_per_session']) n_traps = int(cfg_g['number_of_trapping_per_session']) - # 'dummy':'dummy' is added because of current bug in AMT for replacing variable names. See issue #6 + # 'dummy':'dummy' is added because of current bug in AMT for replacing variable names. See issue #6 for i in range(0, n_clips): rating_urls.append({"ref": f"${{Q{i}_R}}", "processed": f"${{Q{i}_P}}", 'dummy': 'dummy'}) diff --git a/src/result_parser.py b/src/result_parser.py index 869cad8..127c3a2 100644 --- a/src/result_parser.py +++ b/src/result_parser.py @@ -1907,7 +1907,7 @@ def combine_prolific_hit_server(prolific_ans_path, hitapp_ans_path): return merged_ans_path, not_in_hitapp -def analyze_results(config, test_method, answer_path,prolific_ans_path, list_of_req, quality_bonus): +def analyze_results(config, test_method, answer_path,prolific_ans_path, list_of_req, quality_bonus, platform="mturk"): """ main method for calculating the results :param config: @@ -1915,6 +1915,8 @@ def analyze_results(config, test_method, answer_path,prolific_ans_path, list_of_ :param answer_path: :param list_of_req: :param quality_bonus: + :param platform: target crowdsourcing platform ("mturk" or "prolific"). Bonus + reports are MTurk-specific and are skipped for "prolific". :return: """ global question_name_suffix @@ -2106,27 +2108,31 @@ def analyze_results(config, test_method, answer_path,prolific_ans_path, list_of_ merged_cond['M'] = ((merged_cond['MOS_SIG']-1) / 4 + (merged_cond['MOS_OVRL']-1) /4 ) / 2 merged_cond.to_csv(os.path.splitext(answer_path)[0]+ f"_votes_per_cond_all-scales.csv", index=False) - bonus_file = os.path.splitext(answer_path)[0] + '_quantity_bonus_report.csv' - quantity_bonus_df = calc_quantity_bonuses(full_data, list_of_req, bonus_file) + if platform == "prolific": + logger.info("Platform is Prolific; skipping bonus report generation " + "(bonuses are handled on Prolific, not via this report).") + else: + bonus_file = os.path.splitext(answer_path)[0] + '_quantity_bonus_report.csv' + quantity_bonus_df = calc_quantity_bonuses(full_data, list_of_req, bonus_file) - if quality_bonus: - quality_bonus_path = os.path.splitext(answer_path)[0] + '_quality_bonus_report.csv' - if 'all' not in list_of_req: - quantity_bonus_df = calc_quantity_bonuses(full_data, ['all'], None) - if use_condition_level: - votes_to_use = vote_per_condition - else: - votes_to_use = votes_per_file - calc_quality_bonuses( - quantity_bonus_df, - accepted_sessions, - votes_to_use, - config, - quality_bonus_path, - n_workers, - test_method, - use_condition_level, - ) + if quality_bonus: + quality_bonus_path = os.path.splitext(answer_path)[0] + '_quality_bonus_report.csv' + if 'all' not in list_of_req: + quantity_bonus_df = calc_quantity_bonuses(full_data, ['all'], None) + if use_condition_level: + votes_to_use = vote_per_condition + else: + votes_to_use = votes_per_file + calc_quality_bonuses( + quantity_bonus_df, + accepted_sessions, + votes_to_use, + config, + quality_bonus_path, + n_workers, + test_method, + use_condition_level, + ) all_votes_per_file_path = ( os.path.splitext(answer_path)[0] + f"_all_votes_per_clip.csv" @@ -2188,6 +2194,14 @@ def analyze_results(config, test_method, answer_path,prolific_ans_path, list_of_ warnings.warn("If you are using HIT App server, Note: the WorkerId, HITIds, ect. are internal " "HIT APP server ids. Therefore bonus reports cannot be used." ) + # Target crowdsourcing platform. An explicit [general] platform in the config wins; + # otherwise it is inferred (Prolific when a Prolific export is provided, MTurk otherwise). + # Bonus reports are MTurk-specific, so they are not generated for Prolific. + if config.has_option("general", "platform"): + platform = config["general"]["platform"].strip().lower() + else: + platform = "prolific" if prolific_ans_path is not None else "mturk" + assert os.path.exists(answer_path), f"No input file found in [{answer_path}]" list_of_possible_status = ['all', 'submitted'] @@ -2207,4 +2221,4 @@ def analyze_results(config, test_method, answer_path,prolific_ans_path, list_of_ logger.addHandler(file_handler) logger.info(f"Start analyzing the results of {test_method} test") # start - analyze_results(config, test_method, answer_path,prolific_ans_path, list_of_req, args.quality_bonus) \ No newline at end of file + analyze_results(config, test_method, answer_path, prolific_ans_path, list_of_req, args.quality_bonus, platform) \ No newline at end of file From 2549d2ac4b6a63967e7edb138ac8a4f93e29b0b5 Mon Sep 17 00:00:00 2001 From: Babak Naderi Date: Fri, 3 Jul 2026 15:43:01 +0200 Subject: [PATCH 017/111] Keep Prolific submissions that match a completed HIT App assignment The Prolific/HIT App merge dropped Prolific rows with an empty study URL and no 32-char completion code (abandoned/returned submissions). This also dropped 'TIMED-OUT' submissions where the worker actually completed the task on the HIT App server, so their HIT App row found no match and was wrongly listed in *_not_found_in_prolific.csv. Now a Prolific row is kept when its submission id matches a completed HIT App assignment (browser_info present), and the HIT App assignment id is stripped as well as lower-cased so matching is whitespace-insensitive on both sides. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/result_parser.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/result_parser.py b/src/result_parser.py index 127c3a2..a88bc70 100644 --- a/src/result_parser.py +++ b/src/result_parser.py @@ -1817,7 +1817,7 @@ def combine_prolific_hit_server(prolific_ans_path, hitapp_ans_path): hitapp_ans["hitapp_workerid"] = hitapp_ans["WorkerId"] - hitapp_ans["hitapp_assignmentid"] = hitapp_ans["AssignmentId"].str.lower() + hitapp_ans["hitapp_assignmentid"] = hitapp_ans["AssignmentId"].str.strip().str.lower() hitapp_ans["hitapp_hitid"] = hitapp_ans["HITId"] hitapp_ans["hitapp_hittypeid"] = hitapp_ans["HITTypeId"] hitapp_ans["HITTypeId"] = hitapp_ans["Answer.studyId"] @@ -1840,8 +1840,17 @@ def combine_prolific_hit_server(prolific_ans_path, hitapp_ans_path): 'Total approvals':'prolific_total_approvals', 'URL':'study_url'}, inplace=True) - # marke prolific_ans to remove when study_url is nan and lenght of Answer.v_code is not 32 - prolific_ans['to_remove'] = prolific_ans['study_url'].isna() & (prolific_ans['Answer.v_code'].str.strip().str.len() != 32) + # mark prolific_ans to remove when study_url is nan and lenght of Answer.v_code is not 32 + # (abandoned/returned submissions). Keep any row whose submission id matches a completed + # HIT App assignment: that proves the participant actually did the task, e.g. a Prolific + # "TIMED-OUT" submission where the worker still submitted their ratings to the HIT App server. + completed_hitapp_ids = set(hitapp_ans['hitapp_assignmentid'].dropna()) + submission_id_norm = prolific_ans['prolific_submission_id'].str.strip().str.lower() + prolific_ans['to_remove'] = ( + prolific_ans['study_url'].isna() + & (prolific_ans['Answer.v_code'].str.strip().str.len() != 32) + & (~submission_id_norm.isin(completed_hitapp_ids)) + ) # drop the rows prolific_ans.drop(prolific_ans[prolific_ans['to_remove']].index, inplace=True) # remove the to_remove column From dd38736ea0cffe4fcc779d8321dc8f687a52934b Mon Sep 17 00:00:00 2001 From: Babak Naderi Date: Fri, 3 Jul 2026 16:04:09 +0200 Subject: [PATCH 018/111] Write result parser log fresh each run (mode='w') The per-run log file (*_logs.txt) used the default append-mode FileHandler, so successive runs stacked in one file and stale lines from earlier runs (e.g. an old 'not found in the Prolific data' count) lingered and misled. Open it in write mode so each run starts fresh, consistent with the other per-run outputs which are overwritten. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/result_parser.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/result_parser.py b/src/result_parser.py index a88bc70..2d86578 100644 --- a/src/result_parser.py +++ b/src/result_parser.py @@ -2225,7 +2225,9 @@ def analyze_results(config, test_method, answer_path,prolific_ans_path, list_of_ logger.setLevel(logging.INFO) console_handler = logging.StreamHandler() file_log_path = os.path.splitext(answer_path)[0] + f'_logs.txt' - file_handler = logging.FileHandler(file_log_path) + # open in write mode so each run starts a fresh log (consistent with the other + # per-run outputs, which are overwritten); avoids stale lines from earlier runs. + file_handler = logging.FileHandler(file_log_path, mode='w') logger.addHandler(console_handler) logger.addHandler(file_handler) logger.info(f"Start analyzing the results of {test_method} test") From 3579e5f27d0ebcbd1a33a1b280da4d0630985b2f Mon Sep 17 00:00:00 2001 From: Babak Naderi Date: Fri, 3 Jul 2026 16:21:43 +0200 Subject: [PATCH 019/111] Do not write empty result CSVs Add a save_csv helper that writes a dataframe only when it has at least one row and removes any stale file left at the same path by a previous run. Route the report/subset outputs through it (incomplete_submissions, not_found_in_prolific, accept/reject/block/extend lists, accept_reject_gui, quantity/quality bonus) so empty files such as Batch_369_incomplete_submissions.csv are no longer created. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/result_parser.py | 43 ++++++++++++++++++++++++++++++++----------- 1 file changed, 32 insertions(+), 11 deletions(-) diff --git a/src/result_parser.py b/src/result_parser.py index 2d86578..604642e 100644 --- a/src/result_parser.py +++ b/src/result_parser.py @@ -1008,6 +1008,27 @@ def evaluate_maximum_hits(data): return result +def save_csv(df, path, **kwargs): + """ + Save a dataframe to CSV only when it has at least one row. + + Empty result files (for example, no incomplete submissions or no rejections) are + not created. Any file left at the same path by a previous run is removed so the + output folder reflects the current run. + + :param df: Dataframe to write. + :param path: Destination CSV path. + :param kwargs: Extra keyword arguments forwarded to ``DataFrame.to_csv``. + :return: True if the file was written, False if skipped because it was empty. + """ + if df is not None and len(df) > 0: + df.to_csv(path, **kwargs) + return True + if os.path.exists(path): + os.remove(path) + return False + + def save_approve_rejected_ones_for_gui(data, path, wrong_vcodes): """ save approved/rejected in a csv-file to be used in GUI @@ -1031,7 +1052,7 @@ def save_approve_rejected_ones_for_gui(data, path, wrong_vcodes): small_df['n_duplicate'] = small_df.groupby('assignmentId')['assignmentId'].transform('size') small_df['n_duplicate'] = small_df['n_duplicate'].apply(lambda x: x - 1) - small_df.to_csv(path, index=False) + save_csv(small_df, path, index=False) def save_approved_ones(data, path): @@ -1051,7 +1072,7 @@ def save_approved_ones(data, path): logger.info(f' overall {c_accepted} answers are accepted, from them {df.shape[0]} were in submitted status') small_df = df[['assignment']].copy() small_df.rename(columns={'assignment': 'assignmentId'}, inplace=True) - small_df.to_csv(path, index=False) + save_csv(small_df, path, index=False) def save_block_list(block_list, path, wrong_v_code_freq): @@ -1071,7 +1092,7 @@ def save_block_list(block_list, path, wrong_v_code_freq): df2['BlockReason'] = "Wrong verification code" # concat the two dataframes df = pd.concat([df, df2], ignore_index=True) - df.to_csv(path, index=False) + save_csv(df, path, index=False) def check_wrong_vcode_should_block(wrong_vcodes): @@ -1123,7 +1144,7 @@ def save_rejected_ones(data, path, wrong_vcodes, not_accepted_reasons, num_rej_p wrong_vcodes_assignments.rename(columns={'AssignmentId': 'assignmentId'}, inplace=True) small_df = pd.concat([small_df, wrong_vcodes_assignments], ignore_index=True) - small_df.to_csv(path, index=False) + save_csv(small_df, path, index=False) def save_hits_to_be_extended(data, path): @@ -1138,7 +1159,7 @@ def save_hits_to_be_extended(data, path): small_df = df[['HITId']].copy() grouped = small_df.groupby(['HITId']).size().reset_index(name='counts') grouped.rename(columns={'counts': 'n_extended_assignments'}, inplace=True) - grouped.to_csv(path, index=False) + save_csv(grouped, path, index=False) def filter_answer_by_status_and_workers(answer_df, all_time_worker_id_in, new_woker_id_in, status_in): @@ -1209,8 +1230,8 @@ def calc_quantity_bonuses(answer_list, conf, path): merged = merged.round({'bonusAmount': 2}) if path is not None: - merged.to_csv(path, index=False) - logger.info(f' Quantity bonuses report is saved in: {path}') + if save_csv(merged, path, index=False): + logger.info(f' Quantity bonuses report is saved in: {path}') return merged @@ -1336,8 +1357,8 @@ def calc_quality_bonuses( smaller_df['reason'] = f'Well done! You belong to top {conf["bonus"]["quality_top_percentage"]}%.' else: smaller_df = pd.DataFrame(columns=['workerId', 'r', 'accept', 'assignmentId', 'bonusAmount', 'reason']) - smaller_df.head(max_workers).to_csv(path, index=False) - logger.info(f' Quality bonuses report is saved in: {path}') + if save_csv(smaller_df.head(max_workers), path, index=False): + logger.info(f' Quality bonuses report is saved in: {path}') def write_dict_as_csv(dic_to_write, file_name, *args, **kwargs): @@ -1827,7 +1848,7 @@ def combine_prolific_hit_server(prolific_ans_path, hitapp_ans_path): # These rows are the ones that are not submitted by the workers hitapp_ans_incomplete = hitapp_ans[hitapp_ans['Answer.browser_info'].isna()] hitapp_ans = hitapp_ans[~hitapp_ans['Answer.browser_info'].isna()] - hitapp_ans_incomplete.to_csv(os.path.splitext(hitapp_ans_path)[0] + '_incomplete_submissions.csv', index=False) + save_csv(hitapp_ans_incomplete, os.path.splitext(hitapp_ans_path)[0] + '_incomplete_submissions.csv', index=False) unique_assignments = hitapp_ans_incomplete['hitapp_assignmentid'].unique() # print the size logger.info(f"** {len(unique_assignments)} submissions are not completed by the workers.") @@ -1902,7 +1923,7 @@ def combine_prolific_hit_server(prolific_ans_path, hitapp_ans_path): # filter hitapp_ans and only keep the ones that are not in merged using the id column hitapp_ans_not_found_in_amt = hitapp_ans[~hitapp_ans['h_id'].isin(merged['h_id'])] - hitapp_ans_not_found_in_amt.to_csv(os.path.splitext(hitapp_ans_path)[0] + '_not_found_in_prolific.csv', index=False) + save_csv(hitapp_ans_not_found_in_amt, os.path.splitext(hitapp_ans_path)[0] + '_not_found_in_prolific.csv', index=False) # print the size logger.info(f"** {len(hitapp_ans_not_found_in_amt)} submissions in HITAPP data are not found in the Prolific data.") From 85caf9662465fd4a003ad99c24ef9954b39c9484 Mon Sep 17 00:00:00 2001 From: Babak Naderi Date: Fri, 3 Jul 2026 16:41:33 +0200 Subject: [PATCH 020/111] Add detailed rejection-reason breakdown (matrix + combinations) The flat 'Rejection reasons' list over-counts because a submission can fail several checks. Add a per-submission breakdown that reports, for rejected submissions: the marginal count per reason, how many were rejected by a single reason (e.g. removed only because of performance), and the most common reason combinations. Performance and max-hits are attributed only when the submission would otherwise have been accepted, so 'only performance' is meaningful. Writes two files: *_rejection_reason_matrix.csv (reason co-occurrence, with total and only-this-reason columns) and *_rejection_reason_combinations.csv. Store per-submission accept_failures / rejected_by_performance / rejected_by_max_hits, and make write_dict_as_csv robust to per-row key differences (union of keys + restval). Document the new outputs in the analyze-results agent. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/agents/analyze-results.agent.md | 13 ++++ src/result_parser.py | 92 ++++++++++++++++++++++++- 2 files changed, 103 insertions(+), 2 deletions(-) diff --git a/.github/agents/analyze-results.agent.md b/.github/agents/analyze-results.agent.md index 67b2bd5..a973e1e 100644 --- a/.github/agents/analyze-results.agent.md +++ b/.github/agents/analyze-results.agent.md @@ -169,6 +169,17 @@ Calculate: `rejection_percentage = XXXX / YYYY * 100` **⚠️ If rejection rate > 35%**: flag as alarming. Ask the user to investigate the rejection reasons in the data cleaning report. +**Rejection reason breakdown**: because a submission can fail several checks at +once, a flat per-reason count over-counts. Use the parser's "Rejection breakdown" +log and the two files it writes to report the marginal count per reason, how many +were rejected by a single reason ("only-this-reason", e.g. removed *only* because +of performance), and the most common reason combinations: +- `Batch_XXX_rejection_reason_matrix.csv` — reason co-occurrence matrix (diagonal / + `total` = submissions failing that reason; `only_this_reason` = failed that reason + alone; off-diagonal = failed both). +- `Batch_XXX_rejection_reason_combinations.csv` — each distinct reason combination + with its count and percentage. + #### 4b. Gold question performance Read `detailed_gold_question_performance.csv` from the working directory. @@ -211,6 +222,8 @@ After analysis, direct the user to the key output files: | `Batch_XXX_votes_per_worker_[SCALE].csv` | Per-worker rating statistics | | `Batch_XXX_all_votes_per_clip.csv` | All individual votes per clip (key: `all_votes` in name) | | `Batch_XXX_data_cleaning_report.csv` | Detailed per-submission data cleaning report | +| `Batch_XXX_rejection_reason_matrix.csv` | Reason co-occurrence matrix (total, only-this-reason, and pairwise counts) | +| `Batch_XXX_rejection_reason_combinations.csv` | Count/percentage of each distinct reason combination | | `detailed_gold_question_performance.csv` | Per-gold-clip acceptance/rejection statistics | | `Batch_XXX_quantity_bonus_report.csv` | Quantity bonus calculations (MTurk only; not generated when the platform is Prolific) | diff --git a/src/result_parser.py b/src/result_parser.py index 604642e..3a34c2a 100644 --- a/src/result_parser.py +++ b/src/result_parser.py @@ -728,6 +728,83 @@ def check_a_cmp(file_a, file_b, ans, audio_a_played, audio_b_played): return answer_is_correct # p835 +def report_rejection_breakdown(data, wrong_vcodes, answer_path): + """ + Build and save a detailed breakdown of why submissions were rejected. + + A submission can fail several checks at once, so a flat per-reason count + over-counts rejections. This assigns each rejected submission a single set of + reasons and reports: the marginal count per reason, how many were rejected by + that reason alone ("only"), the overall total, and the most common reason + combinations. A reason co-occurrence matrix and a combinations table are also + written as CSVs next to the answers file. + + Reason semantics: content checks (``gold``, ``tps``, ``math``, + ``all_audio_played``) come from the per-submission acceptance checks; + ``performance`` and ``max_hits`` are attributed only when the submission would + otherwise have been accepted; ``wrong_verification_code`` submissions are + tracked separately. + + :param data: List of per-submission dicts (worker_list) after all rejection steps. + :param wrong_vcodes: Dataframe of wrong-verification-code submissions, or None. + :param answer_path: Path used to derive the output CSV file names. + :return: None. + """ + combos = [] + for d in data: + if d.get('accept', 1) == 1: + continue + reasons = set(d.get('accept_failures', []) or []) + if d.get('rejected_by_performance'): + reasons.add('performance') + if d.get('rejected_by_max_hits'): + reasons.add('max_hits') + if not reasons: + reasons.add('other') + combos.append(tuple(sorted(reasons))) + # wrong-verification-code submissions are tracked outside worker_list + if wrong_vcodes is not None and len(wrong_vcodes) > 0: + combos.extend([('wrong_verification_code',)] * len(wrong_vcodes)) + + total_rejected = len(combos) + if total_rejected == 0: + logger.info('Rejection breakdown: no rejected submissions.') + return + + marginal = collections.Counter(r for combo in combos for r in combo) + only = collections.Counter(combo[0] for combo in combos if len(combo) == 1) + combo_counter = collections.Counter(combos) + + logger.info(f'Rejection breakdown ({total_rejected} rejected submissions):') + logger.info(' per reason (a submission may fail several):') + for reason, n in marginal.most_common(): + logger.info(f' {reason:24s} total={n:5d} ({100 * n / total_rejected:5.1f}%) ' + f'only-this-reason={only.get(reason, 0)}') + logger.info(' most common reason combinations:') + for combo, n in combo_counter.most_common(10): + logger.info(f' {" + ".join(combo):45s} {n:5d} ({100 * n / total_rejected:5.1f}%)') + + # combinations table + combo_rows = [{'reasons': ' + '.join(combo), 'n_reasons': len(combo), 'count': n, + 'percent': round(100 * n / total_rejected, 2)} + for combo, n in combo_counter.most_common()] + save_csv(pd.DataFrame(combo_rows), + os.path.splitext(answer_path)[0] + '_rejection_reason_combinations.csv', index=False) + + # reason co-occurrence matrix (diagonal = total with that reason) + reasons_sorted = sorted(marginal.keys()) + matrix = pd.DataFrame(0, index=reasons_sorted, columns=reasons_sorted) + for combo in combos: + for i in combo: + for j in combo: + matrix.loc[i, j] += 1 + matrix.insert(0, 'only_this_reason', [only.get(r, 0) for r in reasons_sorted]) + matrix.insert(0, 'total', [marginal[r] for r in reasons_sorted]) + matrix.index.name = 'reason' + matrix.to_csv(os.path.splitext(answer_path)[0] + '_rejection_reason_matrix.csv') + logger.info(f' Rejection matrix saved in: {os.path.splitext(answer_path)[0]}_rejection_reason_matrix.csv') + + def data_cleaning(filename, method, wrong_vcodes): """ Data screening process @@ -846,6 +923,8 @@ def data_cleaning(filename, method, wrong_vcodes): else: d['accept'] = 0 d['Approve'] = '' + # record the content checks this submission failed (for the rejection breakdown) + d['accept_failures'] = failures_accept not_accepted_reasons.extend(failures_accept) should_be_used, failures = check_if_session_should_be_used(d) d['failures'] = failures @@ -888,6 +967,7 @@ def data_cleaning(filename, method, wrong_vcodes): write_dict_as_csv(worker_list, report_file) save_approved_ones(worker_list, approved_file) save_rejected_ones(worker_list, rejected_file, wrong_vcodes, not_accepted_reasons, num_rej_perform) + report_rejection_breakdown(worker_list, wrong_vcodes, filename) save_approve_rejected_ones_for_gui(worker_list, accept_reject_gui_file, wrong_vcodes) save_hits_to_be_extended(worker_list, extending_hits_file) if len(block_list) > 0: @@ -956,6 +1036,9 @@ def evaluate_rater_performance(data, use_sessions, reject_on_failure=False): d['rater_performance_pass'] = 0 num_not_used_submissions += 1 if reject_on_failure: + # attribute to performance only if it would otherwise have been accepted + if d['accept'] == 1: + d['rejected_by_performance'] = 1 d['accept'] = 0 d['Approve'] = "" tmp = grouped_rej[grouped_rej['worker_id'].str.contains(d['worker_id'])] @@ -998,6 +1081,9 @@ def evaluate_maximum_hits(data): for d in data: if d['worker_id'] in cheater_workers_work_count: if cheater_workers_work_count[d['worker_id']] >= int(config['acceptance_criteria']['allowedMaxHITsInProject']): + # attribute to max_hits only if it would otherwise have been accepted + if d['accept'] == 1: + d['rejected_by_max_hits'] = 1 d['accept'] = 0 d['Reject'] += f"More than allowed limit of {config['acceptance_criteria']['allowedMaxHITsInProject']}" d['accept_and_use'] = 0 @@ -1372,10 +1458,12 @@ def write_dict_as_csv(dic_to_write, file_name, *args, **kwargs): with open(file_name, 'w', newline='') as output_file: if headers is None: if len(dic_to_write) > 0: - headers = list(dic_to_write[0].keys()) + # union of keys across all rows so per-submission extra keys + # (e.g. rejected_by_performance) don't raise and are all captured + headers = list(dict.fromkeys(k for row in dic_to_write for k in row.keys())) else: headers = [] - writer = csv.DictWriter(output_file, fieldnames=headers) + writer = csv.DictWriter(output_file, fieldnames=headers, restval='', extrasaction='ignore') writer.writeheader() for d in dic_to_write: writer.writerow(d) From 2ee56e833b8501340f6eb752388dd013dac6444a Mon Sep 17 00:00:00 2001 From: Babak Naderi Date: Fri, 3 Jul 2026 17:00:01 +0200 Subject: [PATCH 021/111] Fix two bugs in rater performance evaluation - Guard against a KeyError in evaluate_rater_performance when a batch has no usable submissions: the old check tested for 'not_used_count' but the body referenced 'used_count', which is absent in the all-not-used case. Now both count columns are ensured before computing acceptance_rate. - Look up a worker's row with an exact match instead of worker_id.str.contains(...), which does regex/substring matching and could match the wrong worker (or error on regex-special ids). Behavior on normal data is unchanged (verified: r1 still 149 performance / 271 total rejections). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/result_parser.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/result_parser.py b/src/result_parser.py index 3a34c2a..636bfc7 100644 --- a/src/result_parser.py +++ b/src/result_parser.py @@ -1005,11 +1005,12 @@ def evaluate_rater_performance(data, use_sessions, reject_on_failure=False): grouped = df.groupby(['worker_id', 'accept_and_use']).size().unstack(fill_value=0).reset_index() grouped = grouped.rename(columns={0: 'not_used_count', 1: 'used_count'}) - # check if not_used_count is in grouped - if 'not_used_count' in grouped.columns: - grouped['acceptance_rate'] = (grouped['used_count'] * 100)/(grouped['used_count'] + grouped['not_used_count']) - else: - grouped['acceptance_rate'] = 100 + # ensure both counts exist even when the whole batch is all-used or all-not-used, + # otherwise the acceptance_rate computation below raises a KeyError + for col in ['not_used_count', 'used_count']: + if col not in grouped.columns: + grouped[col] = 0 + grouped['acceptance_rate'] = (grouped['used_count'] * 100) / (grouped['used_count'] + grouped['not_used_count']) #grouped.to_csv('tmp.csv') if 'rater_min_acceptance_rate_current_test' in config[section]: @@ -1041,7 +1042,7 @@ def evaluate_rater_performance(data, use_sessions, reject_on_failure=False): d['rejected_by_performance'] = 1 d['accept'] = 0 d['Approve'] = "" - tmp = grouped_rej[grouped_rej['worker_id'].str.contains(d['worker_id'])] + tmp = grouped_rej[grouped_rej['worker_id'] == d['worker_id']] if len(d['Reject'])>0: d['Reject'] = d['Reject'] + f" Failed in performance criteria- only {tmp['acceptance_rate'].iloc[0]:.2f}% of submissions passed data cleansing." else: From d5914ef672d42cbb74e29a86d53d591e06fb0987 Mon Sep 17 00:00:00 2001 From: Babak Naderi Date: Mon, 6 Jul 2026 13:14:27 +0200 Subject: [PATCH 022/111] Gate rater reject pass on content-QC (accept), use pass on accept_and_use evaluate_rater_performance computed the per-worker rate from accept_and_use for both passes, so the reject pass (which decides payment) gated on the 'usable' rate instead of the content-QC pass rate. Workers who passed data cleansing but produced few aggregatable ratings were rejected/not paid. Now the reject pass groups by 'accept' and the use pass keeps 'accept_and_use', so QC-passing workers are paid but still excluded from the MOS when their usable rate is low. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/result_parser.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/result_parser.py b/src/result_parser.py index 636bfc7..fa4b73a 100644 --- a/src/result_parser.py +++ b/src/result_parser.py @@ -1003,9 +1003,14 @@ def evaluate_rater_performance(data, use_sessions, reject_on_failure=False): # rater_min_accepted_hits_current_test - grouped = df.groupby(['worker_id', 'accept_and_use']).size().unstack(fill_value=0).reset_index() + # The reject pass decides payment, so it must gate on the content-QC flag + # ('accept' = "passed data cleansing"); the use pass decides aggregation, so it + # gates on 'accept_and_use'. used_count/not_used_count are the pass/fail counts + # of whichever flag applies, and acceptance_rate is the pass rate. + flag_col = 'accept' if reject_on_failure else 'accept_and_use' + grouped = df.groupby(['worker_id', flag_col]).size().unstack(fill_value=0).reset_index() grouped = grouped.rename(columns={0: 'not_used_count', 1: 'used_count'}) - # ensure both counts exist even when the whole batch is all-used or all-not-used, + # ensure both counts exist even when the whole batch is all-pass or all-fail, # otherwise the acceptance_rate computation below raises a KeyError for col in ['not_used_count', 'used_count']: if col not in grouped.columns: From 008a33282d55807774d5e77471af156908b1a210 Mon Sep 17 00:00:00 2001 From: Babak Naderi Date: Mon, 6 Jul 2026 15:03:30 +0200 Subject: [PATCH 023/111] Review by submission status: pay used work, don't re-action send_reviews_for_study now fetches each submission's current status once (via the study's submissions list, paginated) and acts accordingly: - Approvals: bulk-approve AWAITING REVIEW; individually approve accepted RETURNED/TIMED-OUT submissions (their work is used, so pay them); skip already-approved; export a CSV of any that still need manual payment. - Reject / ask-return: only submissions still AWAITING REVIEW, skipping ones already returned/rejected/approved. Adds approve_submission helper and an approved/actioned/skipped summary; falls back to per-submission status checks if the bulk fetch fails. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/prolific_utils.py | 171 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 161 insertions(+), 10 deletions(-) diff --git a/src/prolific_utils.py b/src/prolific_utils.py index c04e5da..f8eec58 100644 --- a/src/prolific_utils.py +++ b/src/prolific_utils.py @@ -160,31 +160,182 @@ def get_submission_status(assignment_id): return None +def get_study_id_for_submission(assignment_id): + """ + Resolve the Prolific study id that a submission belongs to. + + :param assignment_id: A Prolific submission id. + :return: The study id string, or None if it could not be determined. + """ + data = get_submission_data(assignment_id) + if data: + return data.get('study_id') or data.get('study') + return None + + +def fetch_submission_status_map(study_id): + """ + Fetch the current status of every submission in a study. + + Pages through the study's submissions once and returns a dict mapping the + lower-cased submission id to its Prolific status (e.g. "AWAITING REVIEW", + "APPROVED", "RETURNED"). This lets the review step act only on submissions + still awaiting review, instead of one status GET per submission. + + :param study_id: The Prolific study id. + :return: Dict of {submission_id (lower-case): status}. + """ + status_map = {} + url = f"{base_url}/studies/{study_id}/submissions/" + headers = { + 'Authorization': f'Token {api_token}', + 'Content-Type': 'application/json', + 'Accept': 'application/json', + } + while url: + try: + response = requests.get(url, headers=headers, timeout=30) + except requests.exceptions.RequestException as e: + logger.info(f"Error listing submissions for study {study_id}: {e}") + break + if response.status_code != 200: + logger.info(f"Error listing submissions: {response.status_code} - {response.text}") + break + data = response.json() + for s in data.get('results', []): + sid = s.get('id') + if sid is not None: + status_map[str(sid).strip().lower()] = s.get('status') + # follow pagination: top-level 'next' or _links.next + next_url = data.get('next') + if not next_url: + links_next = data.get('_links', {}).get('next') + next_url = links_next.get('href') if isinstance(links_next, dict) else links_next + url = next_url + logger.info(f"Fetched status for {len(status_map)} submissions in study {study_id}.") + return status_map + + +def approve_submission(assignment_id): + """ + Approve a single submission via the transition endpoint. + + Used for accepted submissions that cannot go through bulk-approve because they + are not AWAITING REVIEW (e.g. RETURNED or TIMED-OUT but the work was used). + + :param assignment_id: The Prolific submission id. + :return: True if the submission was approved, False otherwise. + """ + url = f"{base_url}/submissions/{assignment_id}/transition/" + headers = { + 'Authorization': f'Token {api_token}', + 'Content-Type': 'application/json', + 'Accept': 'application/json', + } + payload = {"action": "APPROVE"} + try: + response = requests.post(url, headers=headers, data=json.dumps(payload), timeout=10) + except requests.exceptions.RequestException as e: + logger.info(f"Submission {assignment_id} - approve error: {e}") + return False + if response.status_code == 200: + logger.info(f"Submission {assignment_id} approved individually.") + return True + logger.info(f"Submission {assignment_id} could not be approved " + f"(status {response.status_code}): {response.text}") + return False + + def send_reviews_for_study(csv_data_path, detailed_data_cleaning_report=None, block_report=None): - count = 0 df = pd.read_csv(csv_data_path) # Approve is x df_approved = df[df['Approve'] == 'x'] df_rejected = df[df['Approve'] != 'x'] - submission_to_approve = df_approved['assignmentId'].tolist() - bulk_approve_submission(submission_to_approve) + # Fetch the current status of every submission once. Prolific only allows + # approving/rejecting/requesting-return of submissions that are still + # AWAITING REVIEW (bulk-approve even 400s the whole batch if one id is not), + # so on a re-run we must skip anything already approved/rejected/returned. + status_map = {} + all_ids = df['assignmentId'].dropna().astype(str).str.strip().str.lower().tolist() + if all_ids: + study_id = get_study_id_for_submission(all_ids[0]) + if study_id: + status_map = fetch_submission_status_map(study_id) + if not status_map: + logger.warning("Could not fetch study submission statuses in bulk; " + "falling back to a per-submission status check.") + + def _status_of(aid): + st = status_map.get(aid) + if st is None and aid not in status_map: + st = get_submission_status(aid) # fallback for ids missing from the bulk map + return str(st).strip().upper() if st is not None else None + + # ---- approvals: pay for every accepted submission whose work we use ---- + # AWAITING REVIEW go through bulk-approve; RETURNED/TIMED-OUT can't be + # bulk-approved, so try individually and record any that still can't be paid. + submission_to_approve = [] + n_already_approved = 0 + manual_payment_rows = [] + for _, row in df_approved.iterrows(): + aid = str(row['assignmentId']).strip().lower() + wid = str(row['WorkerId']).strip().lower() + st = _status_of(aid) + if st == "AWAITING REVIEW": + submission_to_approve.append(aid) + elif st == "APPROVED": + n_already_approved += 1 + elif st in ("RETURNED", "TIMED-OUT"): + # the work was used, so the worker should be paid; bulk-approve won't + # take these, so attempt an individual approval and flag failures + if not approve_submission(aid): + manual_payment_rows.append({"WorkerId": wid, "assignmentId": aid, + "status": st, "reason": "used but could not be approved"}) + else: + manual_payment_rows.append({"WorkerId": wid, "assignmentId": aid, + "status": st, "reason": "used but not in an approvable state"}) + + if submission_to_approve: + bulk_approve_submission(submission_to_approve) + n_actioned = 0 + n_skipped = 0 for index, row in df_rejected.iterrows(): # WorkerId assignmentId HITId Approve Reject worker_id = row['WorkerId'].lower() assignment_id = row['assignmentId'].lower() hit_id = row['HITId'] - count = count+ 1 - + if row['Approve'] is not None and not pd.isna(row['Approve']) and row['Approve'].strip() != "": continue # already handled in bulk approve + # Only submissions still AWAITING REVIEW can be rejected or asked to return; skip the + # rest (e.g. already RETURNED/REJECTED/APPROVED from a previous review run). + if _status_of(assignment_id) != "AWAITING REVIEW": + logger.info(f"Submission {assignment_id} skipped (status: {status_map.get(assignment_id)}); " + f"only AWAITING REVIEW submissions are rejected/asked to return.") + n_skipped += 1 + continue + reason = row['Reject'] + if args.force_reject: + accept_reject_submission(worker_id, assignment_id, reason) else: - reason = row['Reject'] - if args.force_reject: - accept_reject_submission(worker_id, assignment_id, reason) - else: - ask_return(assignment_id, reason) + ask_return(assignment_id, reason) + n_actioned += 1 + + if manual_payment_rows: + stamp = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S") + out_dir = os.path.dirname(csv_data_path) + manual_path = os.path.join(out_dir, f"prolific_manual_payment_needed_{stamp}.csv") + pd.DataFrame(manual_payment_rows).to_csv(manual_path, index=False) + logger.info(f"{len(manual_payment_rows)} used submission(s) could not be auto-paid " + f"(returned/timed-out); listed for manual payment in {manual_path}") + + logger.info(f"Review complete: {len(submission_to_approve)} bulk-approved, " + f"{n_already_approved} already approved, " + f"{len(manual_payment_rows)} need manual payment, " + f"{n_actioned} {'rejected' if args.force_reject else 'asked to return'}, " + f"{n_skipped} reject/return-skipped (not awaiting review).") # assign participants with rdp to the group to be excluded from the future studies if detailed_data_cleaning_report and rdp_group_id is not None: From 50bfd2c3b54f57434fae6858aefaa83f19f3e24d Mon Sep 17 00:00:00 2001 From: Babak Naderi Date: Mon, 6 Jul 2026 15:51:12 +0200 Subject: [PATCH 024/111] Emit one row per gold question in detailed_gold_question_performance.csv For P.804 the detailed gold report previously wrote one wide row per submission holding both gold clips side by side. Explode it into one row per gold question (single gold_url per line) with flat per-dimension columns (correct/given/wrong), so a submission with two gold clips contributes two rows. Other methods keep their existing format. Update the analyze-results agent to group by gold_url. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/agents/analyze-results.agent.md | 19 ++++++---- src/result_parser.py | 50 +++++++++++++++++++++++-- 2 files changed, 57 insertions(+), 12 deletions(-) diff --git a/.github/agents/analyze-results.agent.md b/.github/agents/analyze-results.agent.md index a973e1e..cf25ba1 100644 --- a/.github/agents/analyze-results.agent.md +++ b/.github/agents/analyze-results.agent.md @@ -184,13 +184,16 @@ of performance), and the most common reason combinations: Read `detailed_gold_question_performance.csv` from the working directory. -- Look for columns matching `wrong*` — these indicate how many times each gold - clip received a wrong answer. -- Look for columns matching `url*` — these identify the gold clip URLs. -- **Any row where the sum of `wrong*` columns > 0** means that gold clip received - at least one wrong answer. -- Calculate the rejection rate per gold clip: - `gold_rejection_rate = wrong_count / total_times_shown * 100` +The report has **one row per gold question** (so a P.804 submission with two gold +clips contributes two rows). Key columns: +- `gold_url` — the gold clip evaluated on that row (exactly one per row). +- `wrong` / `correct` — number of dimensions answered wrong / correct for that gold + on that submission; per-dimension detail is in `_wrong` (e.g. `sig_wrong`). +- `worker_id`, `HITID` — the submission the row belongs to. + +To assess gold clips, **group by `gold_url`**: +- A row counts as a failed presentation when `wrong > 0`. +- `gold_rejection_rate = (rows with wrong > 0 for that gold_url) / (total rows for that gold_url) * 100`. **⚠️ If any gold clip is rejected > 20% of the time**: flag as alarming. Ask the user to check that clip and verify the expected answer is correct. It may @@ -224,7 +227,7 @@ After analysis, direct the user to the key output files: | `Batch_XXX_data_cleaning_report.csv` | Detailed per-submission data cleaning report | | `Batch_XXX_rejection_reason_matrix.csv` | Reason co-occurrence matrix (total, only-this-reason, and pairwise counts) | | `Batch_XXX_rejection_reason_combinations.csv` | Count/percentage of each distinct reason combination | -| `detailed_gold_question_performance.csv` | Per-gold-clip acceptance/rejection statistics | +| `detailed_gold_question_performance.csv` | One row per gold question per submission (single `gold_url` per row) | | `Batch_XXX_quantity_bonus_report.csv` | Quantity bonus calculations (MTurk only; not generated when the platform is Prolific) | **Scale suffixes by method:** diff --git a/src/result_parser.py b/src/result_parser.py index fa4b73a..7e5e7b1 100644 --- a/src/result_parser.py +++ b/src/result_parser.py @@ -728,6 +728,42 @@ def check_a_cmp(file_a, file_b, ans, audio_a_played, audio_b_played): return answer_is_correct # p835 +def explode_p804_gold_rec(rec): + """ + Turn a P.804 gold-question record into one row per gold question. + + A P.804 submission is checked against one or two gold clips, and the raw + record stores them side by side (``url``/``url_2`` and per-dimension columns + with a ``""``/``_2`` postfix). This returns a list with one flat row per gold + clip so the detailed report has a single ``gold_url`` per line. + + :param rec: The gold record dict for one submission (from ``check_gold_question``). + :return: List of per-gold-question row dicts. + """ + items = ['noise', 'col', 'loud', 'disc', 'reverb', 'sig', 'ovrl'] + common = {k: rec.get(k) for k in ('worker_id', 'HITID', 'correct_tps')} + rows = [] + # two gold questions (postfixes "" and "_2") or a single one (no postfix) + postfixes = ['', '_2'] if 'url_2' in rec else [''] + for pf in postfixes: + row = dict(common) + row['gold_url'] = rec.get(f'url{pf}') + row['correct'] = rec.get(f'correct{pf}') + row['wrong'] = rec.get(f'wrong{pf}') + for item in items: + if pf == '' and 'url_2' not in rec: + # single-gold record uses un-postfixed keys (e.g. "noise", "noise_given") + row[item] = rec.get(item) + row[f'{item}_given'] = rec.get(f'{item}_given') + row[f'{item}_wrong'] = rec.get(f'{item}_wrong') + else: + row[item] = rec.get(f'{item}_{pf}') + row[f'{item}_given'] = rec.get(f'{item}_{pf}_given') + row[f'{item}_wrong'] = rec.get(f'{item}_{pf}_wrong') + rows.append(row) + return rows + + def report_rejection_breakdown(data, wrong_vcodes, answer_path): """ Build and save a detailed breakdown of why submissions were rejected. @@ -833,6 +869,7 @@ def data_cleaning(filename, method, wrong_vcodes): not_accepted_reasons = [] rec_list = [] + gold_rows = [] # one row per gold question (P.804 detailed report) for row in reader: correct_cmp_ans = 0 #print(row['answer.8_hearing'] is None) @@ -904,8 +941,13 @@ def data_cleaning(filename, method, wrong_vcodes): rec['HITID'] = row["hitid"] rec['worker_id'] = row["workerid"] rec['correct_tps'] = d["correct_tps"] - rec_list.append(rec) - if method =="p804" and 'url_2' in rec: + if method == "p804": + # detailed report: one row per gold question instead of one wide row per submission + if rec is not None: + gold_rows.extend(explode_p804_gold_rec(rec)) + else: + rec_list.append(rec) + if method =="p804" and rec is not None and 'url_2' in rec: gold_question_wrong = (1 if rec['wrong']>0 else 0)+ (2 if rec['wrong_2']>0 else 0) d["gold_question_wrong"] = gold_question_wrong # remove the comment to only reject on first gold question @@ -938,8 +980,8 @@ def data_cleaning(filename, method, wrong_vcodes): d['accept_and_use'] = 0 worker_list.append(d) - tmp_df = pd.DataFrame(rec_list) - tmp_df.to_csv('detailed_gold_question_performance.csv') + tmp_df = pd.DataFrame(gold_rows if method == "p804" else rec_list) + tmp_df.to_csv('detailed_gold_question_performance.csv', index=False) # logger.info(f"Number of submissions: {len(worker_list)}") report_file = os.path.splitext(filename)[0] + '_data_cleaning_report.csv' From 01b7eadc0371a2c8bf2907456dee80f2f4726d23 Mon Sep 17 00:00:00 2001 From: Babak Naderi Date: Mon, 6 Jul 2026 16:17:17 +0200 Subject: [PATCH 025/111] Add gold_summary.csv: per-gold-clip wrong-rate per scale Aggregate the P.804 per-gold rows into gold_summary.csv with one row per gold clip: url, n_submission, and the percentage of submissions that got each scale (noise/col/loud/disc/reverb/sig/ovrl) wrong. Makes bad or too-hard gold clips easy to spot. Document it in the analyze-results agent. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/agents/analyze-results.agent.md | 6 +++++ src/result_parser.py | 31 +++++++++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/.github/agents/analyze-results.agent.md b/.github/agents/analyze-results.agent.md index cf25ba1..e09caf3 100644 --- a/.github/agents/analyze-results.agent.md +++ b/.github/agents/analyze-results.agent.md @@ -199,6 +199,11 @@ To assess gold clips, **group by `gold_url`**: user to check that clip and verify the expected answer is correct. It may indicate a bad gold clip rather than bad workers. +For P.804, `gold_summary.csv` gives this directly: one row per gold clip with +`url`, `n_submission`, and `_wrong_pct` (the percentage of submissions that +got each scale wrong). Scan the `*_wrong_pct` columns for clips with a high wrong +rate on any scale. + #### 4c. Summary to present Provide the user with a structured summary: @@ -228,6 +233,7 @@ After analysis, direct the user to the key output files: | `Batch_XXX_rejection_reason_matrix.csv` | Reason co-occurrence matrix (total, only-this-reason, and pairwise counts) | | `Batch_XXX_rejection_reason_combinations.csv` | Count/percentage of each distinct reason combination | | `detailed_gold_question_performance.csv` | One row per gold question per submission (single `gold_url` per row) | +| `gold_summary.csv` | Per-gold-clip summary (P.804): `url`, `n_submission`, and per-scale wrong percentage | | `Batch_XXX_quantity_bonus_report.csv` | Quantity bonus calculations (MTurk only; not generated when the platform is Prolific) | **Scale suffixes by method:** diff --git a/src/result_parser.py b/src/result_parser.py index 7e5e7b1..796b875 100644 --- a/src/result_parser.py +++ b/src/result_parser.py @@ -764,6 +764,35 @@ def explode_p804_gold_rec(rec): return rows +def write_gold_summary(gold_rows, path): + """ + Write a per-gold-clip summary of how often each P.804 scale was answered wrong. + + Aggregates the exploded gold rows by gold clip and reports, per clip, how many + submissions were checked against it and the percentage of those submissions + that got each scale wrong. + + :param gold_rows: List of per-gold-question row dicts (from explode_p804_gold_rec). + :param path: Destination CSV path. + :return: None. + """ + items = ['noise', 'col', 'loud', 'disc', 'reverb', 'sig', 'ovrl'] + gdf = pd.DataFrame(gold_rows) + if 'gold_url' not in gdf.columns or len(gdf) == 0: + return + summary = [] + for url, grp in gdf.groupby('gold_url'): + n = len(grp) + row = {'url': url, 'n_submission': n} + for item in items: + wcol = f'{item}_wrong' + n_wrong = int((grp[wcol] == 1).sum()) if wcol in grp.columns else 0 + row[f'{item}_wrong_pct'] = round(100 * n_wrong / n, 2) if n else 0.0 + summary.append(row) + pd.DataFrame(summary).to_csv(path, index=False) + logger.info(f" Gold summary saved in: {path}") + + def report_rejection_breakdown(data, wrong_vcodes, answer_path): """ Build and save a detailed breakdown of why submissions were rejected. @@ -982,6 +1011,8 @@ def data_cleaning(filename, method, wrong_vcodes): worker_list.append(d) tmp_df = pd.DataFrame(gold_rows if method == "p804" else rec_list) tmp_df.to_csv('detailed_gold_question_performance.csv', index=False) + if method == "p804" and gold_rows: + write_gold_summary(gold_rows, 'gold_summary.csv') # logger.info(f"Number of submissions: {len(worker_list)}") report_file = os.path.splitext(filename)[0] + '_data_cleaning_report.csv' From 1831710323e0f670cc6f9d3aa92c5324ac6e51e8 Mon Sep 17 00:00:00 2001 From: Babak Naderi Date: Mon, 6 Jul 2026 16:29:08 +0200 Subject: [PATCH 026/111] Add per-scale expected answer to gold_summary.csv Include _expected next to _wrong_pct for each gold clip, so the expected (correct) answer per scale sits alongside the wrong rate. Blank when a scale is not targeted by the clip. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/agents/analyze-results.agent.md | 9 +++++---- src/result_parser.py | 4 ++++ 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/agents/analyze-results.agent.md b/.github/agents/analyze-results.agent.md index e09caf3..8109217 100644 --- a/.github/agents/analyze-results.agent.md +++ b/.github/agents/analyze-results.agent.md @@ -200,9 +200,10 @@ user to check that clip and verify the expected answer is correct. It may indicate a bad gold clip rather than bad workers. For P.804, `gold_summary.csv` gives this directly: one row per gold clip with -`url`, `n_submission`, and `_wrong_pct` (the percentage of submissions that -got each scale wrong). Scan the `*_wrong_pct` columns for clips with a high wrong -rate on any scale. +`url`, `n_submission`, and per scale both `_expected` (the correct answer, +blank if not targeted) and `_wrong_pct` (percentage of submissions that got +it wrong). A high wrong rate against a clear expected answer flags a bad/mis-keyed +or too-hard gold clip. #### 4c. Summary to present @@ -233,7 +234,7 @@ After analysis, direct the user to the key output files: | `Batch_XXX_rejection_reason_matrix.csv` | Reason co-occurrence matrix (total, only-this-reason, and pairwise counts) | | `Batch_XXX_rejection_reason_combinations.csv` | Count/percentage of each distinct reason combination | | `detailed_gold_question_performance.csv` | One row per gold question per submission (single `gold_url` per row) | -| `gold_summary.csv` | Per-gold-clip summary (P.804): `url`, `n_submission`, and per-scale wrong percentage | +| `gold_summary.csv` | Per-gold-clip summary (P.804): `url`, `n_submission`, and per scale `_expected` + `_wrong_pct` | | `Batch_XXX_quantity_bonus_report.csv` | Quantity bonus calculations (MTurk only; not generated when the platform is Prolific) | **Scale suffixes by method:** diff --git a/src/result_parser.py b/src/result_parser.py index 796b875..0539625 100644 --- a/src/result_parser.py +++ b/src/result_parser.py @@ -785,6 +785,10 @@ def write_gold_summary(gold_rows, path): n = len(grp) row = {'url': url, 'n_submission': n} for item in items: + # expected (correct) answer for this scale on this gold clip (constant per + # clip; blank when the scale is not targeted / has no encoded answer) + expected = grp[item].dropna() if item in grp.columns else pd.Series([], dtype=float) + row[f'{item}_expected'] = int(expected.iloc[0]) if len(expected) else '' wcol = f'{item}_wrong' n_wrong = int((grp[wcol] == 1).sum()) if wcol in grp.columns else 0 row[f'{item}_wrong_pct'] = round(100 * n_wrong / n, 2) if n else 0.0 From 38e6462b51d6c33cdb27c930e3c589b223c1e5f7 Mon Sep 17 00:00:00 2001 From: Babak Naderi Date: Mon, 6 Jul 2026 16:43:24 +0200 Subject: [PATCH 027/111] gold_summary: add max_wrong_pct and per-scale mean rating Add a max_wrong_pct column (the worst per-scale wrong rate for each gold clip, surfaced right after n_submission) and, per scale, _mean (mean rating participants gave). Together with _expected this makes mis-keyed or too-hard gold clips obvious (expected vs mean vs wrong rate). Update the agent. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/agents/analyze-results.agent.md | 11 ++++++----- src/result_parser.py | 24 +++++++++++++++++++----- 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/.github/agents/analyze-results.agent.md b/.github/agents/analyze-results.agent.md index 8109217..f1c463e 100644 --- a/.github/agents/analyze-results.agent.md +++ b/.github/agents/analyze-results.agent.md @@ -200,10 +200,11 @@ user to check that clip and verify the expected answer is correct. It may indicate a bad gold clip rather than bad workers. For P.804, `gold_summary.csv` gives this directly: one row per gold clip with -`url`, `n_submission`, and per scale both `_expected` (the correct answer, -blank if not targeted) and `_wrong_pct` (percentage of submissions that got -it wrong). A high wrong rate against a clear expected answer flags a bad/mis-keyed -or too-hard gold clip. +`url`, `n_submission`, `max_wrong_pct` (worst scale wrong rate for that clip), and +per scale `_expected` (correct answer, blank if not targeted), +`_mean` (mean rating participants gave), and `_wrong_pct`. Sort by +`max_wrong_pct`; when the mean rating is far from the expected answer with a high +wrong rate, the gold clip is likely mis-keyed or too hard on that scale. #### 4c. Summary to present @@ -234,7 +235,7 @@ After analysis, direct the user to the key output files: | `Batch_XXX_rejection_reason_matrix.csv` | Reason co-occurrence matrix (total, only-this-reason, and pairwise counts) | | `Batch_XXX_rejection_reason_combinations.csv` | Count/percentage of each distinct reason combination | | `detailed_gold_question_performance.csv` | One row per gold question per submission (single `gold_url` per row) | -| `gold_summary.csv` | Per-gold-clip summary (P.804): `url`, `n_submission`, and per scale `_expected` + `_wrong_pct` | +| `gold_summary.csv` | Per-gold-clip summary (P.804): `url`, `n_submission`, `max_wrong_pct`, and per scale `_expected` + `_mean` + `_wrong_pct` | | `Batch_XXX_quantity_bonus_report.csv` | Quantity bonus calculations (MTurk only; not generated when the platform is Prolific) | **Scale suffixes by method:** diff --git a/src/result_parser.py b/src/result_parser.py index 0539625..1f5f726 100644 --- a/src/result_parser.py +++ b/src/result_parser.py @@ -766,11 +766,12 @@ def explode_p804_gold_rec(rec): def write_gold_summary(gold_rows, path): """ - Write a per-gold-clip summary of how often each P.804 scale was answered wrong. + Write a per-gold-clip summary of how each P.804 scale performed. Aggregates the exploded gold rows by gold clip and reports, per clip, how many - submissions were checked against it and the percentage of those submissions - that got each scale wrong. + submissions were checked against it, the worst scale wrong rate, and per scale + the expected answer, the mean rating given, and the percentage of submissions + that got that scale wrong. :param gold_rows: List of per-gold-question row dicts (from explode_p804_gold_rec). :param path: Destination CSV path. @@ -784,16 +785,29 @@ def write_gold_summary(gold_rows, path): for url, grp in gdf.groupby('gold_url'): n = len(grp) row = {'url': url, 'n_submission': n} + wrong_pcts = [] for item in items: # expected (correct) answer for this scale on this gold clip (constant per # clip; blank when the scale is not targeted / has no encoded answer) expected = grp[item].dropna() if item in grp.columns else pd.Series([], dtype=float) row[f'{item}_expected'] = int(expected.iloc[0]) if len(expected) else '' + # mean of the ratings participants gave for this scale on this gold clip + given = pd.to_numeric(grp[f'{item}_given'], errors='coerce') if f'{item}_given' in grp.columns \ + else pd.Series([], dtype=float) + row[f'{item}_mean'] = round(given.mean(), 2) if given.notna().any() else '' wcol = f'{item}_wrong' n_wrong = int((grp[wcol] == 1).sum()) if wcol in grp.columns else 0 - row[f'{item}_wrong_pct'] = round(100 * n_wrong / n, 2) if n else 0.0 + pct = round(100 * n_wrong / n, 2) if n else 0.0 + row[f'{item}_wrong_pct'] = pct + wrong_pcts.append(pct) + # the biggest problem for this gold clip: worst per-scale wrong rate + row['max_wrong_pct'] = max(wrong_pcts) if wrong_pcts else 0.0 summary.append(row) - pd.DataFrame(summary).to_csv(path, index=False) + df = pd.DataFrame(summary) + # surface max_wrong_pct right after n_submission + front = ['url', 'n_submission', 'max_wrong_pct'] + df = df[front + [c for c in df.columns if c not in front]] + df.to_csv(path, index=False) logger.info(f" Gold summary saved in: {path}") From 0be1991b0cc3a3902db333f61fe4562a2471d4b8 Mon Sep 17 00:00:00 2001 From: Babak Naderi Date: Mon, 6 Jul 2026 16:50:04 +0200 Subject: [PATCH 028/111] Write gold detail/summary next to the answers file detailed_gold_question_performance.csv and gold_summary.csv were written with bare filenames, so they landed in the current working directory instead of next to the --answers file like every other output. Prefix them with the answers base name (e.g. Batch_XXX_detailed_gold_question_performance.csv, Batch_XXX_gold_summary.csv) so their location no longer depends on the working directory. Update the analyze-results agent references. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/agents/analyze-results.agent.md | 8 ++++---- src/result_parser.py | 6 ++++-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/agents/analyze-results.agent.md b/.github/agents/analyze-results.agent.md index f1c463e..0595c8b 100644 --- a/.github/agents/analyze-results.agent.md +++ b/.github/agents/analyze-results.agent.md @@ -182,7 +182,7 @@ of performance), and the most common reason combinations: #### 4b. Gold question performance -Read `detailed_gold_question_performance.csv` from the working directory. +Read `Batch_XXX_detailed_gold_question_performance.csv` (next to the answers file). The report has **one row per gold question** (so a P.804 submission with two gold clips contributes two rows). Key columns: @@ -199,7 +199,7 @@ To assess gold clips, **group by `gold_url`**: user to check that clip and verify the expected answer is correct. It may indicate a bad gold clip rather than bad workers. -For P.804, `gold_summary.csv` gives this directly: one row per gold clip with +For P.804, `Batch_XXX_gold_summary.csv` gives this directly: one row per gold clip with `url`, `n_submission`, `max_wrong_pct` (worst scale wrong rate for that clip), and per scale `_expected` (correct answer, blank if not targeted), `_mean` (mean rating participants gave), and `_wrong_pct`. Sort by @@ -234,8 +234,8 @@ After analysis, direct the user to the key output files: | `Batch_XXX_data_cleaning_report.csv` | Detailed per-submission data cleaning report | | `Batch_XXX_rejection_reason_matrix.csv` | Reason co-occurrence matrix (total, only-this-reason, and pairwise counts) | | `Batch_XXX_rejection_reason_combinations.csv` | Count/percentage of each distinct reason combination | -| `detailed_gold_question_performance.csv` | One row per gold question per submission (single `gold_url` per row) | -| `gold_summary.csv` | Per-gold-clip summary (P.804): `url`, `n_submission`, `max_wrong_pct`, and per scale `_expected` + `_mean` + `_wrong_pct` | +| `Batch_XXX_detailed_gold_question_performance.csv` | One row per gold question per submission (single `gold_url` per row) | +| `Batch_XXX_gold_summary.csv` | Per-gold-clip summary (P.804): `url`, `n_submission`, `max_wrong_pct`, and per scale `_expected` + `_mean` + `_wrong_pct` | | `Batch_XXX_quantity_bonus_report.csv` | Quantity bonus calculations (MTurk only; not generated when the platform is Prolific) | **Scale suffixes by method:** diff --git a/src/result_parser.py b/src/result_parser.py index 1f5f726..a029f3f 100644 --- a/src/result_parser.py +++ b/src/result_parser.py @@ -1027,10 +1027,12 @@ def data_cleaning(filename, method, wrong_vcodes): d['accept_and_use'] = 0 worker_list.append(d) + # write these next to the answers file (same base as every other output) + gold_detail_path = os.path.splitext(filename)[0] + '_detailed_gold_question_performance.csv' tmp_df = pd.DataFrame(gold_rows if method == "p804" else rec_list) - tmp_df.to_csv('detailed_gold_question_performance.csv', index=False) + tmp_df.to_csv(gold_detail_path, index=False) if method == "p804" and gold_rows: - write_gold_summary(gold_rows, 'gold_summary.csv') + write_gold_summary(gold_rows, os.path.splitext(filename)[0] + '_gold_summary.csv') # logger.info(f"Number of submissions: {len(worker_list)}") report_file = os.path.splitext(filename)[0] + '_data_cleaning_report.csv' From f10d149fa0d37d65a3964ac05b2b83604dfa8b5a Mon Sep 17 00:00:00 2001 From: Babak Naderi Date: Mon, 6 Jul 2026 17:15:41 +0200 Subject: [PATCH 029/111] Add --gold_overrides: correct gold-clip answers/variance per item Add an optional --gold_overrides CSV (url + per-scale corrected answer and optional _var). For P.804 gold checks, listed clips use the override values and variance; clips not listed keep the answers encoded in the answers CSV. Document the workflow (review gold_summary, then override) in the agent. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/agents/analyze-results.agent.md | 7 +++ src/result_parser.py | 79 +++++++++++++++++++++++-- 2 files changed, 82 insertions(+), 4 deletions(-) diff --git a/.github/agents/analyze-results.agent.md b/.github/agents/analyze-results.agent.md index 0595c8b..3e7075e 100644 --- a/.github/agents/analyze-results.agent.md +++ b/.github/agents/analyze-results.agent.md @@ -206,6 +206,13 @@ per scale `_expected` (correct answer, blank if not targeted), `max_wrong_pct`; when the mean rating is far from the expected answer with a high wrong rate, the gold clip is likely mis-keyed or too hard on that scale. +**Correcting gold answers**: to override mis-keyed gold clips, build a CSV with a +`url` column plus, per scale, `` (corrected correct answer) and optional +`_var` (variance), and pass it as `--gold_overrides`. Listed clips use these +values; clips not listed keep the encoded answers. A good starting point per scale +is the rounded `_mean` for scales with a high `_wrong_pct`; review +before re-running. + #### 4c. Summary to present Provide the user with a structured summary: diff --git a/src/result_parser.py b/src/result_parser.py index a029f3f..4c7d898 100644 --- a/src/result_parser.py +++ b/src/result_parser.py @@ -311,6 +311,68 @@ def decode_answer(url, encoded): +gold_overrides = {} + + +def load_gold_overrides(path): + """ + Load a CSV of corrected gold-clip answers/variance, keyed by clip URL. + + Each row is one gold clip ``url`` plus, per scale, an overriding correct + answer (column ````) and optional variance (column ``_var``). + Empty cells fall back to the encoded answer / config variance, and any gold + clip whose URL is absent from the file keeps the values encoded in the + answers CSV. + + :param path: Path to the overrides CSV. + :return: Dict of {url: {column: value}} keeping only non-empty cells. + """ + df = pd.read_csv(path, dtype=str) + overrides = {} + for _, r in df.iterrows(): + url = str(r.get('url', '')).strip() + if not url or url.lower() == 'nan': + continue + vals = {} + for k, v in r.items(): + if k == 'url' or v is None: + continue + v = str(v).strip() + if v != '' and v.lower() != 'nan': + vals[k] = v + overrides[url] = vals + logger.info(f"Loaded gold answer overrides for {len(overrides)} clip(s).") + return overrides + + +def resolve_gold_answer(gq_url, item, encoded_correct_ans, default_var): + """ + Return the correct answer and variance for a gold scale, applying overrides. + + If ``gq_url`` has an override for ``item``, that value (and its ``_var`` + when present) is used; otherwise the answer is decoded from the answers CSV + and the default (config) variance applies. + + :param gq_url: The gold clip URL. + :param item: The scale name (e.g. "loud"). + :param encoded_correct_ans: The encoded answer from the answers CSV. + :param default_var: The variance to use when not overridden. + :return: Tuple of (correct answer as int or None, variance as int). + """ + override = gold_overrides.get(gq_url) + if override is not None and override.get(item) not in (None, ''): + raw_var = override.get(f'{item}_var') + try: + var = int(float(raw_var)) if raw_var not in (None, '') else default_var + except (TypeError, ValueError): + var = default_var + try: + return int(float(override[item])), var + except (TypeError, ValueError): + pass + return decode_answer(gq_url, encoded_correct_ans), default_var + + def check_person_rec_qualification(row): correct_ans ={'dist1':'N', 'dist2':'N', 'dist3':'Y', 'dist4':'N', 'dist5':'Y'} pref = 'answer.' @@ -463,7 +525,7 @@ def check_gold_question_P804(method, row): for item in items: # check for all subdimensions encoded_correct_ans = row["input.gold_"+item+"_ans"] - decodec_correct_ans = decode_answer(gq_url, encoded_correct_ans) + decodec_correct_ans, item_var = resolve_gold_answer(gq_url, item, encoded_correct_ans, gq_var) rec[item] = decodec_correct_ans rec[f'{item}_given'] = row[f"answer.{q_name}_{item}"] #print("correct ans:", decodec_correct_ans) @@ -473,7 +535,7 @@ def check_gold_question_P804(method, row): #print('Given ans:'+ans) #print(row['assignmentid']) if (decodec_correct_ans is not None) and (int(ans) not in range( - decodec_correct_ans - gq_var, decodec_correct_ans + gq_var + 1)): + decodec_correct_ans - item_var, decodec_correct_ans + item_var + 1)): correct_gq = 0 given_ans_report.append(f"{item}: "+ans) wrong += 1 @@ -521,14 +583,14 @@ def check2_gold_questions_P804(method, row): for item in items: # check for all subdimensions encoded_correct_ans = row["input.gold_"+item+f"_ans{pf}"] - decodec_correct_ans = decode_answer(gq_url, encoded_correct_ans) + decodec_correct_ans, item_var = resolve_gold_answer(gq_url, item, encoded_correct_ans, gq_var) rec[f'{item}_{pf}'] = decodec_correct_ans rec[f'{item}_{pf}_given'] = row[f"answer.{q_name}_{item}"] ans = row[f"answer.{q_name}_{item}"] if (decodec_correct_ans is not None) and ( len(ans)==0 or (int(float(ans)) not in range( - decodec_correct_ans - gq_var, decodec_correct_ans + gq_var + 1))): + decodec_correct_ans - item_var, decodec_correct_ans + item_var + 1))): correct_gq = 0 given_ans_report.append(f"{item}_{pf}: "+ans) wrong += 1 @@ -2386,6 +2448,12 @@ def analyze_results(config, test_method, answer_path,prolific_ans_path, list_of_ "Prolific does not include the reward in its export, so provide it here to " "compute payment-per-hour statistics. Optional; takes precedence over --rewards." , required=False, default=None) + parser.add_argument( + '--gold_overrides', required=False, default=None, + help="Optional path to a CSV of corrected gold-clip answers. Columns: url plus, per " + "scale, (corrected correct answer) and optional _var (variance). " + "Listed clips use these values as correct; clips not listed keep the answers " + "encoded in the answers CSV.") #parser.add_argument('--adc' , help="name of Advance Data Cleaning script. If set, the answers will be filtered by that as well", default=None) args = parser.parse_args() @@ -2439,5 +2507,8 @@ def analyze_results(config, test_method, answer_path,prolific_ans_path, list_of_ logger.addHandler(console_handler) logger.addHandler(file_handler) logger.info(f"Start analyzing the results of {test_method} test") + if args.gold_overrides is not None: + assert os.path.exists(args.gold_overrides), f"No gold overrides file at [{args.gold_overrides}]" + gold_overrides = load_gold_overrides(args.gold_overrides) # start analyze_results(config, test_method, answer_path, prolific_ans_path, list_of_req, args.quality_bonus, platform) \ No newline at end of file From e15a7732ee11cd4bc078ed34a934c501a8192d9a Mon Sep 17 00:00:00 2001 From: Babak Naderi Date: Mon, 6 Jul 2026 17:46:23 +0200 Subject: [PATCH 030/111] gold_overrides: a listed clip is authoritative; blank scale is skipped Change the override semantics so that when a gold clip URL is present in the --gold_overrides file, that row fully defines its gold answers: filled scales are checked with the given value/variance, and blank scales are skipped (not checked) instead of falling back to the encoded answer. Clips not listed keep the encoded answers. Update the help text and agent doc. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/agents/analyze-results.agent.md | 11 ++++---- src/result_parser.py | 35 +++++++++++++++---------- 2 files changed, 27 insertions(+), 19 deletions(-) diff --git a/.github/agents/analyze-results.agent.md b/.github/agents/analyze-results.agent.md index 3e7075e..088af8f 100644 --- a/.github/agents/analyze-results.agent.md +++ b/.github/agents/analyze-results.agent.md @@ -207,11 +207,12 @@ per scale `_expected` (correct answer, blank if not targeted), wrong rate, the gold clip is likely mis-keyed or too hard on that scale. **Correcting gold answers**: to override mis-keyed gold clips, build a CSV with a -`url` column plus, per scale, `` (corrected correct answer) and optional -`_var` (variance), and pass it as `--gold_overrides`. Listed clips use these -values; clips not listed keep the encoded answers. A good starting point per scale -is the rounded `_mean` for scales with a high `_wrong_pct`; review -before re-running. +`url` column plus, per scale, `` (correct answer) and optional `_var` +(variance), and pass it as `--gold_overrides`. A listed clip is authoritative: the +scales you fill are checked with those values and blank scales are skipped (not +checked); clips not listed keep the encoded answers. A good starting point per +scale is the rounded `_mean` for scales with a high `_wrong_pct`; +review before re-running. #### 4c. Summary to present diff --git a/src/result_parser.py b/src/result_parser.py index 4c7d898..8650750 100644 --- a/src/result_parser.py +++ b/src/result_parser.py @@ -318,11 +318,11 @@ def load_gold_overrides(path): """ Load a CSV of corrected gold-clip answers/variance, keyed by clip URL. - Each row is one gold clip ``url`` plus, per scale, an overriding correct - answer (column ````) and optional variance (column ``_var``). - Empty cells fall back to the encoded answer / config variance, and any gold - clip whose URL is absent from the file keeps the values encoded in the - answers CSV. + Each row is one gold clip ``url`` plus, per scale, the correct answer (column + ````) and optional variance (column ``_var``). A listed clip is + authoritative: provided scale values are used, and a blank scale is skipped + (that scale is not checked). Any gold clip whose URL is absent from the file + keeps the values encoded in the answers CSV. :param path: Path to the overrides CSV. :return: Dict of {url: {column: value}} keeping only non-empty cells. @@ -349,27 +349,34 @@ def resolve_gold_answer(gq_url, item, encoded_correct_ans, default_var): """ Return the correct answer and variance for a gold scale, applying overrides. - If ``gq_url`` has an override for ``item``, that value (and its ``_var`` - when present) is used; otherwise the answer is decoded from the answers CSV + If ``gq_url`` is listed in the overrides file, that row is authoritative: a + provided value for ``item`` (with its ``_var`` when present) is used, + while a blank scale is skipped by returning ``None`` (so that scale is not + checked). If the URL is not listed, the answer is decoded from the answers CSV and the default (config) variance applies. :param gq_url: The gold clip URL. :param item: The scale name (e.g. "loud"). :param encoded_correct_ans: The encoded answer from the answers CSV. :param default_var: The variance to use when not overridden. - :return: Tuple of (correct answer as int or None, variance as int). + :return: Tuple of (correct answer as int or None, variance as int); a None + answer means the scale should be skipped. """ override = gold_overrides.get(gq_url) - if override is not None and override.get(item) not in (None, ''): + if override is not None: + # listed clip: the row fully defines its gold answers + val = override.get(item) + if val in (None, ''): + return None, default_var # blank scale -> skip (do not check) raw_var = override.get(f'{item}_var') try: var = int(float(raw_var)) if raw_var not in (None, '') else default_var except (TypeError, ValueError): var = default_var try: - return int(float(override[item])), var + return int(float(val)), var except (TypeError, ValueError): - pass + return None, default_var return decode_answer(gq_url, encoded_correct_ans), default_var @@ -2451,9 +2458,9 @@ def analyze_results(config, test_method, answer_path,prolific_ans_path, list_of_ parser.add_argument( '--gold_overrides', required=False, default=None, help="Optional path to a CSV of corrected gold-clip answers. Columns: url plus, per " - "scale, (corrected correct answer) and optional _var (variance). " - "Listed clips use these values as correct; clips not listed keep the answers " - "encoded in the answers CSV.") + "scale, (correct answer) and optional _var (variance). A listed " + "clip is authoritative: provided scales are checked with these values and blank " + "scales are skipped. Clips not listed keep the answers encoded in the answers CSV.") #parser.add_argument('--adc' , help="name of Advance Data Cleaning script. If set, the answers will be filtered by that as well", default=None) args = parser.parse_args() From 4c7f81bbddf3a987e14d8db7bbcac282ca213327 Mon Sep 17 00:00:00 2001 From: Babak Naderi Date: Mon, 6 Jul 2026 18:18:38 +0200 Subject: [PATCH 031/111] Order votes_per_cond_all-scales columns per scale The condition all-scales report sorted columns alphabetically (95%CI_*, MOS_*, std_* grouped by metric). Order them per scale instead: condition_name, n, M, then MOS/std/95%CI grouped together for each scale in scale order. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/result_parser.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/result_parser.py b/src/result_parser.py index 8650750..9d1b2ae 100644 --- a/src/result_parser.py +++ b/src/result_parser.py @@ -2390,8 +2390,14 @@ def analyze_results(config, test_method, answer_path,prolific_ans_path, list_of_ merged.to_csv(os.path.splitext(answer_path)[0]+ f"_votes_per_clip_all-scales.csv", index=False) if use_condition_level: - merged_cond.sort_index(inplace = True, axis = 1) merged_cond['M'] = ((merged_cond['MOS_SIG']-1) / 4 + (merged_cond['MOS_OVRL']-1) /4 ) / 2 + # group columns per scale (MOS, std, 95%CI together) rather than alphabetically + ordered = ['condition_name', 'n', 'M'] + for item in suffixes: + ordered += [f'MOS{item.upper()}', f'std{item}', f'95%CI{item}'] + ordered = [c for c in ordered if c in merged_cond.columns] + \ + [c for c in merged_cond.columns if c not in ordered] + merged_cond = merged_cond[ordered] merged_cond.to_csv(os.path.splitext(answer_path)[0]+ f"_votes_per_cond_all-scales.csv", index=False) if platform == "prolific": From 4d682ed556b8e3d618485d3bd7366bbcb248e9e4 Mon Sep 17 00:00:00 2001 From: Babak Naderi Date: Mon, 6 Jul 2026 18:38:37 +0200 Subject: [PATCH 032/111] Carry Prolific status through result_parser into the review file; use it result_parser now keeps the Prolific export 'Status' column (renamed prolific_status), carries it through the merge and per-submission records, and writes it into *_accept_reject_gui.csv. prolific_utils' review step now reads prolific_status from that file instead of fetching each submission's status from the API. This removes the paginated study-submissions fetch (which could loop on Prolific's _links.next) and makes the review fast. Approvals: AWAITING REVIEW bulk-approved, RETURNED/TIMED-OUT approved individually (else manual-payment CSV); reject/return only for AWAITING REVIEW. Falls back to treating all as AWAITING REVIEW if the column is absent. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/prolific_utils.py | 98 +++++++++---------------------------------- src/result_parser.py | 12 ++++-- 2 files changed, 29 insertions(+), 81 deletions(-) diff --git a/src/prolific_utils.py b/src/prolific_utils.py index f8eec58..2805f93 100644 --- a/src/prolific_utils.py +++ b/src/prolific_utils.py @@ -160,62 +160,6 @@ def get_submission_status(assignment_id): return None -def get_study_id_for_submission(assignment_id): - """ - Resolve the Prolific study id that a submission belongs to. - - :param assignment_id: A Prolific submission id. - :return: The study id string, or None if it could not be determined. - """ - data = get_submission_data(assignment_id) - if data: - return data.get('study_id') or data.get('study') - return None - - -def fetch_submission_status_map(study_id): - """ - Fetch the current status of every submission in a study. - - Pages through the study's submissions once and returns a dict mapping the - lower-cased submission id to its Prolific status (e.g. "AWAITING REVIEW", - "APPROVED", "RETURNED"). This lets the review step act only on submissions - still awaiting review, instead of one status GET per submission. - - :param study_id: The Prolific study id. - :return: Dict of {submission_id (lower-case): status}. - """ - status_map = {} - url = f"{base_url}/studies/{study_id}/submissions/" - headers = { - 'Authorization': f'Token {api_token}', - 'Content-Type': 'application/json', - 'Accept': 'application/json', - } - while url: - try: - response = requests.get(url, headers=headers, timeout=30) - except requests.exceptions.RequestException as e: - logger.info(f"Error listing submissions for study {study_id}: {e}") - break - if response.status_code != 200: - logger.info(f"Error listing submissions: {response.status_code} - {response.text}") - break - data = response.json() - for s in data.get('results', []): - sid = s.get('id') - if sid is not None: - status_map[str(sid).strip().lower()] = s.get('status') - # follow pagination: top-level 'next' or _links.next - next_url = data.get('next') - if not next_url: - links_next = data.get('_links', {}).get('next') - next_url = links_next.get('href') if isinstance(links_next, dict) else links_next - url = next_url - logger.info(f"Fetched status for {len(status_map)} submissions in study {study_id}.") - return status_map - - def approve_submission(assignment_id): """ Approve a single submission via the transition endpoint. @@ -252,25 +196,23 @@ def send_reviews_for_study(csv_data_path, detailed_data_cleaning_report=None, bl df_approved = df[df['Approve'] == 'x'] df_rejected = df[df['Approve'] != 'x'] - # Fetch the current status of every submission once. Prolific only allows - # approving/rejecting/requesting-return of submissions that are still - # AWAITING REVIEW (bulk-approve even 400s the whole batch if one id is not), - # so on a re-run we must skip anything already approved/rejected/returned. - status_map = {} - all_ids = df['assignmentId'].dropna().astype(str).str.strip().str.lower().tolist() - if all_ids: - study_id = get_study_id_for_submission(all_ids[0]) - if study_id: - status_map = fetch_submission_status_map(study_id) - if not status_map: - logger.warning("Could not fetch study submission statuses in bulk; " - "falling back to a per-submission status check.") - - def _status_of(aid): - st = status_map.get(aid) - if st is None and aid not in status_map: - st = get_submission_status(aid) # fallback for ids missing from the bulk map - return str(st).strip().upper() if st is not None else None + # The current Prolific status is carried in the review file (prolific_status column, + # populated by result_parser from the Prolific export). Prolific only allows + # approving/rejecting/requesting-return of AWAITING REVIEW submissions (bulk-approve + # even 400s the whole batch if one id is not), so we act based on that status and + # skip anything already approved/rejected/returned. + has_status = 'prolific_status' in df.columns + if not has_status: + logger.warning("No 'prolific_status' column in the review file; treating all " + "submissions as AWAITING REVIEW. Re-run result_parser to add it.") + + def _status_of(row): + if not has_status: + return "AWAITING REVIEW" + val = row.get('prolific_status') + if pd.isna(val) or str(val).strip() == "": + return None # unknown for this row + return str(val).strip().upper() # ---- approvals: pay for every accepted submission whose work we use ---- # AWAITING REVIEW go through bulk-approve; RETURNED/TIMED-OUT can't be @@ -281,7 +223,7 @@ def _status_of(aid): for _, row in df_approved.iterrows(): aid = str(row['assignmentId']).strip().lower() wid = str(row['WorkerId']).strip().lower() - st = _status_of(aid) + st = _status_of(row) if st == "AWAITING REVIEW": submission_to_approve.append(aid) elif st == "APPROVED": @@ -311,8 +253,8 @@ def _status_of(aid): continue # already handled in bulk approve # Only submissions still AWAITING REVIEW can be rejected or asked to return; skip the # rest (e.g. already RETURNED/REJECTED/APPROVED from a previous review run). - if _status_of(assignment_id) != "AWAITING REVIEW": - logger.info(f"Submission {assignment_id} skipped (status: {status_map.get(assignment_id)}); " + if _status_of(row) != "AWAITING REVIEW": + logger.info(f"Submission {assignment_id} skipped (status: {row.get('prolific_status') if has_status else 'n/a'}); " f"only AWAITING REVIEW submissions are rejected/asked to return.") n_skipped += 1 continue diff --git a/src/result_parser.py b/src/result_parser.py index 9d1b2ae..6fa57f4 100644 --- a/src/result_parser.py +++ b/src/result_parser.py @@ -1005,6 +1005,8 @@ def data_cleaning(filename, method, wrong_vcodes): d['HITId'] = row['hitid'] d['assignment'] = row['assignmentid'] d['status'] = row['assignmentstatus'] + # Prolific submission status (from the export), carried through for the review step + d['prolific_status'] = row.get('prolific_status', '') if 'prolific_status' in row else '' d['ip'] = row['x-real-ip'] if 'x-real-ip' in row else None d['study_url'] = row['study_url'] if 'study_url' in row else None @@ -1292,7 +1294,10 @@ def save_approve_rejected_ones_for_gui(data, path, wrong_vcodes): """ df = pd.DataFrame(data) df = df[df.status == 'Submitted'] - small_df = df[['worker_id','assignment', 'HITId', 'Approve', 'Reject']].copy() + gui_cols = ['worker_id', 'assignment', 'HITId', 'Approve', 'Reject'] + if 'prolific_status' in df.columns: + gui_cols.append('prolific_status') + small_df = df[gui_cols].copy() small_df.rename(columns={'assignment': 'assignmentId', 'worker_id':'WorkerId'}, inplace=True) if wrong_vcodes is not None: @@ -2088,7 +2093,7 @@ def combine_prolific_hit_server(prolific_ans_path, hitapp_ans_path): columns_to_remove = prolific_ans.columns.difference(['Submission id','Participant id', 'Completion code', 'Country of birth', 'Country of residence', 'Ethnicity simplified', 'Language', - 'Nationality', 'Primary language', 'Sex', 'Time taken', 'Total approvals', 'URL']) + 'Nationality', 'Primary language', 'Sex', 'Time taken', 'Total approvals', 'URL', 'Status']) prolific_ans.drop(columns=columns_to_remove, inplace=True) @@ -2115,7 +2120,8 @@ def combine_prolific_hit_server(prolific_ans_path, hitapp_ans_path): 'Time taken': "WorkTimeInSeconds", 'Submission id':'prolific_submission_id', 'Total approvals':'prolific_total_approvals', - 'URL':'study_url'}, inplace=True) + 'URL':'study_url', + 'Status':'prolific_status'}, inplace=True) # mark prolific_ans to remove when study_url is nan and lenght of Answer.v_code is not 32 # (abandoned/returned submissions). Keep any row whose submission id matches a completed From 98aa53f6fb1d5e075f0e43588bfa96d3c426416e Mon Sep 17 00:00:00 2001 From: Babak Naderi Date: Mon, 6 Jul 2026 18:44:30 +0200 Subject: [PATCH 033/111] Guard RDP group assignment against missing remote_desktop_failed column send_reviews_for_study crashed with KeyError when the data cleaning report has no 'remote_desktop_failed' column (studies without RDP detection), which also prevented the block-list group assignment from running. Skip the RDP step with a log note when the column is absent. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/prolific_utils.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/src/prolific_utils.py b/src/prolific_utils.py index 2805f93..ef7da3a 100644 --- a/src/prolific_utils.py +++ b/src/prolific_utils.py @@ -282,13 +282,17 @@ def _status_of(row): # assign participants with rdp to the group to be excluded from the future studies if detailed_data_cleaning_report and rdp_group_id is not None: df_detailed = pd.read_csv(detailed_data_cleaning_report) - # filter the participants with rdp - df_rdp = df_detailed[df_detailed['remote_desktop_failed'] == 1] - participant_ids = df_rdp['worker_id'].tolist() - if len(participant_ids) > 0: - unique_participant_ids = list(set(participant_ids)) - logger.info(f"Adding {len(unique_participant_ids)} participants with rdp to the group") - add_participants_to_group(rdp_group_id, unique_participant_ids) + if 'remote_desktop_failed' in df_detailed.columns: + # filter the participants with rdp + df_rdp = df_detailed[df_detailed['remote_desktop_failed'] == 1] + participant_ids = df_rdp['worker_id'].tolist() + if len(participant_ids) > 0: + unique_participant_ids = list(set(participant_ids)) + logger.info(f"Adding {len(unique_participant_ids)} participants with rdp to the group") + add_participants_to_group(rdp_group_id, unique_participant_ids) + else: + logger.info("No 'remote_desktop_failed' column in the data cleaning report; " + "skipping RDP group assignment.") if block_report is not None and low_quality_group_id is not None: # if block report is provided, save the participants with rdp to the block report From 23915d529ad57e6d9ea227813b4637b44a3a5657 Mon Sep 17 00:00:00 2001 From: Babak Naderi Date: Mon, 6 Jul 2026 19:38:02 +0200 Subject: [PATCH 034/111] Improve gold-clip generation: decouple loudness from OVRL, delay loudness/coloration, force reference review - Loudness alone no longer flags the overall (ovrl) dimension; only the loud scale is targeted. Combined loudness+distortion / loudness+noise still flag ovrl via the other artifact. - Loudness and coloration degradations now keep a clean-reference prefix (GOLD_CLEAN_PREFIX_SEC, default 3s) and apply only afterwards, with a short crossfade; other artifacts (noise, distortion) still apply from the start and persist throughout when combined. - create-study agent now requires the user to review candidate clean reference clips (keep only clips they'd rate 5 on all scales) before gold generation. - Update docs/gold_clips.md P.804 table and notes accordingly. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/agents/create-study.agent.md | 28 ++++++++++++++ docs/gold_clips.md | 12 +++++- src/create_gold_clips.py | 55 ++++++++++++++++++++++++---- 3 files changed, 87 insertions(+), 8 deletions(-) diff --git a/.github/agents/create-study.agent.md b/.github/agents/create-study.agent.md index e037b66..9d5b17d 100644 --- a/.github/agents/create-study.agent.md +++ b/.github/agents/create-study.agent.md @@ -132,6 +132,9 @@ Do not guess these values if they are missing: - Can they identify clean clips by a URL pattern (e.g. `*/clean/*`, `*/reference/*`)? - Would they like the agent to download a small subset of rating clips for the user to **listen to and manually remove any clips with distortion** before gold generation? + **Mandatory**: however the candidate clean clips are obtained, they **must** go through + the user review in section 5a (keep only clips the user would rate 5 on all scales) + before gold generation. This step is required and must not be skipped. **Important**: never use the sample clips bundled in this repository (`src\test_inputs\`). Source clips must come from the same dataset as the rating clips. - **Trapping clips**: if `trapping_clips.csv` is not provided, the quality of source @@ -439,6 +442,31 @@ If downloading clips from Azure private storage, either: **How many source clips?** Use `BEST_PRACTICE_GOLD_SOURCE_COUNT` capped at `BEST_PRACTICE_MAX_GOLD_SOURCE_CLIPS`. +#### 5a. MANDATORY — user review of clean reference clips before generation + +The clean reference clips are the foundation of every gold question: the generator assumes +each source clip is **perfect** and would be rated **5 on all scales**. If a reference clip +has any audible flaw, every gold clip derived from it will have a wrong expected answer and +will wrongly fail workers. Therefore this review step is **required** and must not be skipped, +regardless of how the candidate clips were obtained. + +**You must:** + +1. Copy the candidate clean/reference clips into a temporary review directory, e.g. + `RATING_CLIPS_PATH\gold_source_review`. +2. **[ASK]** Instruct the user, in these exact terms, to review them: + - "Please listen to every clip in `gold_source_review`." + - "**Keep only the clips you would personally rate 5 on ALL scales** + (coloration, discontinuity, loudness, noise, reverb, signal, and overall)." + - "If a clip is not a perfect 5 on every scale, **delete that file** from the folder." + - "Do not edit the clips — only keep or delete them." +3. **Wait for the user to explicitly confirm** they have finished reviewing and deleting. + Do not proceed on assumption. +4. After confirmation, use **only the clips that remain** in the review directory as the + gold source (copy the survivors into `RATING_CLIPS_PATH\gold_source`). If the user + deleted everything, or too few remain, ask for more candidate clips and repeat — do not + fall back to unreviewed clips. + Generate gold clips (filenames are **anonymized** by default — do **not** use `--no_anonymize`): diff --git a/docs/gold_clips.md b/docs/gold_clips.md index 9b55984..856227a 100644 --- a/docs/gold_clips.md +++ b/docs/gold_clips.md @@ -92,13 +92,23 @@ of 1 are listed in the CSV; empty cells mean the dimension is not targeted (impl | Coloration | Resonant/muffled/telephone filter | 1 | | | | 1 | 1 | | Coloration + noise | Coloration + noise | | | | 1 | 1 | 1 | | Distortion + noise | Clipping + noise | | | | 1 | 1 | 1 | -| Loudness | Too loud (+25 dB) or too quiet (-25 dB) | | | 1 | | | 1 | +| Loudness | Too loud (+25 dB) or too quiet (-25 dB) | | | 1 | | | | | Loudness + distortion | Loudness + clipping | | | 1 | | 1 | 1 | | Loudness + noise | Loudness + noise | | | 1 | 1 | | 1 | **Note:** When distortion is combined with noise, only `sig` and `noise` are flagged because the specific type of underlying distortion is not clearly distinguishable to raters. +**Note:** Loudness alone does not flag `ovrl`. A level offset by itself is not treated as an overall +quality degradation, so only the `loud` dimension is targeted. When loudness is combined with another +artifact (distortion or noise), that other artifact drives the `ovrl` flag. + +**Note:** The loudness and coloration degradations keep the first few seconds of audio as a clean +reference (`GOLD_CLEAN_PREFIX_SEC`, default 3 seconds) and only apply the degradation afterwards, so a +rater can perceive the change relative to the clean start. Other artifacts (noise, distortion, +discontinuity) are applied from the beginning of the clip; when combined with loudness or coloration, +the other artifact is present throughout while the loudness/coloration change appears after the prefix. + **Output CSV columns:** `gold_clips`, `col_ans`, `disc_ans`, `loud_ans`, `noise_ans`, `reverb_ans`, `sig_ans`, `ovrl_ans` ## Source Clip Recommendations diff --git a/src/create_gold_clips.py b/src/create_gold_clips.py index a895579..a84d0e4 100644 --- a/src/create_gold_clips.py +++ b/src/create_gold_clips.py @@ -43,6 +43,11 @@ # any degradation (or none, for the clean/score-5 case) is applied. GOLD_SOURCE_TARGET_DBOV = -26.0 +# Loudness and coloration degradations keep the first this-many seconds free of the +# degradation (a clean reference), then apply it. Other artifacts (noise, distortion, +# discontinuity) are applied from the beginning of the clip. +GOLD_CLEAN_PREFIX_SEC = 3.0 + GOLD_TYPES = { 'clean': { 'suffix': 'clean', @@ -89,7 +94,7 @@ }, 'loudness': { 'suffix': 'loud', - 'p804': {'loud_ans': 1, 'ovrl_ans': 1}, + 'p804': {'loud_ans': 1}, }, 'loudness_distortion': { 'suffix': 'loud_distorted', @@ -386,6 +391,35 @@ def _apply_random_post_processing(signal, sr, gold_type): return signal +def _apply_delayed(base, degraded_full, sr, delay_sec=GOLD_CLEAN_PREFIX_SEC, fade_ms=30.0): + """ + Keep the first ``delay_sec`` seconds of ``base`` and switch to ``degraded_full`` + afterwards, with a short crossfade to avoid a click at the boundary. + + Used so that loudness/coloration degradations only appear after an initial + clean-reference portion, while any other artifact already present in ``base`` + (e.g. noise or distortion) continues throughout. + + :param base: The signal without the delayed degradation (may already contain + other artifacts). Used for the clean prefix. + :param degraded_full: The full-length signal with the delayed degradation applied. + :param sr: Sample rate in Hz. + :param delay_sec: Length of the clean prefix in seconds. + :param fade_ms: Crossfade length in milliseconds at the switch point. + :return: The combined signal as a numpy array. + """ + n = int(delay_sec * sr) + # clip too short to hold the clean prefix -> apply the degradation to all of it + if n >= len(base): + return degraded_full + result = np.concatenate([base[:n], degraded_full[n:]]).astype(float) + fade = min(int(fade_ms / 1000.0 * sr), len(base) - n) + if fade > 0: + ramp = np.linspace(0.0, 1.0, fade) + result[n:n + fade] = base[n:n + fade] * (1.0 - ramp) + degraded_full[n:n + fade] * ramp + return result + + def process_clip(signal, sr, gold_type, snr_db=-5.0, clip_threshold=0.005): """ Apply the specified degradation type to an audio signal, followed by @@ -413,19 +447,26 @@ def process_clip(signal, sr, gold_type, snr_db=-5.0, clip_threshold=0.005): result = apply_discontinuity(signal, sr) result = add_background_noise(result, sr, snr_db) elif gold_type == 'coloration': - result = apply_coloration(signal, sr) + # coloration starts after the clean-reference prefix + result = _apply_delayed(signal, apply_coloration(signal, sr), sr) elif gold_type == 'coloration_noise': - result = apply_coloration(signal, sr) - result = add_background_noise(result, sr, snr_db) + # noise from the beginning; coloration delayed after the clean prefix + base = add_background_noise(signal, sr, snr_db) + result = _apply_delayed(base, apply_coloration(base, sr), sr) elif gold_type == 'distortion_noise': result = apply_signal_distortion(signal, clip_threshold) result = add_background_noise(result, sr, snr_db) elif gold_type == 'loudness': - result = apply_loudness(signal, sr) + # loudness starts after the clean-reference prefix + result = _apply_delayed(signal, apply_loudness(signal, sr), sr) elif gold_type == 'loudness_distortion': - result = apply_loudness(apply_signal_distortion(signal, clip_threshold), sr) + # distortion from the beginning; loudness delayed after the clean prefix + base = apply_signal_distortion(signal, clip_threshold) + result = _apply_delayed(base, apply_loudness(base, sr), sr) elif gold_type == 'loudness_noise': - result = apply_loudness(add_background_noise(signal, sr, snr_db), sr) + # noise from the beginning; loudness delayed after the clean prefix + base = add_background_noise(signal, sr, snr_db) + result = _apply_delayed(base, apply_loudness(base, sr), sr) else: raise ValueError(f"Unknown gold type: {gold_type}") From b467c521a6c3d683a41072f1a5b7edf06f9d82c5 Mon Sep 17 00:00:00 2001 From: Babak Naderi Date: Mon, 6 Jul 2026 21:58:36 +0200 Subject: [PATCH 035/111] Fix Prolific bulk approver leaving submissions unapproved Prolific's bulk-approve endpoint is asynchronous (HTTP 200 only means the batch was accepted, not completed) and rejects the whole batch if any id is not currently AWAITING REVIEW. A single pass therefore left many submissions unapproved, so re-running approved only 'a few more' each time but never all. - bulk_approve_submission now verifies against live status and retries: it filters the target list to ids still AWAITING REVIEW before each bulk send (avoiding the whole-batch rejection from stale ids), waits for async processing with a growing back-off, and finally approves any stragglers individually via the synchronous transition endpoint. Returns the ids it could not approve. - Add fetch_submission_status_map() using page-based pagination with a stable ordering (started_at) and paginate-until-404, cross-checked against meta.count and retried once if short. The server-side status filter is ignored and _links.next loops forever, so neither is used; without stable ordering pages duplicate/skip rows (this caused earlier miscounts). - Add get_study_id_for_submission() helper. - send_reviews_for_study now reports how many were actually approved vs. failed and writes a prolific_approve_failed_*.csv listing any that could not be approved. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/prolific_utils.py | 260 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 242 insertions(+), 18 deletions(-) diff --git a/src/prolific_utils.py b/src/prolific_utils.py index ef7da3a..213c106 100644 --- a/src/prolific_utils.py +++ b/src/prolific_utils.py @@ -14,6 +14,7 @@ import datetime import glob import logging +import time base_url = "https://api.prolific.com/api/v1" @@ -65,32 +66,246 @@ def _message_to_prolific(message): logger.info(f"Response: {response.text}") -def bulk_approve_submission(assignment_ids): +def _norm_status(value): + """ + Normalize a Prolific submission status string for comparison. + + :param value: Raw status value (may be None). + :return: Upper-cased status with underscores turned into spaces, or "" if empty. + """ + if value is None: + return "" + return str(value).strip().upper().replace("_", " ") - url = f"{base_url}/submissions/bulk-approve/" +def get_study_id_for_submission(assignment_id): + """ + Look up the parent study id for a single submission. + + :param assignment_id: A Prolific submission id. + :return: The study id string, or None if it could not be determined. + """ + url = f"{base_url}/submissions/{assignment_id}/" headers = { 'Authorization': f'Token {api_token}', - 'Content-Type': 'application/json', + 'Content-Type': 'application/json', 'Accept': 'application/json', - } - - payload = { - "submission_ids": assignment_ids - - } - try: - response = requests.post(url, headers=headers, data=json.dumps(payload), timeout=30) + response = requests.get(url, headers=headers, timeout=30) except requests.exceptions.RequestException as e: - logger.info(f"Submission {assignment_ids} - An error occurred: {e}") - return + logger.info(f"Submission {assignment_id} - study lookup error: {e}") + return None if response.status_code == 200: - logger.info(f"{len(assignment_ids)} Submission approved successfully.") + data = response.json() + return data.get("study_id") or data.get("study") + logger.info(f"Could not look up study for submission {assignment_id} " + f"(status {response.status_code}).") + return None + + +def fetch_submission_status_map(study_id, page_size=100, max_pages=500): + """ + Fetch the current status of every submission in a study. + + Uses page-based pagination (the ``page``/``page_size`` query parameters) with a + stable ``ordering`` and tallies the ``status`` field client-side. Several quirks of + the endpoint are worked around: + + - the server-side ``status`` filter is silently ignored, so filtering is done here; + - the ``_links.next`` link loops forever, so it is not followed; + - without a stable ``ordering`` the same submission can appear on several pages + while others are skipped, so ``ordering=started_at`` is always sent; + - a page can occasionally come back short mid-stream, so pagination continues + until the API returns 404/empty rather than stopping on a short page. + + The result is cross-checked against the ``meta.count`` reported by the API and the + fetch is retried once if it comes back short. + + :param study_id: The Prolific study id. + :param page_size: Number of submissions requested per page (the server caps it at 100). + :param max_pages: Safety cap on the number of pages fetched. + :return: A dict mapping submission id -> normalized status string. + """ + if not study_id: + return {} + url = f"{base_url}/submissions/" + headers = { + 'Authorization': f'Token {api_token}', + 'Content-Type': 'application/json', + 'Accept': 'application/json', + } + + def _one_pass(): + status_map = {} + expected = None + page = 1 + while page <= max_pages: + params = {"study": study_id, "page_size": page_size, "page": page, + "ordering": "started_at"} + try: + response = requests.get(url, headers=headers, params=params, timeout=60) + except requests.exceptions.RequestException as e: + logger.info(f"Study {study_id} submission list error on page {page}: {e}") + break + # a 404 signals we have paged past the last page; stop quietly + if response.status_code == 404: + break + if response.status_code != 200: + logger.info(f"Study {study_id} submission list error on page {page} " + f"(status {response.status_code}): {response.text}") + break + body = response.json() + if expected is None: + expected = (body.get("meta") or {}).get("count") + results = body.get("results", []) + if not results: + break + for s in results: + status_map[s.get("id")] = _norm_status(s.get("status")) + page += 1 + return status_map, expected + + status_map, expected = _one_pass() + # retry once if the API told us how many to expect and we came up short + if expected is not None and len(status_map) < expected: + logger.info(f"Study {study_id}: fetched {len(status_map)}/{expected} submissions; " + f"retrying the status fetch once.") + retry_map, _ = _one_pass() + status_map.update(retry_map) + return status_map + + +def _bulk_approve_request(assignment_ids, max_batch_size=500, pause_between_batches=2.0): + """ + Send bulk-approve requests to Prolific in sequential batches. + + Prolific processes bulk approvals asynchronously (a 200 response means the batch + was accepted, not that it has completed) and recommends at most 1000 ids per + request sent sequentially to avoid wallet contention. Callers must verify + completion afterwards; see ``bulk_approve_submission``. + + :param assignment_ids: List of Prolific submission ids to approve. + :param max_batch_size: Maximum number of ids per request. + :param pause_between_batches: Seconds to wait between batches. + :return: None + """ + url = f"{base_url}/submissions/bulk-approve/" + headers = { + 'Authorization': f'Token {api_token}', + 'Content-Type': 'application/json', + 'Accept': 'application/json', + } + total_batches = (len(assignment_ids) + max_batch_size - 1) // max_batch_size + for batch_index, start_idx in enumerate(range(0, len(assignment_ids), max_batch_size), start=1): + batch_ids = assignment_ids[start_idx:start_idx + max_batch_size] + payload = {"submission_ids": batch_ids} + try: + response = requests.post(url, headers=headers, data=json.dumps(payload), timeout=60) + except requests.exceptions.RequestException as e: + logger.info(f"Bulk-approve batch {batch_index}/{total_batches} ({len(batch_ids)} ids) - " + f"an error occurred: {e}") + continue + if response.status_code == 200: + logger.info(f"Bulk-approve batch {batch_index}/{total_batches}: {len(batch_ids)} " + f"submission(s) accepted (processing asynchronously).") + else: + logger.info(f"Bulk-approve batch {batch_index}/{total_batches} error: " + f"{response.status_code} - {response.text}") + if pause_between_batches and batch_index < total_batches: + time.sleep(pause_between_batches) + + +def bulk_approve_submission(assignment_ids, study_id=None, max_retries=6, poll_wait=12.0): + """ + Approve submissions in Prolific, verifying completion and retrying stragglers. + + Prolific's bulk-approve endpoint is asynchronous: a 200 response only means the + batch was accepted, and the whole batch is rejected if any id is not currently + AWAITING REVIEW. A single pass therefore often leaves many submissions unapproved, + which is why re-running the review approves "a few more" each time but never all. + This function instead: + + 1. reads the live study status and keeps only ids still AWAITING REVIEW + (skipping ids Prolific has already approved/returned since the export, which + would otherwise cause a whole-batch rejection), + 2. sends the pending ids via bulk-approve, + 3. waits, re-reads the live status, and retries the still-pending ids with a + growing back-off, and + 4. finally approves any remaining stragglers one-by-one via the synchronous + transition endpoint. + + :param assignment_ids: List of Prolific submission ids to approve. + :param study_id: The study the submissions belong to; looked up from the first + submission if not provided. + :param max_retries: Maximum number of bulk-approve + verify passes. + :param poll_wait: Base seconds to wait for asynchronous processing before verifying; + the wait grows on each retry. + :return: List of submission ids that could not be approved. + """ + if not assignment_ids: + logger.info("No submissions provided for bulk approval.") + return [] + + # normalize and de-duplicate while preserving order + target = list(dict.fromkeys(str(a).strip() for a in assignment_ids if str(a).strip())) + + if study_id is None: + study_id = get_study_id_for_submission(target[0]) + if study_id is None: + logger.warning("Could not determine study id for bulk approval; falling back to " + "per-submission verification (slower).") + + def _still_awaiting(ids): + """ + Return the subset of ids whose current Prolific status is AWAITING REVIEW. + + :param ids: List of submission ids to check. + :return: The subset of ids still awaiting review. + """ + if not ids: + return [] + status_map = fetch_submission_status_map(study_id) if study_id else {} + if status_map: + return [aid for aid in ids if status_map.get(aid) == "AWAITING REVIEW"] + # no study-wide map available: verify each id individually + return [aid for aid in ids + if _norm_status(get_submission_status(aid)) == "AWAITING REVIEW"] + + # Only ids currently AWAITING REVIEW can be bulk-approved; filtering up-front avoids a + # whole-batch rejection caused by ids Prolific has already resolved since the export. + pending = _still_awaiting(target) + n_already = len(target) - len(pending) + if n_already: + logger.info(f"{n_already}/{len(target)} targeted submission(s) already resolved " + f"(not awaiting review); {len(pending)} to approve.") + to_approve = len(pending) + + for attempt in range(1, max_retries + 1): + if not pending: + break + _bulk_approve_request(pending) + # bulk approval is asynchronous; give it time to settle, with a growing back-off + time.sleep(min(poll_wait * attempt, 60.0)) + pending = _still_awaiting(pending) + logger.info(f"Bulk-approve pass {attempt}/{max_retries}: " + f"{to_approve - len(pending)}/{to_approve} approved, " + f"{len(pending)} still awaiting review.") + + # synchronous fallback for anything still awaiting review after the retries + failures = [] + if pending: + logger.info(f"Approving {len(pending)} remaining submission(s) individually " + f"via the synchronous endpoint.") + for aid in pending: + if not approve_submission(aid): + failures.append(aid) + + if failures: + logger.warning(f"{len(failures)} submission(s) could not be approved: {failures}") else: - logger.info(f"Error: {response.status_code}") - logger.info(f"Response: {response.text}") + logger.info(f"Bulk approval complete: {to_approve} submission(s) approved.") + return failures def ask_return(assignment_id, reason): @@ -238,8 +453,9 @@ def _status_of(row): manual_payment_rows.append({"WorkerId": wid, "assignmentId": aid, "status": st, "reason": "used but not in an approvable state"}) + approve_failures = [] if submission_to_approve: - bulk_approve_submission(submission_to_approve) + approve_failures = bulk_approve_submission(submission_to_approve) n_actioned = 0 n_skipped = 0 @@ -273,11 +489,19 @@ def _status_of(row): logger.info(f"{len(manual_payment_rows)} used submission(s) could not be auto-paid " f"(returned/timed-out); listed for manual payment in {manual_path}") - logger.info(f"Review complete: {len(submission_to_approve)} bulk-approved, " + logger.info(f"Review complete: {len(submission_to_approve) - len(approve_failures)} approved, " + f"{len(approve_failures)} could not be approved, " f"{n_already_approved} already approved, " f"{len(manual_payment_rows)} need manual payment, " f"{n_actioned} {'rejected' if args.force_reject else 'asked to return'}, " f"{n_skipped} reject/return-skipped (not awaiting review).") + if approve_failures: + stamp = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S") + out_dir = os.path.dirname(csv_data_path) + fail_path = os.path.join(out_dir, f"prolific_approve_failed_{stamp}.csv") + pd.DataFrame({"assignmentId": approve_failures}).to_csv(fail_path, index=False) + logger.warning(f"{len(approve_failures)} submission(s) could not be approved; " + f"listed in {fail_path}") # assign participants with rdp to the group to be excluded from the future studies if detailed_data_cleaning_report and rdp_group_id is not None: From 89014e582c970c19628f03008ad035c7824d2e2e Mon Sep 17 00:00:00 2001 From: Babak Naderi Date: Tue, 7 Jul 2026 11:31:18 +0200 Subject: [PATCH 036/111] Make individual approval resilient to transient rate limiting The synchronous fallback in bulk_approve_submission fired individual approvals back-to-back, which sporadically failed with transient rate-limit errors even though the submissions were still AWAITING REVIEW. - approve_submission now retries on HTTP 429/5xx with a growing back-off. - The bulk-approve fallback loop now spaces individual approvals ~1s apart. Verified against live r2/r3 reviews: the 8+2 submissions that first failed all approved cleanly on a spaced retry. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/prolific_utils.py | 41 ++++++++++++++++++++++++++++++----------- 1 file changed, 30 insertions(+), 11 deletions(-) diff --git a/src/prolific_utils.py b/src/prolific_utils.py index 213c106..4e118d5 100644 --- a/src/prolific_utils.py +++ b/src/prolific_utils.py @@ -300,6 +300,8 @@ def _still_awaiting(ids): for aid in pending: if not approve_submission(aid): failures.append(aid) + # space the calls out so we don't trip Prolific's rate limiting + time.sleep(1.0) if failures: logger.warning(f"{len(failures)} submission(s) could not be approved: {failures}") @@ -375,14 +377,19 @@ def get_submission_status(assignment_id): return None -def approve_submission(assignment_id): +def approve_submission(assignment_id, max_attempts=4, backoff=3.0): """ Approve a single submission via the transition endpoint. Used for accepted submissions that cannot go through bulk-approve because they - are not AWAITING REVIEW (e.g. RETURNED or TIMED-OUT but the work was used). + are not AWAITING REVIEW (e.g. RETURNED or TIMED-OUT but the work was used) and as + the synchronous fallback for bulk approvals. Retries on transient errors (HTTP 429 + rate limiting or 5xx) with a growing back-off, since firing many approvals in quick + succession can otherwise fail sporadically. :param assignment_id: The Prolific submission id. + :param max_attempts: Maximum number of attempts on transient errors. + :param backoff: Base seconds to wait between attempts; grows with each retry. :return: True if the submission was approved, False otherwise. """ url = f"{base_url}/submissions/{assignment_id}/transition/" @@ -392,16 +399,28 @@ def approve_submission(assignment_id): 'Accept': 'application/json', } payload = {"action": "APPROVE"} - try: - response = requests.post(url, headers=headers, data=json.dumps(payload), timeout=10) - except requests.exceptions.RequestException as e: - logger.info(f"Submission {assignment_id} - approve error: {e}") + for attempt in range(1, max_attempts + 1): + try: + response = requests.post(url, headers=headers, data=json.dumps(payload), timeout=30) + except requests.exceptions.RequestException as e: + logger.info(f"Submission {assignment_id} - approve error: {e}") + if attempt < max_attempts: + time.sleep(backoff * attempt) + continue + return False + if response.status_code == 200: + logger.info(f"Submission {assignment_id} approved individually.") + return True + # 429 (rate limit) and 5xx are transient: wait and retry + if response.status_code == 429 or response.status_code >= 500: + if attempt < max_attempts: + logger.info(f"Submission {assignment_id} approve transient error " + f"(status {response.status_code}); retrying.") + time.sleep(backoff * attempt) + continue + logger.info(f"Submission {assignment_id} could not be approved " + f"(status {response.status_code}): {response.text}") return False - if response.status_code == 200: - logger.info(f"Submission {assignment_id} approved individually.") - return True - logger.info(f"Submission {assignment_id} could not be approved " - f"(status {response.status_code}): {response.text}") return False From 91e548eaf979c9217f41d61c558d5b227c8b2b06 Mon Sep 17 00:00:00 2001 From: Babak Naderi Date: Tue, 7 Jul 2026 13:07:58 +0200 Subject: [PATCH 037/111] Stop result parser on duplicate URLs in gold overrides file load_gold_overrides silently kept only the last row for a repeated gold clip URL (dict overwrite), so a duplicated/conflicting entry could go unnoticed. It now detects duplicate URLs up-front and raises a clear error listing them, asking the user to proof and fix the file before re-running. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/result_parser.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/result_parser.py b/src/result_parser.py index 6fa57f4..97eb57a 100644 --- a/src/result_parser.py +++ b/src/result_parser.py @@ -324,10 +324,30 @@ def load_gold_overrides(path): (that scale is not checked). Any gold clip whose URL is absent from the file keeps the values encoded in the answers CSV. + Each gold clip URL must appear at most once. If any URL is duplicated the values + would be ambiguous (only the last row would take effect), so this stops with an + error and asks the user to proof and fix the file. + :param path: Path to the overrides CSV. :return: Dict of {url: {column: value}} keeping only non-empty cells. + :raises Exception: If the file contains duplicate gold clip URLs. """ df = pd.read_csv(path, dtype=str) + # collect the non-blank URLs first so duplicates can be detected before use + urls = [] + for _, r in df.iterrows(): + url = str(r.get('url', '')).strip() + if not url or url.lower() == 'nan': + continue + urls.append(url) + counts = collections.Counter(urls) + duplicates = {u: c for u, c in counts.items() if c > 1} + if duplicates: + listing = "\n ".join(f"{u} (appears {c} times)" for u, c in duplicates.items()) + raise Exception( + f"Duplicate gold clip URL(s) found in the gold overrides file [{path}]:\n " + f"{listing}\nEach gold clip must appear at most once. Please proof the file, " + f"remove or merge the duplicate row(s), and re-run.") overrides = {} for _, r in df.iterrows(): url = str(r.get('url', '')).strip() From 8f33280d2036d5edac8815a410c7ed19882bdb9e Mon Sep 17 00:00:00 2001 From: Babak Naderi Date: Tue, 7 Jul 2026 14:36:14 +0200 Subject: [PATCH 038/111] Do not set expected SIG for gold clips in high-noise/low-loudness cases In high noise the signal is masked and in a very quiet clip signal detail is hidden, so the sig dimension cannot be judged reliably. Drop sig_ans from the noise-combined types (discontinuity/coloration/distortion + noise) and from loudness + distortion. Standalone signal_distortion, discontinuity, and coloration (no noise/loudness) still target sig. The ovrl flag is retained, driven by the noise or loudness/distortion degradation. Update docs/gold_clips.md table and notes accordingly. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/gold_clips.md | 14 ++++++++------ src/create_gold_clips.py | 8 ++++---- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/docs/gold_clips.md b/docs/gold_clips.md index 856227a..6dec501 100644 --- a/docs/gold_clips.md +++ b/docs/gold_clips.md @@ -88,16 +88,18 @@ of 1 are listed in the CSV; empty cells mean the dimension is not targeted (impl | Background noise | Pink noise | | | | 1 | | 1 | | Signal distortion | Hard clipping | | | | | 1 | 1 | | Discontinuity | Random segment dropouts (choppy) | | 1 | | | 1 | 1 | -| Discontinuity + noise | Choppy + noise | | | | 1 | 1 | 1 | +| Discontinuity + noise | Choppy + noise | | | | 1 | | 1 | | Coloration | Resonant/muffled/telephone filter | 1 | | | | 1 | 1 | -| Coloration + noise | Coloration + noise | | | | 1 | 1 | 1 | -| Distortion + noise | Clipping + noise | | | | 1 | 1 | 1 | +| Coloration + noise | Coloration + noise | | | | 1 | | 1 | +| Distortion + noise | Clipping + noise | | | | 1 | | 1 | | Loudness | Too loud (+25 dB) or too quiet (-25 dB) | | | 1 | | | | -| Loudness + distortion | Loudness + clipping | | | 1 | | 1 | 1 | +| Loudness + distortion | Loudness + clipping | | | 1 | | | 1 | | Loudness + noise | Loudness + noise | | | 1 | 1 | | 1 | -**Note:** When distortion is combined with noise, only `sig` and `noise` are flagged because the -specific type of underlying distortion is not clearly distinguishable to raters. +**Note:** In high-noise and low-loudness cases the `sig` dimension cannot be judged reliably (the noise +masks the signal, and a very quiet clip hides signal detail), so `sig` is not flagged for the +noise-combined types (discontinuity/coloration/distortion + noise) or for loudness + distortion. In +these cases the `ovrl` flag is still driven by the noise or loudness/distortion degradation. **Note:** Loudness alone does not flag `ovrl`. A level offset by itself is not treated as an overall quality degradation, so only the `loud` dimension is targeted. When loudness is combined with another diff --git a/src/create_gold_clips.py b/src/create_gold_clips.py index a84d0e4..9cc3f6b 100644 --- a/src/create_gold_clips.py +++ b/src/create_gold_clips.py @@ -78,7 +78,7 @@ }, 'discontinuity_noise': { 'suffix': 'choppy_noisy', - 'p804': {'sig_ans': 1, 'noise_ans': 1, 'ovrl_ans': 1}, + 'p804': {'noise_ans': 1, 'ovrl_ans': 1}, }, 'coloration': { 'suffix': 'colored', @@ -86,11 +86,11 @@ }, 'coloration_noise': { 'suffix': 'colored_noisy', - 'p804': {'sig_ans': 1, 'noise_ans': 1, 'ovrl_ans': 1}, + 'p804': {'noise_ans': 1, 'ovrl_ans': 1}, }, 'distortion_noise': { 'suffix': 'distorted_noisy', - 'p804': {'sig_ans': 1, 'noise_ans': 1, 'ovrl_ans': 1}, + 'p804': {'noise_ans': 1, 'ovrl_ans': 1}, }, 'loudness': { 'suffix': 'loud', @@ -98,7 +98,7 @@ }, 'loudness_distortion': { 'suffix': 'loud_distorted', - 'p804': {'loud_ans': 1, 'sig_ans': 1, 'ovrl_ans': 1}, + 'p804': {'loud_ans': 1, 'ovrl_ans': 1}, }, 'loudness_noise': { 'suffix': 'loud_noisy', From c881d26706ab4eab8d98219ad993ff0faa8600b0 Mon Sep 17 00:00:00 2001 From: Babak Naderi Date: Tue, 7 Jul 2026 16:59:00 +0200 Subject: [PATCH 039/111] Split gold loudness into too-loud and too-quiet variants Loudness issues come in two kinds (too loud / too quiet) and only the too-quiet (low-gain) case hides signal detail. The single random loudness types are replaced with deterministic high (too loud) and low (too quiet) variants: - loudness_high / loudness_low: loud only. - loudness_high_distortion: loud, sig, ovrl (distortion still audible when loud). - loudness_low_distortion: loud, ovrl (sig dropped; low gain hides signal detail). - loudness_high_noise / loudness_low_noise: loud, noise, ovrl (noise masks sig). apply_loudness is now called with an explicit target level per variant, so generation is deterministic instead of randomly loud/quiet. Update docs table and the sig note. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/gold_clips.md | 20 +++++++++------ src/create_gold_clips.py | 54 +++++++++++++++++++++++++++++----------- 2 files changed, 51 insertions(+), 23 deletions(-) diff --git a/docs/gold_clips.md b/docs/gold_clips.md index 6dec501..239618f 100644 --- a/docs/gold_clips.md +++ b/docs/gold_clips.md @@ -92,14 +92,18 @@ of 1 are listed in the CSV; empty cells mean the dimension is not targeted (impl | Coloration | Resonant/muffled/telephone filter | 1 | | | | 1 | 1 | | Coloration + noise | Coloration + noise | | | | 1 | | 1 | | Distortion + noise | Clipping + noise | | | | 1 | | 1 | -| Loudness | Too loud (+25 dB) or too quiet (-25 dB) | | | 1 | | | | -| Loudness + distortion | Loudness + clipping | | | 1 | | | 1 | -| Loudness + noise | Loudness + noise | | | 1 | 1 | | 1 | - -**Note:** In high-noise and low-loudness cases the `sig` dimension cannot be judged reliably (the noise -masks the signal, and a very quiet clip hides signal detail), so `sig` is not flagged for the -noise-combined types (discontinuity/coloration/distortion + noise) or for loudness + distortion. In -these cases the `ovrl` flag is still driven by the noise or loudness/distortion degradation. +| Loudness (too loud) | Gain raised so speech is too loud | | | 1 | | | | +| Loudness (too quiet) | Gain lowered so speech is too quiet | | | 1 | | | | +| Loudness (too loud) + distortion | Too loud + clipping | | | 1 | | 1 | 1 | +| Loudness (too quiet) + distortion | Too quiet + clipping | | | 1 | | | 1 | +| Loudness (too loud) + noise | Too loud + noise | | | 1 | 1 | | 1 | +| Loudness (too quiet) + noise | Too quiet + noise | | | 1 | 1 | | 1 | + +**Note:** The `sig` dimension cannot be judged reliably when the signal is masked or hidden, so it is not +flagged for: the noise-combined types (discontinuity/coloration/distortion + noise, where noise masks the +signal) and the **too-quiet** loudness + distortion case (a very low-gain clip hides signal detail). The +**too-loud** loudness + distortion case still flags `sig`, since the distortion remains audible. In all +these cases the `ovrl` flag is retained, driven by the noise or loudness/distortion degradation. **Note:** Loudness alone does not flag `ovrl`. A level offset by itself is not treated as an overall quality degradation, so only the `loud` dimension is targeted. When loudness is combined with another diff --git a/src/create_gold_clips.py b/src/create_gold_clips.py index 9cc3f6b..ae0787d 100644 --- a/src/create_gold_clips.py +++ b/src/create_gold_clips.py @@ -92,16 +92,29 @@ 'suffix': 'distorted_noisy', 'p804': {'noise_ans': 1, 'ovrl_ans': 1}, }, - 'loudness': { - 'suffix': 'loud', + 'loudness_high': { + 'suffix': 'too_loud', 'p804': {'loud_ans': 1}, }, - 'loudness_distortion': { - 'suffix': 'loud_distorted', + 'loudness_low': { + 'suffix': 'too_quiet', + 'p804': {'loud_ans': 1}, + }, + 'loudness_high_distortion': { + 'suffix': 'too_loud_distorted', + 'p804': {'loud_ans': 1, 'sig_ans': 1, 'ovrl_ans': 1}, + }, + 'loudness_low_distortion': { + # too quiet (low gain): signal detail is hidden, so sig is not judged + 'suffix': 'too_quiet_distorted', 'p804': {'loud_ans': 1, 'ovrl_ans': 1}, }, - 'loudness_noise': { - 'suffix': 'loud_noisy', + 'loudness_high_noise': { + 'suffix': 'too_loud_noisy', + 'p804': {'loud_ans': 1, 'noise_ans': 1, 'ovrl_ans': 1}, + }, + 'loudness_low_noise': { + 'suffix': 'too_quiet_noisy', 'p804': {'loud_ans': 1, 'noise_ans': 1, 'ovrl_ans': 1}, }, } @@ -456,17 +469,28 @@ def process_clip(signal, sr, gold_type, snr_db=-5.0, clip_threshold=0.005): elif gold_type == 'distortion_noise': result = apply_signal_distortion(signal, clip_threshold) result = add_background_noise(result, sr, snr_db) - elif gold_type == 'loudness': - # loudness starts after the clean-reference prefix - result = _apply_delayed(signal, apply_loudness(signal, sr), sr) - elif gold_type == 'loudness_distortion': - # distortion from the beginning; loudness delayed after the clean prefix + elif gold_type == 'loudness_high': + # too loud, applied after the clean-reference prefix + result = _apply_delayed(signal, apply_loudness(signal, sr, LOUDNESS_TOO_LOUD_DBOV), sr) + elif gold_type == 'loudness_low': + # too quiet (low gain), applied after the clean-reference prefix + result = _apply_delayed(signal, apply_loudness(signal, sr, LOUDNESS_TOO_QUIET_DBOV), sr) + elif gold_type == 'loudness_high_distortion': + # distortion from the beginning; too-loud gain delayed after the clean prefix + base = apply_signal_distortion(signal, clip_threshold) + result = _apply_delayed(base, apply_loudness(base, sr, LOUDNESS_TOO_LOUD_DBOV), sr) + elif gold_type == 'loudness_low_distortion': + # distortion from the beginning; too-quiet gain delayed after the clean prefix base = apply_signal_distortion(signal, clip_threshold) - result = _apply_delayed(base, apply_loudness(base, sr), sr) - elif gold_type == 'loudness_noise': - # noise from the beginning; loudness delayed after the clean prefix + result = _apply_delayed(base, apply_loudness(base, sr, LOUDNESS_TOO_QUIET_DBOV), sr) + elif gold_type == 'loudness_high_noise': + # noise from the beginning; too-loud gain delayed after the clean prefix + base = add_background_noise(signal, sr, snr_db) + result = _apply_delayed(base, apply_loudness(base, sr, LOUDNESS_TOO_LOUD_DBOV), sr) + elif gold_type == 'loudness_low_noise': + # noise from the beginning; too-quiet gain delayed after the clean prefix base = add_background_noise(signal, sr, snr_db) - result = _apply_delayed(base, apply_loudness(base, sr), sr) + result = _apply_delayed(base, apply_loudness(base, sr, LOUDNESS_TOO_QUIET_DBOV), sr) else: raise ValueError(f"Unknown gold type: {gold_type}") From 22a99be194c2218426fdd4ac7d87b96392e59ab2 Mon Sep 17 00:00:00 2001 From: Babak Naderi Date: Tue, 7 Jul 2026 18:07:43 +0200 Subject: [PATCH 040/111] Make gold post-processing bumps subtle so degradations stay perceptible The anti-fingerprint Gaussian bumps in _apply_random_post_processing used an amplitude of 3-8x the signal peak, which dominated the audio: after RMS restoration the speech became quiet and thumpy, and the intended degradation (coloration especially) was masked. On a colored clip this pushed the crest factor to ~49 and muffled even the clean reference prefix. Reduce the bump amplitude to 0.05-0.2x the peak. This keeps the crest-factor/ kurtosis perturbation for fingerprint blurring while restoring the intended perceptual character: the clean 3s prefix stays clean and the coloration is clearly audible after it (verified: colored crest ~18 vs ~49, prefix centroid ~2980 vs after-3s ~1260). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/create_gold_clips.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/create_gold_clips.py b/src/create_gold_clips.py index ae0787d..1a08192 100644 --- a/src/create_gold_clips.py +++ b/src/create_gold_clips.py @@ -358,13 +358,15 @@ def _apply_random_post_processing(signal, sr, gold_type): if gold_type in ('signal_distortion', 'both', 'distortion_noise', 'discontinuity', 'discontinuity_noise', 'coloration', 'coloration_noise'): - # Add smooth Gaussian-windowed bumps to raise crest factor and kurtosis - # without creating audible clicks + # Add smooth Gaussian-windowed bumps to subtly raise crest factor and kurtosis + # without creating audible clicks or masking the intended degradation. The + # amplitude is a small fraction of the signal peak; larger values (previously + # several times the peak) dominated the audio and hid the real degradation. n_bumps = np.random.randint(8, 25) peak_val = np.max(np.abs(signal)) for _ in range(n_bumps): center = np.random.randint(0, len(signal)) - amp = np.random.uniform(3.0, 8.0) * peak_val + amp = np.random.uniform(0.05, 0.2) * peak_val sign = np.random.choice([-1, 1]) # Gaussian window width: 20–80 samples (~2.5–10 ms at 8kHz) width = np.random.randint(20, 80) From 9b5a432861b4ac82b33d1ac62f35b57691c55c16 Mon Sep 17 00:00:00 2001 From: Babak Naderi Date: Tue, 7 Jul 2026 18:12:13 +0200 Subject: [PATCH 041/111] Make gold coloration more extreme and sample-rate consistent Strengthen apply_coloration so the timbre change is clearly audible: remove the clean passthrough (mix_orig=0), raise the muffled/resonant mix, and use steeper (4th-order) low-pass filters. Specify all cut-offs in Hz and convert with the sample rate so the effect is consistent across 16/24/48 kHz (previously lp_cutoff was a raw Nyquist-normalized value, giving a different absolute cut-off per rate). Colored-clip spectral centroid now drops to ~450-880 Hz (was ~1200-1500), a clear muffled/telephone timbre, and is consistent across sample rates. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/create_gold_clips.py | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/src/create_gold_clips.py b/src/create_gold_clips.py index 1a08192..06d45f2 100644 --- a/src/create_gold_clips.py +++ b/src/create_gold_clips.py @@ -222,6 +222,10 @@ def apply_coloration(signal, sr): Simulate coloration (voice timbre change) by randomly applying one of three heavy coloration styles: muffled, resonant, or telephone effect. + Cut-offs are specified in Hz and converted with the sample rate so the effect is + consistent across sample rates (16/24/48 kHz). The styles are deliberately strong, + with no clean passthrough, so the timbre change is clearly audible. + :param signal: Audio signal as a numpy array. :param sr: Sample rate. :return: Colored signal as a numpy array. @@ -231,22 +235,27 @@ def apply_coloration(signal, sr): style = np.random.choice(['muffled_heavy', 'resonant_heavy', 'telephone']) if style == 'muffled_heavy': - center_freq, bandwidth, lp_cutoff = 1200, 200, 0.12 - mix_orig, mix_resonant, mix_muffled = 0.1, 0.3, 0.6 + # very muffled / "underwater": aggressive low-pass, no clean passthrough + center_freq_hz, bandwidth_hz, lp_cutoff_hz = 900, 180, 800 + mix_orig, mix_resonant, mix_muffled = 0.0, 0.2, 0.8 elif style == 'resonant_heavy': - center_freq, bandwidth, lp_cutoff = 1000, 150, 0.20 - mix_orig, mix_resonant, mix_muffled = 0.1, 0.7, 0.2 + # strong narrow-band resonance (hollow / tinny) + center_freq_hz, bandwidth_hz, lp_cutoff_hz = 1000, 120, 2200 + mix_orig, mix_resonant, mix_muffled = 0.0, 0.85, 0.15 else: # telephone - center_freq, bandwidth, lp_cutoff = 800, 200, 0.15 - mix_orig, mix_resonant, mix_muffled = 0.0, 0.6, 0.4 - - low = max(0.01, (center_freq - bandwidth / 2) / (sr / 2.0)) - high = min(0.99, (center_freq + bandwidth / 2) / (sr / 2.0)) - high = max(low + 0.01, high) + # band-limited "old telephone" timbre + center_freq_hz, bandwidth_hz, lp_cutoff_hz = 750, 180, 1600 + mix_orig, mix_resonant, mix_muffled = 0.0, 0.7, 0.3 + + nyquist = sr / 2.0 + low = max(0.001, (center_freq_hz - bandwidth_hz / 2.0) / nyquist) + high = min(0.999, (center_freq_hz + bandwidth_hz / 2.0) / nyquist) + high = max(low + 0.001, high) b_bp, a_bp = butter(4, [low, high], btype='band') resonant = lfilter(b_bp, a_bp, signal) - b_lp, a_lp = butter(3, lp_cutoff, btype='low') + lp_norm = min(0.999, lp_cutoff_hz / nyquist) + b_lp, a_lp = butter(4, lp_norm, btype='low') muffled = lfilter(b_lp, a_lp, signal) colored = mix_orig * signal + mix_resonant * resonant + mix_muffled * muffled From dc2721cd114f7240a2dfdeaa27fd905758399283 Mon Sep 17 00:00:00 2001 From: Babak Naderi Date: Tue, 7 Jul 2026 18:15:12 +0200 Subject: [PATCH 042/111] Apply coloration before adding noise for realistic colored+noise gold Previously coloration_noise added noise first and then coloured the already-noisy signal, which also colours the noise and sounds unrealistic. Now the speech is coloured first (after the clean prefix) and broadband noise is added on top, so the noise keeps its full bandwidth. Verified: the colored_noisy clip retains HF energy above 4 kHz (~0.015) while coloration alone removes it (~0.000). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/create_gold_clips.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/create_gold_clips.py b/src/create_gold_clips.py index 06d45f2..903d87a 100644 --- a/src/create_gold_clips.py +++ b/src/create_gold_clips.py @@ -474,9 +474,11 @@ def process_clip(signal, sr, gold_type, snr_db=-5.0, clip_threshold=0.005): # coloration starts after the clean-reference prefix result = _apply_delayed(signal, apply_coloration(signal, sr), sr) elif gold_type == 'coloration_noise': - # noise from the beginning; coloration delayed after the clean prefix - base = add_background_noise(signal, sr, snr_db) - result = _apply_delayed(base, apply_coloration(base, sr), sr) + # colour the speech first (delayed after the clean prefix), then add broadband + # noise on top, so the noise itself is not coloured (more realistic than + # colouring an already-noisy signal) + colored = _apply_delayed(signal, apply_coloration(signal, sr), sr) + result = add_background_noise(colored, sr, snr_db) elif gold_type == 'distortion_noise': result = apply_signal_distortion(signal, clip_threshold) result = add_background_noise(result, sr, snr_db) From dc2be4b12466cd5f213cff182cb74bde7554b381 Mon Sep 17 00:00:00 2001 From: Babak Naderi Date: Tue, 7 Jul 2026 18:17:56 +0200 Subject: [PATCH 043/111] Reduce gold clean-reference prefix from 3s to 2s GOLD_CLEAN_PREFIX_SEC 3.0 -> 2.0, so the loudness/coloration degradations start after a 2-second clean reference instead of 3. Update docs note. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/gold_clips.md | 2 +- src/create_gold_clips.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/gold_clips.md b/docs/gold_clips.md index 239618f..3b820a9 100644 --- a/docs/gold_clips.md +++ b/docs/gold_clips.md @@ -110,7 +110,7 @@ quality degradation, so only the `loud` dimension is targeted. When loudness is artifact (distortion or noise), that other artifact drives the `ovrl` flag. **Note:** The loudness and coloration degradations keep the first few seconds of audio as a clean -reference (`GOLD_CLEAN_PREFIX_SEC`, default 3 seconds) and only apply the degradation afterwards, so a +reference (`GOLD_CLEAN_PREFIX_SEC`, default 2 seconds) and only apply the degradation afterwards, so a rater can perceive the change relative to the clean start. Other artifacts (noise, distortion, discontinuity) are applied from the beginning of the clip; when combined with loudness or coloration, the other artifact is present throughout while the loudness/coloration change appears after the prefix. diff --git a/src/create_gold_clips.py b/src/create_gold_clips.py index 903d87a..5f32b66 100644 --- a/src/create_gold_clips.py +++ b/src/create_gold_clips.py @@ -46,7 +46,7 @@ # Loudness and coloration degradations keep the first this-many seconds free of the # degradation (a clean reference), then apply it. Other artifacts (noise, distortion, # discontinuity) are applied from the beginning of the clip. -GOLD_CLEAN_PREFIX_SEC = 3.0 +GOLD_CLEAN_PREFIX_SEC = 2.0 GOLD_TYPES = { 'clean': { From c74bb275bf5969d3c4f060376e3b2a58d8ba663d Mon Sep 17 00:00:00 2001 From: Babak Naderi Date: Thu, 9 Jul 2026 12:12:53 +0200 Subject: [PATCH 044/111] Rename 'Wrong Verification Code' to a clearer category These submissions are Prolific submissions with no matching completed HIT App task (the participant submitted on Prolific, often with a NOCODE/SCREENOUT completion code, but never completed the rating task). 'Wrong Verification Code' was misleading, so rename the category to 'Submitted on Prolific but no completed HIT App task' across the reject reason, block reason, feedback, per-worker log, and the not-accepted-reasons report; change the rejection-breakdown combo key to 'submitted_no_completed_hitapp_task'; and fix the 'verificatioon' typo. prolific_utils._message_to_prolific now matches the new reason (keeping the old 'wrong verification code' string for backward compatibility) so these still route to REJECT with category NO_CODE. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/prolific_utils.py | 2 +- src/result_parser.py | 26 ++++++++++++++------------ 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/src/prolific_utils.py b/src/prolific_utils.py index 4e118d5..ebe5106 100644 --- a/src/prolific_utils.py +++ b/src/prolific_utils.py @@ -30,7 +30,7 @@ def _message_to_prolific(message): "control clip incorrectly", "All clips should be played", "Both earplugs should be used.", "Qualification did not passed" ] - if "wrong verification code" in message: + if "no completed HIT App task" in message or "wrong verification code" in message: return "REJECT", "NO_CODE" if any(phrase in message for phrase in control_failed_phrases): return "REJECT", "FAILED_CHECK" diff --git a/src/result_parser.py b/src/result_parser.py index 97eb57a..cebfa0c 100644 --- a/src/result_parser.py +++ b/src/result_parser.py @@ -914,11 +914,11 @@ def report_rejection_breakdown(data, wrong_vcodes, answer_path): Reason semantics: content checks (``gold``, ``tps``, ``math``, ``all_audio_played``) come from the per-submission acceptance checks; ``performance`` and ``max_hits`` are attributed only when the submission would - otherwise have been accepted; ``wrong_verification_code`` submissions are - tracked separately. + otherwise have been accepted; submissions that reached Prolific but have no + completed HIT App task are tracked separately. :param data: List of per-submission dicts (worker_list) after all rejection steps. - :param wrong_vcodes: Dataframe of wrong-verification-code submissions, or None. + :param wrong_vcodes: Dataframe of submissions with no completed HIT App task, or None. :param answer_path: Path used to derive the output CSV file names. :return: None. """ @@ -934,9 +934,10 @@ def report_rejection_breakdown(data, wrong_vcodes, answer_path): if not reasons: reasons.add('other') combos.append(tuple(sorted(reasons))) - # wrong-verification-code submissions are tracked outside worker_list + # submissions that reached Prolific but have no completed HIT App task are + # tracked outside worker_list if wrong_vcodes is not None and len(wrong_vcodes) > 0: - combos.extend([('wrong_verification_code',)] * len(wrong_vcodes)) + combos.extend([('submitted_no_completed_hitapp_task',)] * len(wrong_vcodes)) total_rejected = len(combos) if total_rejected == 0: @@ -1323,7 +1324,7 @@ def save_approve_rejected_ones_for_gui(data, path, wrong_vcodes): if wrong_vcodes is not None: wrong_vcodes_assignments = wrong_vcodes[['WorkerId','AssignmentId', 'HITId']].copy() wrong_vcodes_assignments["Approve"] = "" - wrong_vcodes_assignments["Reject"] = "wrong verification code or incomplete submission" + wrong_vcodes_assignments["Reject"] = "Submitted on Prolific but no completed HIT App task" wrong_vcodes_assignments.rename(columns={'AssignmentId': 'assignmentId'}, inplace=True) small_df = pd.concat([small_df, wrong_vcodes_assignments], ignore_index=True) @@ -1368,7 +1369,7 @@ def save_block_list(block_list, path, wrong_v_code_freq): if wrong_v_code_freq is not None and len(wrong_v_code_freq) > 0: df2 = pd.DataFrame(wrong_v_code_freq, columns=['Worker ID']) df2['UPDATE BlockStatus'] = "Block" - df2['BlockReason'] = "Wrong verification code" + df2['BlockReason'] = "Submitted on Prolific but no completed HIT App task" # concat the two dataframes df = pd.concat([df, df2], ignore_index=True) save_csv(df, path, index=False) @@ -1377,12 +1378,12 @@ def save_block_list(block_list, path, wrong_v_code_freq): def check_wrong_vcode_should_block(wrong_vcodes): if wrong_vcodes is None: return [] - # count the number of wrong verification code per worker + # count the number of submissions with no completed HIT App task per worker small_df = wrong_vcodes[['WorkerId']].copy() grouped = small_df.groupby(['WorkerId']).size().reset_index(name='counts') - # get the workers that have more than 5 wrong verification code + # get the workers that have more than 5 submissions with no completed HIT App task grouped = grouped[grouped.counts >= 5] - logger.info(f"{len(grouped.index)} workers have more than 5 wrong verification code") + logger.info(f"{len(grouped.index)} workers have more than 5 submissions with no completed HIT App task") cheater_workers_list = list(grouped['WorkerId']) return cheater_workers_list @@ -1408,7 +1409,8 @@ def save_rejected_ones(data, path, wrong_vcodes, not_accepted_reasons, num_rej_p not_accepted_reasons_list = list(collections.Counter(not_accepted_reasons).items()) if wrong_vcodes is not None: - not_accepted_reasons_list.append(('Wrong Verification Code', len(wrong_vcodes.index))) + not_accepted_reasons_list.append( + ('Submitted on Prolific but no completed HIT App task', len(wrong_vcodes.index))) if num_rej_perform != 0: not_accepted_reasons_list.append(('Performance', num_rej_perform)) @@ -1419,7 +1421,7 @@ def save_rejected_ones(data, path, wrong_vcodes, not_accepted_reasons, num_rej_p small_df.rename(columns={'assignment': 'assignmentId', 'Reject': 'feedback'}, inplace=True) if wrong_vcodes is not None: wrong_vcodes_assignments = wrong_vcodes[['AssignmentId']].copy() - wrong_vcodes_assignments["feedback"] = "Wrong verificatioon code or incomplete submission" + wrong_vcodes_assignments["feedback"] = "Submitted on Prolific but no completed HIT App task" wrong_vcodes_assignments.rename(columns={'AssignmentId': 'assignmentId'}, inplace=True) small_df = pd.concat([small_df, wrong_vcodes_assignments], ignore_index=True) From b4335892a353b2c639e8a884c46cfa9cef5d5d41 Mon Sep 17 00:00:00 2001 From: Babak Naderi Date: Thu, 9 Jul 2026 15:17:08 +0200 Subject: [PATCH 045/111] Strip @author lines from generated study HTML The HIT-app templates carry an '@author' attribution comment that was copied into every generated study HTML published to crowd workers. master_script now removes any @author line during generation via a _strip_author_lines() helper, applied to all HTML generators (before Template rendering) and to the qualification page. The templates themselves keep their attribution. Verified: regenerated study HTML and preview contain 0 @author occurrences. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/master_script.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/master_script.py b/src/master_script.py index ff10fd8..2989c9a 100644 --- a/src/master_script.py +++ b/src/master_script.py @@ -11,6 +11,7 @@ import asyncio import base64 import random +import re import string import configparser as CP @@ -33,6 +34,19 @@ p835_personalized = "pp835" +def _strip_author_lines(text): + """ + Remove any line containing an ``@author`` tag from a template's text. + + Generated HIT-app HTML files are published to crowd workers, so personal author + attribution carried over from the templates is stripped out during generation. + + :param text: The template file content. + :return: The content with any line containing ``@author`` removed. + """ + return re.sub(r'(?m)^.*@author.*\r?\n?', '', text) + + def _get_cookie_value(cfg, key): """ Return the cookie value from cfg if available, otherwise generate a random string. @@ -71,6 +85,7 @@ def create_analyzer_cfg_acr(cfg, template_path, out_path): with open(template_path, 'r') as file: content = file.read() file.seek(0) + content = _strip_author_lines(content) t = Template(content) cfg_file = t.render(cfg=config) @@ -117,6 +132,7 @@ def create_analyzer_cfg_general(cfg, cfg_section, template_path, out_path, gener with open(template_path, 'r') as file: content = file.read() file.seek(0) + content = _strip_author_lines(content) t = Template(content) cfg_file = t.render(cfg=config) @@ -160,6 +176,7 @@ def create_analyzer_cfg_dcr_ccr(cfg, template_path, out_path, general_cfg, n_HIT with open(template_path, 'r') as file: content = file.read() file.seek(0) + content = _strip_author_lines(content) t = Template(content) cfg_file = t.render(cfg=config) @@ -225,6 +242,7 @@ async def create_hit_app_ccr_dcr(cfg, template_path, out_path, training_path, cf with open(template_path, 'r') as file: content = file.read() file.seek(0) + content = _strip_author_lines(content) t = Template(content) html = t.render(cfg=config) @@ -305,6 +323,7 @@ async def create_hit_app_acr(cfg, template_path, out_path, training_path, trap_p with open(template_path, 'r') as file: content = file.read() file.seek(0) + content = _strip_author_lines(content) t = Template(content) html = t.render(cfg=config) @@ -390,6 +409,7 @@ async def create_hit_app_p835(cfg, template_path, out_path, training_path, trap_ with open(template_path, 'r') as file: content = file.read() file.seek(0) + content = _strip_author_lines(content) t = Template(content) html = t.render(cfg=config) @@ -518,6 +538,7 @@ async def create_hit_app_pp835_p804( with open(template_path, "r", encoding="utf-8") as file: content = file.read() file.seek(0) + content = _strip_author_lines(content) t = Template(content) html = t.render(cfg=config) @@ -768,6 +789,7 @@ def create_qualification_only(args): template_path = os.path.join(os.path.dirname(__file__), "P808Template/Qualification.html") with open(template_path, "r", encoding="utf-8") as file: html = file.read() + html = _strip_author_lines(html) os.makedirs(args.project, exist_ok=True) out_path = os.path.join(args.project, f"{args.project}_qualification.html") From a6f34c15d56683d81433dbeb6e1fbbd5afc04e11 Mon Sep 17 00:00:00 2001 From: Babak Naderi Date: Thu, 9 Jul 2026 17:18:53 +0200 Subject: [PATCH 046/111] bw-check qualification: variable pool + role-based checking (Part 1 backend) Generalize the bandwidth-check qualification from a single hardcoded 5-clip set to a sampled pool, keeping full backward compatibility. - Resource CSVs gain comb_bw1..5 (clips), ans_comb_bw1..5 (band role wb/swb/fb/sq), and comb_bw_hash1..5 (sha256(url:dq|sq), same format as compute_math_hash): general.csv (public) keeps the existing hardcoded set; general_assets_internal.csv gets 46 role-labeled sets with shuffled positions. - create_input samples a whole 5-clip bw set per session (row-aligned) and writes the clips, roles, and hashes into the publish batch, mirroring math/math_ans/math_hash. - result_parser.check_qualification_answer now reads the per-position role from input.ans_comb_bw* and applies bw_min/bw_max by role (any position), falling back to the legacy fixed comb_bw1=wb/2=swb/3=fb/4,5=sq pattern when the batch has no roles. Verified: role-based pass/fail for FB and SWB, plus legacy fallback. Templates rendering the per-session clips + the client-side hash check and button come in Part 2; until then use the public general.csv set (batch == template). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/assets_master_script/general.csv | 38 +++++++++---------- src/create_input.py | 18 ++++++++- src/result_parser.py | 56 +++++++++++++++++++--------- 3 files changed, 75 insertions(+), 37 deletions(-) diff --git a/src/assets_master_script/general.csv b/src/assets_master_script/general.csv index da6e0a3..e3578db 100644 --- a/src/assets_master_script/general.csv +++ b/src/assets_master_script/general.csv @@ -1,19 +1,19 @@ -math,math_ans,math_hash,pair_a,pair_b,hearing_test_url,hearing_test_ans -https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/math/math1.wav,3,64e89493798b7fdab45235a9c52921b74e7ef071056dceb0f1e4b8d2c87ce1df,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_jnd/40S_female1.wav,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_jnd/50S_female1.wav,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_hearing_test/s1.wav,246 -https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/math/math2.wav,7,071343f6e8a48edc01ecfab7160d4da9501787007168e1cce2ee1345eb633426,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_jnd/40S_female2.wav,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_jnd/50S_female2.wav,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_hearing_test/s2.wav,626 -https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/math/math3.wav,6,d288c7e348c6b7ae478278f18d2ff94f0da0debddc89c7b10edcc216c1f6a576,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_jnd/40S_male1.wav,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_jnd/50S_male1.wav,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_hearing_test/s3.wav,802 -,,,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_jnd/40S_male2.wav,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_jnd/50S_male2.wav,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_hearing_test/s4.wav,913 -,,,,,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_hearing_test/s7.wav,135 -,,,,,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_hearing_test/s8.wav,156 -,,,,,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_hearing_test/s9.wav,282 -,,,,,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_hearing_test/s10.wav,286 -,,,,,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_hearing_test/s12.wav,340 -,,,,,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_hearing_test/s13.wav,359 -,,,,,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_hearing_test/s14.wav,401 -,,,,,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_hearing_test/s15.wav,468 -,,,,,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_hearing_test/s16.wav,534 -,,,,,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_hearing_test/s17.wav,591 -,,,,,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_hearing_test/s18.wav,628 -,,,,,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_hearing_test/s19.wav,680 -,,,,,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_hearing_test/s20.wav,815 -,,,,,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_hearing_test/s21.wav,962 +math,math_ans,math_hash,pair_a,pair_b,hearing_test_url,hearing_test_ans,comb_bw1,comb_bw2,comb_bw3,comb_bw4,comb_bw5,ans_comb_bw1,ans_comb_bw2,ans_comb_bw3,ans_comb_bw4,ans_comb_bw5,comb_bw_hash1,comb_bw_hash2,comb_bw_hash3,comb_bw_hash4,comb_bw_hash5 +https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/math/math1.wav,3,64e89493798b7fdab45235a9c52921b74e7ef071056dceb0f1e4b8d2c87ce1df,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_jnd/40S_female1.wav,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_jnd/50S_female1.wav,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_hearing_test/s1.wav,246,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/bw-test/d_g1_cmb.wav,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/bw-test/d_g2_cmb.wav,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/bw-test/d_g3_cmb.wav,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/bw-test/d_g4_cmb.wav,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/bw-test/d_g5_cmb.wav,wb,swb,fb,sq,sq,7d0a671e6a2f9d0bcdabfb26c2230a254f4303131af3b46914210dad33f80c29,b9b3344f87ae5dae9417ca7b313642cf72c57e13fd4a0d02162463c87edf9286,bcc2eb6b94b8740b2133520e7693041efc02e5e978e955c25fab6344c8f85e92,233a2f7b99881e96dcf266478a9cc4d99e120e93d0dd4c4a22fce090cdc7a50f,56e1c8e311d3d763f71f685f57ea3eb7e5a980575ebac90d905a1429c6791cdc +https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/math/math2.wav,7,071343f6e8a48edc01ecfab7160d4da9501787007168e1cce2ee1345eb633426,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_jnd/40S_female2.wav,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_jnd/50S_female2.wav,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_hearing_test/s2.wav,626,,,,,,,,,,,,,,, +https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/math/math3.wav,6,d288c7e348c6b7ae478278f18d2ff94f0da0debddc89c7b10edcc216c1f6a576,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_jnd/40S_male1.wav,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_jnd/50S_male1.wav,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_hearing_test/s3.wav,802,,,,,,,,,,,,,,, +,,,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_jnd/40S_male2.wav,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_jnd/50S_male2.wav,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_hearing_test/s4.wav,913,,,,,,,,,,,,,,, +,,,,,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_hearing_test/s7.wav,135,,,,,,,,,,,,,,, +,,,,,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_hearing_test/s8.wav,156,,,,,,,,,,,,,,, +,,,,,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_hearing_test/s9.wav,282,,,,,,,,,,,,,,, +,,,,,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_hearing_test/s10.wav,286,,,,,,,,,,,,,,, +,,,,,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_hearing_test/s12.wav,340,,,,,,,,,,,,,,, +,,,,,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_hearing_test/s13.wav,359,,,,,,,,,,,,,,, +,,,,,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_hearing_test/s14.wav,401,,,,,,,,,,,,,,, +,,,,,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_hearing_test/s15.wav,468,,,,,,,,,,,,,,, +,,,,,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_hearing_test/s16.wav,534,,,,,,,,,,,,,,, +,,,,,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_hearing_test/s17.wav,591,,,,,,,,,,,,,,, +,,,,,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_hearing_test/s18.wav,628,,,,,,,,,,,,,,, +,,,,,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_hearing_test/s19.wav,680,,,,,,,,,,,,,,, +,,,,,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_hearing_test/s20.wav,815,,,,,,,,,,,,,,, +,,,,,https://audiosamplesp808.blob.core.windows.net/p808-assets/clips/sample_hearing_test/s21.wav,962,,,,,,,,,,,,,,, diff --git a/src/create_input.py b/src/create_input.py index d36341b..a69700c 100644 --- a/src/create_input.py +++ b/src/create_input.py @@ -285,6 +285,22 @@ def create_input_for_acr(cfg, df, output_path, method): (n_sessions // math_hash_source.count()) + 1)[:n_sessions] output_df['math_hash'] = math_hash_output + # bandwidth-check sets: sample a whole 5-clip set per session so each session's + # clips (comb_bw*), per-position roles (ans_comb_bw*, checked server-side) and + # per-position hashes (comb_bw_hash*, used client-side) stay row-aligned. + bw_clip_cols = [f'comb_bw{i}' for i in range(1, 6)] + if all(c in df.columns for c in bw_clip_cols): + bw_set_cols = [c for c in df.columns + if c.startswith('comb_bw') or c.startswith('ans_comb_bw')] + bw_sets = df[bw_set_cols].dropna(subset=bw_clip_cols).reset_index(drop=True) + if len(bw_sets) > 0: + bw_idx = np.tile(np.arange(len(bw_sets)), + (n_sessions // len(bw_sets)) + 1)[:n_sessions] + np.random.shuffle(bw_idx) + bw_selected = bw_sets.iloc[bw_idx].reset_index(drop=True) + for c in bw_set_cols: + output_df[c] = bw_selected[c].to_numpy() + # CMPs: 4 pairs are needed for 1 session nPairs = 4 * n_sessions pair_a = df['pair_a'].dropna() @@ -397,7 +413,7 @@ def create_input_for_acr(cfg, df, output_path, method): tmp = df_small[['gold_url', 'gold_ovrl_ans', 'gold_sig_ans', 'gold_noise_ans', 'gold_col_ans', 'gold_loud_ans', 'gold_disc_ans', 'gold_reverb_ans']].copy() tmp = tmp.dropna(subset=['gold_url']) tmp = tmp.sample(frac=1, ignore_index=True) - # get dataframe length + # get dataframe length size = len(tmp) g_clips = tmp.copy() for i in range (1, (n_gold_clips//size)+1): diff --git a/src/result_parser.py b/src/result_parser.py index cebfa0c..5d362ac 100644 --- a/src/result_parser.py +++ b/src/result_parser.py @@ -757,36 +757,58 @@ def check_math(input, output, audio_played, expected_ans=None): def check_qualification_answer(row): checked = True # TODO hearing test - correct ans should be added in the inputs -update in master script is needed - - # check bw contrill + + # check bw control if "answer.comb_bw1" not in row: return checked, '' # check if 'bw_min' and 'bw_max' are in config if 'bw_min' not in config['acceptance_criteria'] or 'bw_max' not in config['acceptance_criteria']: return checked, '' - - bw_v2_test_data ={"comb_bw1":'dq', "comb_bw2":'dq', "comb_bw3":'dq', "comb_bw4":'sq', "comb_bw5":'sq'} - bw_messages= {"comb_bw1":'BW TP failed', "comb_bw2":'SWB failed', "comb_bw3":'FB failed', "comb_bw4":'BW TP failed', "comb_bw5":'BW TP failed'} - ans_array= [0, 0, 0 ,0, 0] + + # Per-position band role. Prefer the role delivered in the publish batch + # (input.ans_comb_bw*); fall back to the legacy fixed pattern (comb_bw1=wb obvious, + # comb_bw2=swb, comb_bw3=fb, comb_bw4/5=sq) so older studies are unaffected. + legacy_roles = {1: 'wb', 2: 'swb', 3: 'fb', 4: 'sq', 5: 'sq'} + + def role_of(i): + key = f'input.ans_comb_bw{i}' + if key in row and str(row.get(key)).strip().lower() not in ('', 'nan', 'none'): + return str(row[key]).strip().lower() + return legacy_roles[i] + + role_msg = {'wb': 'BW TP failed', 'swb': 'SWB failed', 'fb': 'FB failed', 'sq': 'BW TP failed'} msg = '' + # was the worker correct at the wb/swb/fb position, and at each sq position? + band_correct = {} + sq_correct = [] for i in range(1, 6): - if row[f'answer.comb_bw{i}'] != bw_v2_test_data[f'comb_bw{i}']: - msg += bw_messages[f'comb_bw{i}'] + ', ' - ans_array[i-1] = 0 + role = role_of(i) + expected = 'sq' if role == 'sq' else 'dq' + ok = str(row[f'answer.comb_bw{i}']).strip().lower() == expected + if not ok: + msg += role_msg.get(role, 'BW failed') + ', ' + if role == 'sq': + sq_correct.append(ok) else: - ans_array[i-1] = 1 - + band_correct[role] = ok bw_min = config['acceptance_criteria']['bw_min'].upper() bw_max = config['acceptance_criteria']['bw_max'].upper() - if ans_array[0] + ans_array[3] + ans_array[4] !=3: - #failed in trapping of obvious questions + + # trapping of the obvious questions: the wide-band different clip and both same clips + if not (band_correct.get('wb', False) and len(sq_correct) == 2 and all(sq_correct)): + # failed in trapping of obvious questions return False, msg - if (bw_min == 'SWB' and ans_array[1] != 1) or (bw_min == 'FB' and ans_array[1]+ans_array[2] != 2): + # bw_min: the worker must correctly hear the required bands + if bw_min == 'SWB' and not band_correct.get('swb', False): + return False, msg + if bw_min == 'FB' and not (band_correct.get('swb', False) and band_correct.get('fb', False)): + return False, msg + # bw_max: the worker must NOT hear bands above the maximum (answers "same") + if bw_max == 'NB-WB' and (band_correct.get('swb', False) or band_correct.get('fb', False)): + return False, msg + if bw_max == 'SWB' and band_correct.get('fb', False): return False, msg - - if (bw_max == 'NB-WB' and ans_array[1]+ans_array[2] != 0) or (bw_max == 'SWB' and ans_array[2] != 0): - return False, msg return checked, msg From d2b7849d80bd0b03403f19fcecaf879f10173444 Mon Sep 17 00:00:00 2001 From: Babak Naderi Date: Thu, 9 Jul 2026 17:47:44 +0200 Subject: [PATCH 047/111] bw-check qualification Part 2: wire P808_multi.html to batch + hash check + button - Bandwidth-check clips now render from the per-session publish batch (data-src=\) instead of hardcoded URLs. - Replace the plain bw_v2_test_data answer key with a client-side SHA-256 hash check (verifyBwHash, reusing the \/crypto.subtle pattern): the plain dq/sq answer is never rendered; only the per-position hash is. validate_bw_test is now async and passes only when all clips are identified correctly (the authoritative bw_min/bw_max decision stays server-side in result_parser). - Add a 'Run qualification check' button (wired via addEventListener in the DOMContentLoaded handler, no inline JS) to kick off the local check. Verified on a regenerated v5 preview: 5 clips filled from bw-test-extended, hashes injected, no plain answer, button present; main HTML keeps the runtime placeholders. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/P808Template/P808_multi.html | 118 ++++++++++++++++--------------- 1 file changed, 61 insertions(+), 57 deletions(-) diff --git a/src/P808Template/P808_multi.html b/src/P808Template/P808_multi.html index 746836a..4bdc667 100644 --- a/src/P808Template/P808_multi.html +++ b/src/P808Template/P808_multi.html @@ -525,6 +525,7 @@ updateResult(); // just to check checkScripts(); + $("#run_qualification_check").on("click", validateQualificationAnswer); }); async function generateHash(content) { @@ -683,67 +684,66 @@ if (checkQualNumCorrect($("input[name='num5']").val().trim(), config.qualification.num5)) count_correct_ht++; - bw_check = validate_bw_test(); - - //console.log("count_correct_ht:"+count_correct_ht); - passed = m_tongue_check && device_check && working_area_checked && hearing_self_report_check && bw_check && - count_correct_ht >= 3; - - - // add a qualification for 'qualificationValidFor' minutes - createCookie(config.qualificationCookieName, passed, config.qualificationValidFor); - if (passed){ - // activate the rest - $("#setup").removeClass("disabled2_section"); - $("#device_check_section").removeClass("disabled2_section"); - $("#training").removeClass("disabled2_section"); - $("#details_instruction").removeClass("disabled2_section"); - $("#rating_section").removeClass("disabled2_section"); - }else{ - // show the error message - disableTheHIT(Hide_HIT_REASON.QUALIFICATON); - } + // bandwidth check is async (hash-based); finish the decision once it resolves + validate_bw_test().then(function (bw_check) { + passed = m_tongue_check && device_check && working_area_checked && + hearing_self_report_check && bw_check && count_correct_ht >= 3; + + // add a qualification for 'qualificationValidFor' minutes + createCookie(config.qualificationCookieName, passed, config.qualificationValidFor); + if (passed) { + // activate the rest + $("#setup").removeClass("disabled2_section"); + $("#device_check_section").removeClass("disabled2_section"); + $("#training").removeClass("disabled2_section"); + $("#details_instruction").removeClass("disabled2_section"); + $("#rating_section").removeClass("disabled2_section"); + } else { + // show the error message + disableTheHIT(Hide_HIT_REASON.QUALIFICATON); + } + }); } - bw_v2_test_data ={ - "comb_bw1":'dq', - "comb_bw2":'dq', - "comb_bw3":'dq', - "comb_bw4":'sq', - "comb_bw5":'sq', - } + // Per-position correct-answer hashes injected from the publish batch + // (sha256(clip_url:dq|sq)). The plain answers are never rendered into the page. + var bwHashes = ["${comb_bw_hash1}", "${comb_bw_hash2}", "${comb_bw_hash3}", "${comb_bw_hash4}", "${comb_bw_hash5}"]; /** - * Validate the band-width check + * Verify one bandwidth-check answer against its hash (no plain answer in the page). **/ - function validate_bw_test() { - data = bw_v2_test_data; - count_correct = 0; - ans_array = Array(5).fill(0); - for (var i = 1; i <6; i++) { - q_name = 'comb_bw'+i; - ans = $("input[name='"+q_name+"']:checked").val(); - if (ans == data[q_name]) - ans_array[i-1] = 1; - else - ans_array[i-1] = 0; - } - console.log(ans_array); - bw_min = config['bw_min'].toUpperCase(); - bw_max = config['bw_max'].toUpperCase(); - - if (ans_array[0] + ans_array[3] + ans_array[4] !=3) - // failed in trapping of obvious questions - return false; - if ((bw_min == 'SWB' && ans_array[1] != 1) || (bw_min == 'FB' && ans_array[1]+ans_array[2] != 2)) - return false; + async function verifyBwHash(i, userAnswer) { + var h = bwHashes[i - 1]; + if (!h || h === "$" + "{comb_bw_hash" + i + "}") + return true; + var url = document.getElementById("comb_bw" + i).getAttribute("data-src"); + var data = new TextEncoder().encode(url + ":" + String(userAnswer).trim()); + var buf = await crypto.subtle.digest("SHA-256", data); + var hex = Array.from(new Uint8Array(buf)).map(function (b) { + return b.toString(16).padStart(2, "0"); + }).join(""); + return hex === h; + } - if ((bw_max == 'NB-WB' && ans_array[1]+ans_array[2] != 0) || (bw_max == 'SWB' && ans_array[2] != 0)) - return false; + /** + * Validate the band-width check locally (hash-based). Resolves to true only when + * every clip is identified correctly. The authoritative per-band bw_min/bw_max + * decision is done server-side by the result parser. + **/ + async function validate_bw_test() { + for (var i = 1; i < 6; i++) { + q_name = 'comb_bw' + i; + ans = $("input[name='" + q_name + "']:checked").val(); + if (!ans) + return false; + var ok = await verifyBwHash(i, ans); + if (!ok) + return false; + } return true; -} + } @@ -2986,7 +2986,7 @@

        -   +  
        @@ -3000,7 +3000,7 @@

        -   +  
        @@ -3013,7 +3013,7 @@

        -   +  
        @@ -3026,7 +3026,7 @@

        -   +  
        @@ -3039,7 +3039,7 @@

        -   +  
        @@ -3054,6 +3054,10 @@

        +
        + +
        +

        From 9817d18b1bc3ac97758a4710f06e97218b8d4df8 Mon Sep 17 00:00:00 2001 From: Babak Naderi Date: Thu, 9 Jul 2026 18:22:42 +0200 Subject: [PATCH 048/111] bw-check qualification UX: shuffle on load, manual check button, retry + screen-out Address review feedback on P808_multi.html: 1. Reliable Fisher-Yates shuffle of the qualification audio cards (.rnd/.rnd2/.rnd4) so the bandwidth-check clip order is randomized on every page load (the old shuffle could leave the fixed order). 2. Remove the auto-validation on the 16th answer; the setup section no longer activates until the participant explicitly runs the check. 3. Rename the button to 'Check answers' and rework the flow: - pass -> activate setup and smooth-scroll to it; - first failure -> alert asking to review and try again (one retry); - second failure -> disable the remaining sections and show the screen-out section, which now instructs the participant to submit the form and enter a Prolific completion code rendered from the \ template variable (set by the HIT app server). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/P808Template/P808_multi.html | 61 +++++++++++++++++++++----------- 1 file changed, 40 insertions(+), 21 deletions(-) diff --git a/src/P808Template/P808_multi.html b/src/P808Template/P808_multi.html index 4bdc667..79579bc 100644 --- a/src/P808Template/P808_multi.html +++ b/src/P808Template/P808_multi.html @@ -627,6 +627,7 @@ Initialize the qualification section */ var qualification_ans = new Set(); + var qualification_attempts = 0; function initializeQualification() { randomizeHearingtestItems(); makeCheckboxBeRequired(); @@ -640,13 +641,8 @@ check if the qualification all answered */ function isQualificationDone() { - qualification_ans.add(this.name); - console.log("check if the qualification is done " + this.name + "," + qualification_ans.size); - - //validateQualificationAnswer(); - if (qualification_ans.size == 16) - validateQualificationAnswer(); + // validation runs only when the participant clicks "Check answers" } /* @@ -692,15 +688,23 @@ // add a qualification for 'qualificationValidFor' minutes createCookie(config.qualificationCookieName, passed, config.qualificationValidFor); if (passed) { - // activate the rest + // activate the rest and scroll to the setup section $("#setup").removeClass("disabled2_section"); $("#device_check_section").removeClass("disabled2_section"); $("#training").removeClass("disabled2_section"); $("#details_instruction").removeClass("disabled2_section"); $("#rating_section").removeClass("disabled2_section"); + document.getElementById("setup").scrollIntoView({ behavior: "smooth", block: "start" }); } else { - // show the error message - disableTheHIT(Hide_HIT_REASON.QUALIFICATON); + qualification_attempts++; + if (qualification_attempts < 2) { + // first failure: let the participant review and try once more + alert("Some of your answers are not correct. Please review your answers and click \"Check answers\" again."); + } else { + // second failure: not qualified -> screen-out + disableTheHIT(Hide_HIT_REASON.QUALIFICATON); + document.getElementById("qualification_rejected").scrollIntoView({ behavior: "smooth", block: "start" }); + } } }); @@ -751,14 +755,23 @@ randomize clips in the hearing test */ function randomizeHearingtestItems() { - sections = [".rnd", ".rnd2", ".rnd4"]; + var sections = [".rnd", ".rnd2", ".rnd4"]; for (var j = 0; j < sections.length; j++) { - var cards = $(sections[j]); - for (var i = 0; i < cards.length; i++) { - var target = Math.floor(Math.random() * cards.length - 1) + 1; - var target2 = Math.floor(Math.random() * cards.length - 1) + 1; - cards.eq(target).before(cards.eq(target2)); - } + var $cards = $(sections[j]); + if ($cards.length < 2) + continue; + var $parent = $cards.eq(0).parent(); + var arr = $cards.toArray(); + // Fisher-Yates shuffle, then re-append in the shuffled order + for (var i = arr.length - 1; i > 0; i--) { + var k = Math.floor(Math.random() * (i + 1)); + var tmp = arr[i]; + arr[i] = arr[k]; + arr[k] = tmp; + } + for (var m = 0; m < arr.length; m++) { + $parent.append(arr[m]); + } } } /* @@ -3055,7 +3068,7 @@

        - +

        @@ -3630,13 +3643,19 @@

        Well done!

        You performed the maximum number of HITs that you were allo From 5bcdd41eb0c51fc773ed6af8fff1083a4aae9564 Mon Sep 17 00:00:00 2001 From: Babak Naderi Date: Thu, 9 Jul 2026 18:27:27 +0200 Subject: [PATCH 049/111] bw-check qualification: tell participant they have one more try on first failure Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/P808Template/P808_multi.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/P808Template/P808_multi.html b/src/P808Template/P808_multi.html index 79579bc..064019b 100644 --- a/src/P808Template/P808_multi.html +++ b/src/P808Template/P808_multi.html @@ -699,7 +699,7 @@ qualification_attempts++; if (qualification_attempts < 2) { // first failure: let the participant review and try once more - alert("Some of your answers are not correct. Please review your answers and click \"Check answers\" again."); + alert("Some of your answers are not correct. Please review your answers and click \"Check answers\" again. You have one more try."); } else { // second failure: not qualified -> screen-out disableTheHIT(Hide_HIT_REASON.QUALIFICATON); From bfb45095d040349b0a61f9bb96dde323e9f319d4 Mon Sep 17 00:00:00 2001 From: Babak Naderi Date: Thu, 9 Jul 2026 18:39:22 +0200 Subject: [PATCH 050/111] Wire optional screenout_code through master_script config Add _get_screenout_code() to resolve a study-level screen-out completion code from the [hit_app_html] config section and expose it via general_cfg (merged into every HIT app template config). When the key is absent or empty, the helper returns the literal \ placeholder so the value is left untouched for the HIT app server to fill in a later step. P808_multi.html now renders the screen-out code from {{ cfg.screenout_code }} instead of a hardcoded \, so a config-provided code is baked in at generation time while the placeholder otherwise survives. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/P808Template/P808_multi.html | 2 +- src/master_script.py | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/src/P808Template/P808_multi.html b/src/P808Template/P808_multi.html index 064019b..064767a 100644 --- a/src/P808Template/P808_multi.html +++ b/src/P808Template/P808_multi.html @@ -3649,7 +3649,7 @@

        Well done!

        You performed the maximum number of HITs that you were allo completion code on Prolific:

in-ear headphones Over-the-ear headphones loudspeaker built-in speakers in-ear headphones Over-the-ear headphones loudspeaker built-in speakers