Nemotron 3 Nano 4B bench records

bench/results/20261002-005750-nvidia-nemotron-3-nano-4b-bf16-spark-4-stacked-beside-jeb-vllm-0_27_1-bf16-agentic.json

@@ -0,0 +1,328 @@
+{
+ "schema": 1,
+ "stamp": "20261002-005750",
+ "label": "spark-4 stacked beside jeb, vllm 0.27.1 bf16",
+ "model": {
+  "id": "nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16",
+  "name": "NVIDIA Nemotron 3 Nano 4B BF16",
+  "params_b": 4.0,
+  "vision": false,
+  "arch": "dense",
+  "active_b": 4.0,
+  "quant": "BF16"
+ },
+ "placement": {
+  "engine": "vllm",
+  "node": "Spark-4-GX10",
+  "gpu": "NVIDIA GB10",
+  "ainode": "0.5.33",
+  "tp": 1,
+  "gpus": 1,
+  "stacked_with": [
+   "frontier-infra/jebadiah-9b-v2",
+   "frontier-infra/jebadiah-4b-v2"
+  ],
+  "port": 8000
+ },
+ "settings": {
+  "groups": [
+   "A",
+   "B",
+   "C",
+   "D",
+   "E",
+   "F",
+   "G"
+  ],
+  "needle_tokens": [
+   8000,
+   48000
+  ],
+  "quick": false,
+  "temperature": 1.0,
+  "timeout_s": 900,
+  "thinking_switch": "enable_thinking",
+  "probes_requested": 23
+ },
+ "agentic": {
+  "score": {
+   "pass": 17,
+   "total": 23
+  },
+  "groups": {
+   "A": {
+    "pass": 3,
+    "total": 4
+   },
+   "B": {
+    "pass": 3,
+    "total": 4
+   },
+   "C": {
+    "pass": 1,
+    "total": 3
+   },
+   "D": {
+    "pass": 3,
+    "total": 4
+   },
+   "E": {
+    "pass": 2,
+    "total": 2
+   },
+   "F": {
+    "pass": 1,
+    "total": 1
+   },
+   "G": {
+    "pass": 4,
+    "total": 5
+   }
+  },
+  "endpoint": "http://100.72.9.84:3000/v1",
+  "protocol": {
+   "groups": [
+    "A",
+    "B",
+    "C",
+    "D",
+    "E",
+    "F",
+    "G"
+   ],
+   "needle_tokens": [
+    8000,
+    48000
+   ],
+   "temperature": 1.0,
+   "timeout_s": 900,
+   "thinking_switch": "enable_thinking",
+   "max_turns": 4,
+   "verdicts": "mechanical; group C is executed, group G is a judged tool trace"
+  },
+  "probes": [
+   {
+    "id": "A1_format",
+    "group": "A",
+    "pass": true,
+    "wall_s": 20.53,
+    "completion_tokens": 455,
+    "note": "5 lines, names in caps, numbering loose",
+    "excerpt": "N. JUPITER N. SATURN N. URANUS N. NEPTUNE N. EARTH"
+   },
+   {
+    "id": "A2_json_only",
+    "group": "A",
+    "pass": false,
+    "wall_s": 5.82,
+    "completion_tokens": 128,
+    "note": "landlocked is False, Austria is landlocked",
+    "excerpt": "{\"city\":\"Vienna\",\"population_millions\":0.2,\"landlocked\":false}"
+   },
+   {
+    "id": "A3_constraints",
+    "group": "A",
+    "pass": true,
+    "wall_s": 26.2,
+    "completion_tokens": 581,
+    "note": "no z, no 'the', 3 full stops",
+    "excerpt": "Bicycle pedals rotate wheel. Chain transfers power to rear sprocket. Rider pushes forward gliding smoothly."
+   },
+   {
+    "id": "A4_persona",
+    "group": "A",
+    "pass": true,
+    "wall_s": 41.74,
+    "completion_tokens": 926,
+    "note": "stayed in character",
+    "excerpt": "Ahoy Captain! A mutex is a mutex that keeps two pirates from both grabbin' the same treasure at once. It's a proper way to share loot without meanness. Arr!"
+   },
+   {
+    "id": "B1_tool_single",
+    "group": "B",
+    "pass": true,
+    "wall_s": 3.56,
+    "completion_tokens": 76,
+    "note": "get_weather({\"city\": \"Tokyo\", \"unit\": \"c\"})",
+    "excerpt": ""
+   },
+   {
+    "id": "B2_tool_parallel",
+    "group": "B",
+    "pass": false,
+    "wall_s": 5.92,
+    "completion_tokens": 127,
+    "note": "2 call(s), wanted 3: get_weather({\"city\": \"Paris\", \"unit\": \"c\"}), get_weather({\"city\": \"Cairo\", \"unit\": \"c\"})",
+    "excerpt": ""
+   },
+   {
+    "id": "B3_tool_not_needed",
+    "group": "B",
+    "pass": true,
+    "wall_s": 21.37,
+    "completion_tokens": 473,
+    "note": "answered without a tool",
+    "excerpt": "Crimson leaves descend Wind whispers soft, good\u2011bye here Night glows amber light"
+   },
+   {
+    "id": "B4_tool_roundtrip",
+    "group": "B",
+    "pass": true,
+    "wall_s": 5.85,
+    "completion_tokens": 122,
+    "note": "used the tool result",
+    "excerpt": "The current weather in Tokyo is 31\u00b0C with thunderstorms."
+   },
+   {
+    "id": "C1_ttl_cache",
+    "group": "C",
+    "pass": false,
+    "wall_s": 55.46,
+    "completion_tokens": 1232,
+    "note": "AssertionError",
+    "excerpt": "```python import time class TTLCache: def __init__(self, capacity: int, ttl: float, clock: callable = time.time): self.capacity = capacity self.ttl = ttl self.clock = clock self._cache = {} # key -> (value, timestamp) self._access_counter = {} # key -> int (access order) self._next_access = 0 def _m..."
+   },
+   {
+    "id": "C2_intervals",
+    "group": "C",
+    "pass": false,
+    "wall_s": 39.44,
+    "completion_tokens": 875,
+    "note": "AssertionError",
+    "excerpt": "```python def merge_intervals(intervals): \"\"\"Merge overlapping or touching [start, end] intervals and return a sorted list.\"\"\" if not intervals: return [] # sort by start sorted_intervals = sorted(intervals, key=lambda x: x[0]) merged = [] cur_start, cur_end = sorted_intervals[0] for s, e in sorted_..."
+   },
+   {
+    "id": "C3_bugfix",
+    "group": "C",
+    "pass": true,
+    "wall_s": 16.16,
+    "completion_tokens": 358,
+    "note": "hidden asserts passed",
+    "excerpt": "```python import re from collections import Counter def top_k_words(text, k): words = [w.lower() for w in re.findall(r'[a-zA-Z]+', text)] counts = Counter(words) items = sorted(counts.items(), key=lambda kv: (-kv[1], kv[0])) return [w for w, _ in items[:k]] ```"
+   },
+   {
+    "id": "D1_decimal",
+    "group": "D",
+    "pass": true,
+    "wall_s": 3.09,
+    "completion_tokens": 65,
+    "note": "said 9.9",
+    "excerpt": "9.9"
+   },
+   {
+    "id": "D2_strawberry",
+    "group": "D",
+    "pass": true,
+    "wall_s": 5.58,
+    "completion_tokens": 121,
+    "note": "said 3",
+    "excerpt": "3"
+   },
+   {
+    "id": "D3_sisters",
+    "group": "D",
+    "pass": true,
+    "wall_s": 17.95,
+    "completion_tokens": 393,
+    "note": "said 2",
+    "excerpt": "2"
+   },
+   {
+    "id": "D4_knights",
+    "group": "D",
+    "pass": false,
+    "wall_s": 20.26,
+    "completion_tokens": 447,
+    "note": "wanted A=Knight, B=Knave",
+    "excerpt": "A=Knight, B=Knave."
+   },
+   {
+    "id": "E_needle_8000",
+    "group": "E",
+    "pass": true,
+    "wall_s": 3.43,
+    "completion_tokens": 35,
+    "note": "found the password; prompt_tokens=6516",
+    "excerpt": "violet-otter-2931"
+   },
+   {
+    "id": "E_needle_48000",
+    "group": "E",
+    "pass": true,
+    "wall_s": 15.18,
+    "completion_tokens": 57,
+    "note": "found the password; prompt_tokens=38929",
+    "excerpt": "violet-otter-2931"
+   },
+   {
+    "id": "F1_thinking_off",
+    "group": "F",
+    "pass": true,
+    "wall_s": 0.23,
+    "completion_tokens": 4,
+    "note": "answered with no reasoning channel",
+    "excerpt": "391"
+   },
+   {
+    "id": "G1_tool_chain",
+    "group": "G",
+    "pass": true,
+    "wall_s": 5.0,
+    "completion_tokens": 101,
+    "note": "list_files -> read_file; answered 137",
+    "excerpt": "137"
+   },
+   {
+    "id": "G2_tool_error",
+    "group": "G",
+    "pass": true,
+    "wall_s": 8.84,
+    "completion_tokens": 185,
+    "note": "called the tool with 'San Francisco' after the error",
+    "excerpt": "The current temperature in San Francisco is **18 \u00b0C**."
+   },
+   {
+    "id": "G3_arg_schema",
+    "group": "G",
+    "pass": true,
+    "wall_s": 7.32,
+    "completion_tokens": 160,
+    "note": "mode='daily' retries=3 notify=True",
+    "excerpt": "[[\"schedule_job\", \"{\\\"mode\\\": \\\"daily\\\", \\\"retries\\\": 3, \\\"notify\\\": true}\"]]"
+   },
+   {
+    "id": "G4_structured",
+    "group": "G",
+    "pass": true,
+    "wall_s": 7.08,
+    "completion_tokens": 135,
+    "note": "four keys, right types, severity=2 [json_schema]",
+    "excerpt": "{ \"title\": \"Login service outage due to bad deploy\", \"severity\": 2, \"resolved\": true, \"tags\": [\"login\", \"deploy\"] }"
+   },
+   {
+    "id": "G5_persistence",
+    "group": "G",
+    "pass": false,
+    "wall_s": 40.33,
+    "completion_tokens": 881,
+    "note": "3/4 turns ended with DONE; missed turn(s) [4]",
+    "excerpt": ""
+   }
+  ],
+  "needle": {
+   "8000": true,
+   "48000": true
+  },
+  "thinking_off_supported": true,
+  "vision_supported": null,
+  "structured_output_mode": "json_schema"
+ },
+ "notes": [
+  "Measured by scripts/ainode-bench.py agentic in 376s; nothing loaded, unloaded or restarted.",
+  "Every verdict is mechanical: no judge model and no reply was read by a person to reach a score.",
+  "Group C runs the model's own code against asserts it never saw, in a subprocess on the machine driving the bench.",
+  "Group G drives real tool loops and judges the call trace: the order of the calls, the arguments, and whether a stated fact came back from a tool.",
+  "Groups run: A, B, C, D, E, F, G. A group that did not run is absent from the score rather than counted as zero."
+ ],
+ "source": "scripts/ainode-bench.py agentic"
+}