← Gemma 3 4B IT Q4_K_M · 32k ctx, q8_0 KV
Run acfe0336
| Kind | evals (quick) |
| Started | 2026-09-16 21:25 UTC |
| Duration | 3,627 s |
| Engine | llama.cpp@91f6a6cf3 (build 10883) |
| Driver | 595.99.02 |
| Throttled | no |
| Command | lab eval gemma-3-4b-it-q4_k_m-gguf/32k-q8kv --tier quick --benchmarks aime_2025 |
| Lab version | 0.1.0+efb1036 |
Metrics
| Metric | Method | Prompt | Gen | Depth | Value | ± sd | Unit |
|---|
eval_allowance_tokens | aime_2025 | — | — | 0 | 32,607 | — | tokens |
eval_allowance_tokens | bfcl | — | — | 0 | 30,635 | — | tokens |
eval_allowance_tokens | gpqa_diamond | — | — | 0 | 32,512 | — | tokens |
eval_allowance_tokens | livecodebench | — | — | 0 | 32,135 | — | tokens |
eval_excluded | aime_2025 | — | — | 0 | 0.00 | — | count |
eval_excluded | bfcl | — | — | 0 | 0.00 | — | count |
eval_excluded | gpqa_diamond | — | — | 0 | 0.00 | — | count |
eval_excluded | livecodebench | — | — | 0 | 0.00 | — | count |
eval_length_stops | aime_2025 | — | — | 0 | 0.00 | — | count |
eval_length_stops | bfcl | — | — | 0 | 0.00 | — | count |
eval_length_stops | gpqa_diamond | — | — | 0 | 0.00 | — | count |
eval_length_stops | livecodebench | — | — | 0 | 2.00 | — | count |
eval_seconds_per_task | aime_2025 | — | — | 0 | 42.4 | — | s |
eval_seconds_per_task | bfcl | — | — | 0 | 1.35 | — | s |
eval_seconds_per_task | gpqa_diamond | — | — | 0 | 5.19 | — | s |
eval_seconds_per_task | livecodebench | — | — | 0 | 7.73 | — | s |
eval_tokens_per_task | aime_2025 | — | — | 0 | 6,395 | — | tokens |
eval_tokens_per_task | bfcl | — | — | 0 | 125 | — | tokens |
eval_tokens_per_task | gpqa_diamond | — | — | 0 | 777 | — | tokens |
eval_tokens_per_task | livecodebench | — | — | 0 | 915 | — | tokens |
Eval results
| Task | Metric | Value | ± se | Tasks | Attempts | Harness | Subset |
|---|
| AIME 2025 | mean accuracy | 12.5% | 5.3% | 30 | 4 | lab 0.1.0+ba2d048 | aime_2025_full:e0d01ea1cb74 |
| BFCL | accuracy | 53.3% | 2.5% | 400 | 1 | bfcl-eval 2026.3.23 prompting mode | bfcl_quick_400:6d45d3c92a1c |
| GPQA Diamond | accuracy | 32.3% | 3.3% | 198 | 1 | lab 0.1.0+ba2d048 | gpqa_diamond_full:a88f544639ca |
| LiveCodeBench | pass@1 | 18.0% | 3.9% | 100 | 1 | livecodebench 28fef95ea8c9f7a547c8329f2cd3d32b92c1fa24 | livecodebench_v6_newest100:7e85c832cec0 |
Raw
{
"bench": {
"n_prompt": 512,
"n_gen": 128,
"depths": [
0,
4096,
16384
],
"reps": 5,
"delay_s": 3,
"http_prompt_lengths": [
512,
4096
]
},
"allowance": {
"speed_run_id": "a2caf10e-6df5-4154-98d5-84ceb3a54011",
"pp0": 7854.616401,
"tg": [
{
"depth": 0,
"tps": 166.495777
},
{
"depth": 4096,
"tps": 161.324946
},
{
"depth": 16384,
"tps": 155.596682
}
],
"ctx": 32768,
"limit_s": 300,
"headline_tokens": 24768
},
"props": {
"sampling": {
"temperature": 0.800000011920929,
"top_p": 0.949999988079071,
"top_k": 40,
"min_p": 0.05000000074505806,
"typical_p": 1,
"repeat_penalty": 1,
"repeat_last_n": 64,
"presence_penalty": 0,
"frequency_penalty": 0,
"mirostat": 0,
"seed": 4294967295,
"samplers": [
"penalties",
"dry",
"top_n_sigma",
"top_k",
"typ_p",
"top_p",
"min_p",
"xtc",
"temperature"
],
"dry_multiplier": 0
},
"n_ctx": 32768,
"total_slots": 1,
"build_info": "b10883-91f6a6cf3",
"chat_template_caps": {
"supports_object_arguments": false,
"supports_parallel_tool_calls": false,
"supports_preserve_reasoning": false,
"supports_reasoning_effort": false,
"supports_string_content": true,
"supports_system_role": true,
"supports_tool_calls": false,
"supports_tools": false,
"supports_typed_content": true
}
},
"subsets": {
"aime_2025": {
"id": "aime_2025_full:e0d01ea1cb74",
"name": "aime_2025_full",
"source": {
"repo": "opencompass/AIME2025",
"revision": "a6ad95f611d72cf628a80b58bd0432ef6638f958",
"files": [
"aime2025-I.jsonl",
"aime2025-II.jsonl"
]
},
"n_tasks": 30
},
"bfcl": {
"id": "bfcl_quick_400:6d45d3c92a1c",
"name": "bfcl_quick_400",
"source": {
"harness": "bfcl-eval",
"harness_version": "2026.3.23",
"categories": [
"simple_python",
"multiple",
"parallel",
"parallel_multiple",
"multi_turn_base"
]
},
"n_tasks": 400
},
"livecodebench": {
"id": "livecodebench_v6_newest100:7e85c832cec0",
"name": "livecodebench_v6_newest100",
"source": {
"harness": "livecodebench",
"commit": "28fef95ea8c9f7a547c8329f2cd3d32b92c1fa24",
"scenario": "codegeneration",
"lm_style": "OpenAIChat",
"dataset": "livecodebench/code_generation_lite",
"revision": "0fe84c3912ea0c4d4a78037083943e8f0c4dd505",
"file": "test6.jsonl",
"timeout_per_test_s": 6,
"newest": 100,
"date_window": [
"2025-02-15T06:30:00",
"2025-04-06T00:00:00"
]
},
"n_tasks": 100
},
"gpqa_diamond": {
"id": "gpqa_diamond_full:a88f544639ca",
"name": "gpqa_diamond_full",
"source": {
"repo": "Idavidrein/gpqa",
"revision": "633f5ee89ab8ad4522a9f850766b73f62147ffdd",
"files": [
"gpqa_diamond.csv"
]
},
"n_tasks": 198
}
},
"proxy_stats": {
"aime_2025:aime_2025/I-1#1": {
"requests": 1,
"prompt_tokens": 75,
"completion_tokens": 1187,
"allowance_tokens": 32693,
"length_stops": 0,
"errors": 0,
"seconds": 7.632
},
"aime_2025:aime_2025/I-1#2": {
"requests": 1,
"prompt_tokens": 75,
"completion_tokens": 1115,
"allowance_tokens": 32693,
"length_stops": 0,
"errors": 0,
"seconds": 7.161
},
"aime_2025:aime_2025/I-1#3": {
"requests": 1,
"prompt_tokens": 75,
"completion_tokens": 734,
"allowance_tokens": 32693,
"length_stops": 0,
"errors": 0,
"seconds": 4.718
},
"aime_2025:aime_2025/I-1#4": {
"requests": 1,
"prompt_tokens": 75,
"completion_tokens": 1114,
"allowance_tokens": 32693,
"length_stops": 0,
"errors": 0,
"seconds": 7.195
},
"aime_2025:aime_2025/I-2#1": {
"requests": 1,
"prompt_tokens": 186,
"completion_tokens": 1117,
"allowance_tokens": 32582,
"length_stops": 0,
"errors": 0,
"seconds": 7.271
},
"aime_2025:aime_2025/I-2#2": {
"requests": 1,
"prompt_tokens": 186,
"completion_tokens": 1880,
"allowance_tokens": 32582,
"length_stops": 0,
"errors": 0,
"seconds": 12.219
},
"aime_2025:aime_2025/I-2#3": {
"requests": 1,
"prompt_tokens": 186,
"completion_tokens": 1377,
"allowance_tokens": 32582,
"length_stops": 0,
"errors": 0,
"seconds": 8.93
},
"aime_2025:aime_2025/I-2#4": {
"requests": 1,
"prompt_tokens": 186,
"completion_tokens": 1473,
"allowance_tokens": 32582,
"length_stops": 0,
"errors": 0,
"seconds": 9.575
},
"aime_2025:aime_2025/I-3#1": {
"requests": 1,
"prompt_tokens": 155,
"completion_tokens": 2408,
"allowance_tokens": 32613,
"length_stops": 0,
"errors": 0,
"seconds": 15.737
},
"aime_2025:aime_2025/I-3#2": {
"requests": 1,
"prompt_tokens": 155,
"completion_tokens": 1662,
"allowance_tokens": 32613,
"length_stops": 0,
"errors": 0,
"seconds": 10.855
},
"aime_2025:aime_2025/I-3#3": {
"requests": 1,
"prompt_tokens": 155,
"completion_tokens": 1484,
"allowance_tokens": 32613,
"length_stops": 0,
"errors": 0,
"seconds": 9.697
},
"aime_2025:aime_2025/I-3#4": {
"requests": 1,
"prompt_tokens": 155,
"completion_tokens": 2232,
"allowance_tokens": 32613,
"length_stops": 0,
"errors": 0,
"seconds": 14.648
},
"aime_2025:aime_2025/I-4#1": {
"requests": 1,
"prompt_tokens": 99,
"completion_tokens": 3110,
"allowance_tokens": 32669,
"length_stops": 0,
"errors": 0,
"seconds": 20.503
},
"aime_2025:aime_2025/I-4#2": {
"requests": 1,
"prompt_tokens": 99,
"completion_tokens": 1048,
"allowance_tokens": 32669,
"length_stops": 0,
"errors": 0,
"seconds": 6.868
},
"aime_2025:aime_2025/I-4#3": {
"requests": 1,
"prompt_tokens": 99,
"completion_tokens": 2240,
"allowance_tokens": 32669,
"length_stops": 0,
"errors": 0,
"seconds": 14.724
},
"aime_2025:aime_2025/I-4#4": {
"requests": 1,
"prompt_tokens": 99,
"completion_tokens": 2580,
"allowance_tokens": 32669,
"length_stops": 0,
"errors": 0,
"seconds": 17.032
},
"aime_2025:aime_2025/I-5#1": {
"requests": 1,
"prompt_tokens": 119,
"completion_tokens": 1885,
"allowance_tokens": 32649,
"length_stops": 0,
"errors": 0,
"seconds": 12.535
},
"aime_2025:aime_2025/I-5#2": {
"requests": 1,
"prompt_tokens": 119,
"completion_tokens": 1358,
"allowance_tokens": 32649,
"length_stops": 0,
"errors": 0,
"seconds": 8.927
},
"aime_2025:aime_2025/I-5#3": {
"requests": 1,
"prompt_tokens": 119,
"completion_tokens": 1438,
"allowance_tokens": 32649,
"length_stops": 0,
"errors": 0,
"seconds": 9.474
},
"aime_2025:aime_2025/I-5#4": {
"requests": 1,
"prompt_tokens": 119,
"completion_tokens": 1397,
"allowance_tokens": 32649,
"length_stops": 0,
"errors": 0,
"seconds": 9.203
},
"aime_2025:aime_2025/I-6#1": {
"requests": 1,
"prompt_tokens": 117,
"completion_tokens": 1941,
"allowance_tokens": 32651,
"length_stops": 0,
"errors": 0,
"seconds": 12.914
},
"aime_2025:aime_2025/I-6#2": {
"requests": 1,
"prompt_tokens": 117,
"completion_tokens": 1425,
"allowance_tokens": 32651,
"length_stops": 0,
"errors": 0,
"seconds": 9.409
},
"aime_2025:aime_2025/I-6#3": {
"requests": 1,
"prompt_tokens": 117,
"completion_tokens": 1336,
"allowance_tokens": 32651,
"length_stops": 0,
"errors": 0,
"seconds": 8.804
},
"aime_2025:aime_2025/I-6#4": {
"requests": 1,
"prompt_tokens": 117,
"completion_tokens": 976,
"allowance_tokens": 32651,
"length_stops": 0,
"errors": 0,
"seconds": 6.426
},
"aime_2025:aime_2025/I-7#1": {
"requests": 1,
"prompt_tokens": 174,
"completion_tokens": 1883,
"allowance_tokens": 32594,
"length_stops": 0,
"errors": 0,
"seconds": 12.482
},
"aime_2025:aime_2025/I-7#2": {
"requests": 1,
"prompt_tokens": 174,
"completion_tokens": 2253,
"allowance_tokens": 32594,
"length_stops": 0,
"errors": 0,
"seconds": 14.938
},
"aime_2025:aime_2025/I-7#3": {
"requests": 1,
"prompt_tokens": 174,
"completion_tokens": 1830,
"allowance_tokens": 32594,
"length_stops": 0,
"errors": 0,
"seconds": 12.122
},
"aime_2025:aime_2025/I-7#4": {
"requests": 1,
"prompt_tokens": 174,
"completion_tokens": 1331,
"allowance_tokens": 32594,
"length_stops": 0,
"errors": 0,
"seconds": 8.813
},
"aime_2025:aime_2025/I-8#1": {
"requests": 1,
"prompt_tokens": 140,
"completion_tokens": 1621,
"allowance_tokens": 32628,
"length_stops": 0,
"errors": 0,
"seconds": 10.736
},
"aime_2025:aime_2025/I-8#2": {
"requests": 1,
"prompt_tokens": 140,
"completion_tokens": 2186,
"allowance_tokens": 32628,
"length_stops": 0,
"errors": 0,
"seconds": 14.499
},
"aime_2025:aime_2025/I-8#3": {
"requests": 1,
"prompt_tokens": 140,
"completion_tokens": 1969,
"allowance_tokens": 32628,
"length_stops": 0,
"errors": 0,
"seconds": 13.048
},
"aime_2025:aime_2025/I-8#4": {
"requests": 1,
"prompt_tokens": 140,
"completion_tokens": 2199,
"allowance_tokens": 32628,
"length_stops": 0,
"errors": 0,
"seconds": 14.578
},
"aime_2025:aime_2025/I-9#1": {
"requests": 1,
"prompt_tokens": 138,
"completion_tokens": 1638,
"allowance_tokens": 32630,
"length_stops": 0,
"errors": 0,
"seconds": 10.86
},
"aime_2025:aime_2025/I-9#2": {
"requests": 1,
"prompt_tokens": 138,
"completion_tokens": 2386,
"allowance_tokens": 32630,
"length_stops": 0,
"errors": 0,
"seconds": 15.831
},
"aime_2025:aime_2025/I-9#3": {
"requests": 1,
"prompt_tokens": 138,
"completion_tokens": 2051,
"allowance_tokens": 32630,
"length_stops": 0,
"errors": 0,
"seconds": 13.6
},
"aime_2025:aime_2025/I-9#4": {
"requests": 1,
"prompt_tokens": 138,
"completion_tokens": 2060,
"allowance_tokens": 32630,
"length_stops": 0,
"errors": 0,
"seconds": 13.655
},
"aime_2025:aime_2025/I-10#1": {
"requests": 1,
"prompt_tokens": 296,
"completion_tokens": 1901,
"allowance_tokens": 32472,
"length_stops": 0,
"errors": 0,
"seconds": 12.668
},
"aime_2025:aime_2025/I-10#2": {
"requests": 1,
"prompt_tokens": 296,
"completion_tokens": 1602,
"allowance_tokens": 32472,
"length_stops": 0,
"errors": 0,
"seconds": 10.654
},
"aime_2025:aime_2025/I-10#3": {
"requests": 1,
"prompt_tokens": 296,
"completion_tokens": 1447,
"allowance_tokens": 32472,
"length_stops": 0,
"errors": 0,
"seconds": 9.631
},
"aime_2025:aime_2025/I-10#4": {
"requests": 1,
"prompt_tokens": 296,
"completion_tokens": 1209,
"allowance_tokens": 32472,
"length_stops": 0,
"errors": 0,
"seconds": 8.014
},
"aime_2025:aime_2025/I-11#1": {
"requests": 1,
"prompt_tokens": 242,
"completion_tokens": 2273,
"allowance_tokens": 32526,
"length_stops": 0,
"errors": 0,
"seconds": 15.103
},
"aime_2025:aime_2025/I-11#2": {
"requests": 1,
"prompt_tokens": 242,
"completion_tokens": 1263,
"allowance_tokens": 32526,
"length_stops": 0,
"errors": 0,
"seconds": 8.364
},
"aime_2025:aime_2025/I-11#3": {
"requests": 1,
"prompt_tokens": 242,
"completion_tokens": 1818,
"allowance_tokens": 32526,
"length_stops": 0,
"errors": 0,
"seconds": 12.042
},
"aime_2025:aime_2025/I-11#4": {
"requests": 1,
"prompt_tokens": 242,
"completion_tokens": 2236,
"allowance_tokens": 32526,
"length_stops": 0,
"errors": 0,
"seconds": 14.828
},
"aime_2025:aime_2025/I-12#1": {
"requests": 1,
"prompt_tokens": 154,
"completion_tokens": 1287,
"allowance_tokens": 32614,
"length_stops": 0,
"errors": 0,
"seconds": 8.516
},
"aime_2025:aime_2025/I-12#2": {
"requests": 1,
"prompt_tokens": 154,
"completion_tokens": 1951,
"allowance_tokens": 32614,
"length_stops": 0,
"errors": 0,
"seconds": 12.907
},
"aime_2025:aime_2025/I-12#3": {
"requests": 1,
"prompt_tokens": 154,
"completion_tokens": 1735,
"allowance_tokens": 32614,
"length_stops": 0,
"errors": 0,
"seconds": 11.466
},
"aime_2025:aime_2025/I-12#4": {
"requests": 1,
"prompt_tokens": 154,
"completion_tokens": 1644,
"allowance_tokens": 32614,
"length_stops": 0,
"errors": 0,
"seconds": 10.868
},
"aime_2025:aime_2025/I-13#1": {
"requests": 1,
"prompt_tokens": 118,
"completion_tokens": 1172,
"allowance_tokens": 32650,
"length_stops": 0,
"errors": 0,
"seconds": 7.733
},
"aime_2025:aime_2025/I-13#2": {
"requests": 1,
"prompt_tokens": 118,
"completion_tokens": 1261,
"allowance_tokens": 32650,
"length_stops": 0,
"errors": 0,
"seconds": 8.31
},
"aime_2025:aime_2025/I-13#3": {
"requests": 1,
"prompt_tokens": 118,
"completion_tokens": 1139,
"allowance_tokens": 32650,
"length_stops": 0,
"errors": 0,
"seconds": 7.495
},
"aime_2025:aime_2025/I-13#4": {
"requests": 1,
"prompt_tokens": 118,
"completion_tokens": 1355,
"allowance_tokens": 32650,
"length_stops": 0,
"errors": 0,
"seconds": 8.925
},
"aime_2025:aime_2025/I-14#1": {
"requests": 1,
"prompt_tokens": 176,
"completion_tokens": 1004,
"allowance_tokens": 32592,
"length_stops": 0,
"errors": 0,
"seconds": 6.645
},
"aime_2025:aime_2025/I-14#2": {
"requests": 1,
"prompt_tokens": 176,
"completion_tokens": 1202,
"allowance_tokens": 32592,
"length_stops": 0,
"errors": 0,
"seconds": 7.93
},
"aime_2025:aime_2025/I-14#3": {
"requests": 1,
"prompt_tokens": 176,
"completion_tokens": 1104,
"allowance_tokens": 32592,
"length_stops": 0,
"errors": 0,
"seconds": 7.276
},
"aime_2025:aime_2025/I-14#4": {
"requests": 1,
"prompt_tokens": 176,
"completion_tokens": 1227,
"allowance_tokens": 32592,
"length_stops": 0,
"errors": 0,
"seconds": 8.101
},
"aime_2025:aime_2025/I-15#1": {
"requests": 1,
"prompt_tokens": 118,
"completion_tokens": 1719,
"allowance_tokens": 32650,
"length_stops": 0,
"errors": 0,
"seconds": 11.391
},
"aime_2025:aime_2025/I-15#2": {
"requests": 1,
"prompt_tokens": 118,
"completion_tokens": 1353,
"allowance_tokens": 32650,
"length_stops": 0,
"errors": 0,
"seconds": 8.936
},
"aime_2025:aime_2025/I-15#3": {
"requests": 1,
"prompt_tokens": 118,
"completion_tokens": 1351,
"allowance_tokens": 32650,
"length_stops": 0,
"errors": 0,
"seconds": 8.914
},
"aime_2025:aime_2025/I-15#4": {
"requests": 1,
"prompt_tokens": 118,
"completion_tokens": 1844,
"allowance_tokens": 32650,
"length_stops": 0,
"errors": 0,
"seconds": 12.207
},
"aime_2025:aime_2025/II-1#1": {
"requests": 1,
"prompt_tokens": 146,
"completion_tokens": 1718,
"allowance_tokens": 32622,
"length_stops": 0,
"errors": 0,
"seconds": 11.398
},
"aime_2025:aime_2025/II-1#2": {
"requests": 1,
"prompt_tokens": 146,
"completion_tokens": 2332,
"allowance_tokens": 32622,
"length_stops": 0,
"errors": 0,
"seconds": 15.459
},
"aime_2025:aime_2025/II-1#3": {
"requests": 1,
"prompt_tokens": 146,
"completion_tokens": 1805,
"allowance_tokens": 32622,
"length_stops": 0,
"errors": 0,
"seconds": 11.934
},
"aime_2025:aime_2025/II-1#4": {
"requests": 1,
"prompt_tokens": 146,
"completion_tokens": 1662,
"allowance_tokens": 32622,
"length_stops": 0,
"errors": 0,
"seconds": 10.985
},
"aime_2025:aime_2025/II-2#1": {
"requests": 1,
"prompt_tokens": 83,
"completion_tokens": 2920,
"allowance_tokens": 32685,
"length_stops": 0,
"errors": 0,
"seconds": 19.363
},
"aime_2025:aime_2025/II-2#2": {
"requests": 1,
"prompt_tokens": 83,
"completion_tokens": 880,
"allowance_tokens": 32685,
"length_stops": 0,
"errors": 0,
"seconds": 5.78
},
"aime_2025:aime_2025/II-2#3": {
"requests": 1,
"prompt_tokens": 83,
"completion_tokens": 941,
"allowance_tokens": 32685,
"length_stops": 0,
"errors": 0,
"seconds": 6.184
},
"aime_2025:aime_2025/II-2#4": {
"requests": 1,
"prompt_tokens": 83,
"completion_tokens": 1257,
"allowance_tokens": 32685,
"length_stops": 0,
"errors": 0,
"seconds": 8.283
},
"aime_2025:aime_2025/II-3#1": {
"requests": 1,
"prompt_tokens": 107,
"completion_tokens": 1069,
"allowance_tokens": 32661,
"length_stops": 0,
"errors": 0,
"seconds": 7.051
},
"aime_2025:aime_2025/II-3#2": {
"requests": 1,
"prompt_tokens": 107,
"completion_tokens": 949,
"allowance_tokens": 32661,
"length_stops": 0,
"errors": 0,
"seconds": 6.252
},
"aime_2025:aime_2025/II-3#3": {
"requests": 1,
"prompt_tokens": 107,
"completion_tokens": 1088,
"allowance_tokens": 32661,
"length_stops": 0,
"errors": 0,
"seconds": 7.158
},
"aime_2025:aime_2025/II-3#4": {
"requests": 1,
"prompt_tokens": 107,
"completion_tokens": 901,
"allowance_tokens": 32661,
"length_stops": 0,
"errors": 0,
"seconds": 5.938
},
"aime_2025:aime_2025/II-4#1": {
"requests": 1,
"prompt_tokens": 217,
"completion_tokens": 1912,
"allowance_tokens": 32551,
"length_stops": 0,
"errors": 0,
"seconds": 12.687
},
"aime_2025:aime_2025/II-4#2": {
"requests": 1,
"prompt_tokens": 217,
"completion_tokens": 1803,
"allowance_tokens": 32551,
"length_stops": 0,
"errors": 0,
"seconds": 11.937
},
"aime_2025:aime_2025/II-4#3": {
"requests": 1,
"prompt_tokens": 217,
"completion_tokens": 2227,
"allowance_tokens": 32551,
"length_stops": 0,
"errors": 0,
"seconds": 14.77
},
"aime_2025:aime_2025/II-4#4": {
"requests": 1,
"prompt_tokens": 217,
"completion_tokens": 2718,
"allowance_tokens": 32551,
"length_stops": 0,
"errors": 0,
"seconds": 18.03
},
"aime_2025:aime_2025/II-5#1": {
"requests": 1,
"prompt_tokens": 246,
"completion_tokens": 1412,
"allowance_tokens": 32522,
"length_stops": 0,
"errors": 0,
"seconds": 9.368
},
"aime_2025:aime_2025/II-5#2": {
"requests": 1,
"prompt_tokens": 246,
"completion_tokens": 1337,
"allowance_tokens": 32522,
"length_stops": 0,
"errors": 0,
"seconds": 8.846
},
"aime_2025:aime_2025/II-5#3": {
"requests": 1,
"prompt_tokens": 246,
"completion_tokens": 1557,
"allowance_tokens": 32522,
"length_stops": 0,
"errors": 0,
"seconds": 10.313
},
"aime_2025:aime_2025/II-5#4": {
"requests": 1,
"prompt_tokens": 246,
"completion_tokens": 1236,
"allowance_tokens": 32522,
"length_stops": 0,
"errors": 0,
"seconds": 8.181
},
"aime_2025:aime_2025/II-6#1": {
"requests": 1,
"prompt_tokens": 247,
"completion_tokens": 1183,
"allowance_tokens": 32521,
"length_stops": 0,
"errors": 0,
"seconds": 7.854
},
"aime_2025:aime_2025/II-6#2": {
"requests": 1,
"prompt_tokens": 247,
"completion_tokens": 1256,
"allowance_tokens": 32521,
"length_stops": 0,
"errors": 0,
"seconds": 8.312
},
"aime_2025:aime_2025/II-6#3": {
"requests": 1,
"prompt_tokens": 247,
"completion_tokens": 1800,
"allowance_tokens": 32521,
"length_stops": 0,
"errors": 0,
"seconds": 11.921
},
"aime_2025:aime_2025/II-6#4": {
"requests": 1,
"prompt_tokens": 247,
"completion_tokens": 1600,
"allowance_tokens": 32521,
"length_stops": 0,
"errors": 0,
"seconds": 10.587
},
"aime_2025:aime_2025/II-7#1": {
"requests": 1,
"prompt_tokens": 133,
"completion_tokens": 1766,
"allowance_tokens": 32635,
"length_stops": 0,
"errors": 0,
"seconds": 11.685
},
"aime_2025:aime_2025/II-7#2": {
"requests": 1,
"prompt_tokens": 133,
"completion_tokens": 1825,
"allowance_tokens": 32635,
"length_stops": 0,
"errors": 0,
"seconds": 12.08
},
"aime_2025:aime_2025/II-7#3": {
"requests": 1,
"prompt_tokens": 133,
"completion_tokens": 1777,
"allowance_tokens": 32635,
"length_stops": 0,
"errors": 0,
"seconds": 11.774
},
"aime_2025:aime_2025/II-7#4": {
"requests": 1,
"prompt_tokens": 133,
"completion_tokens": 1477,
"allowance_tokens": 32635,
"length_stops": 0,
"errors": 0,
"seconds": 9.777
},
"aime_2025:aime_2025/II-8#1": {
"requests": 1,
"prompt_tokens": 307,
"completion_tokens": 967,
"allowance_tokens": 32461,
"length_stops": 0,
"errors": 0,
"seconds": 6.428
},
"aime_2025:aime_2025/II-8#2": {
"requests": 1,
"prompt_tokens": 307,
"completion_tokens": 875,
"allowance_tokens": 32461,
"length_stops": 0,
"errors": 0,
"seconds": 5.803
},
"aime_2025:aime_2025/II-8#3": {
"requests": 1,
"prompt_tokens": 307,
"completion_tokens": 1073,
"allowance_tokens": 32461,
"length_stops": 0,
"errors": 0,
"seconds": 7.116
},
"aime_2025:aime_2025/II-8#4": {
"requests": 1,
"prompt_tokens": 307,
"completion_tokens": 830,
"allowance_tokens": 32461,
"length_stops": 0,
"errors": 0,
"seconds": 5.512
},
"aime_2025:aime_2025/II-9#1": {
"requests": 1,
"prompt_tokens": 134,
"completion_tokens": 2150,
"allowance_tokens": 32634,
"length_stops": 0,
"errors": 0,
"seconds": 14.266
},
"aime_2025:aime_2025/II-9#2": {
"requests": 1,
"prompt_tokens": 134,
"completion_tokens": 1546,
"allowance_tokens": 32634,
"length_stops": 0,
"errors": 0,
"seconds": 10.226
},
"aime_2025:aime_2025/II-9#3": {
"requests": 1,
"prompt_tokens": 134,
"completion_tokens": 1754,
"allowance_tokens": 32634,
"length_stops": 0,
"errors": 0,
"seconds": 11.612
},
"aime_2025:aime_2025/II-9#4": {
"requests": 1,
"prompt_tokens": 134,
"completion_tokens": 1660,
"allowance_tokens": 32634,
"length_stops": 0,
"errors": 0,
"seconds": 10.986
},
"aime_2025:aime_2025/II-10#1": {
"requests": 1,
"prompt_tokens": 110,
"completion_tokens": 1731,
"allowance_tokens": 32658,
"length_stops": 0,
"errors": 0,
"seconds": 11.462
},
"aime_2025:aime_2025/II-10#2": {
"requests": 1,
"prompt_tokens": 110,
"completion_tokens": 1199,
"allowance_tokens": 32658,
"length_stops": 0,
"errors": 0,
"seconds": 7.908
},
"aime_2025:aime_2025/II-10#3": {
"requests": 1,
"prompt_tokens": 110,
"completion_tokens": 1116,
"allowance_tokens": 32658,
"length_stops": 0,
"errors": 0,
"seconds": 7.346
},
"aime_2025:aime_2025/II-10#4": {
"requests": 1,
"prompt_tokens": 110,
"completion_tokens": 1636,
"allowance_tokens": 32658,
"length_stops": 0,
"errors": 0,
"seconds": 10.801
},
"aime_2025:aime_2025/II-11#1": {
"requests": 1,
"prompt_tokens": 98,
"completion_tokens": 1620,
"allowance_tokens": 32670,
"length_stops": 0,
"errors": 0,
"seconds": 10.706
},
"aime_2025:aime_2025/II-11#2": {
"requests": 1,
"prompt_tokens": 98,
"completion_tokens": 1749,
"allowance_tokens": 32670,
"length_stops": 0,
"errors": 0,
"seconds": 11.559
},
"aime_2025:aime_2025/II-11#3": {
"requests": 1,
"prompt_tokens": 98,
"completion_tokens": 1541,
"allowance_tokens": 32670,
"length_stops": 0,
"errors": 0,
"seconds": 10.17
},
"aime_2025:aime_2025/II-11#4": {
"requests": 1,
"prompt_tokens": 98,
"completion_tokens": 1602,
"allowance_tokens": 32670,
"length_stops": 0,
"errors": 0,
"seconds": 10.576
},
"aime_2025:aime_2025/II-12#1": {
"requests": 1,
"prompt_tokens": 263,
"completion_tokens": 2493,
"allowance_tokens": 32505,
"length_stops": 0,
"errors": 0,
"seconds": 16.566
},
"aime_2025:aime_2025/II-12#2": {
"requests": 1,
"prompt_tokens": 263,
"completion_tokens": 1894,
"allowance_tokens": 32505,
"length_stops": 0,
"errors": 0,
"seconds": 12.554
},
"aime_2025:aime_2025/II-12#3": {
"requests": 1,
"prompt_tokens": 263,
"completion_tokens": 2200,
"allowance_tokens": 32505,
"length_stops": 0,
"errors": 0,
"seconds": 14.582
},
"aime_2025:aime_2025/II-12#4": {
"requests": 1,
"prompt_tokens": 263,
"completion_tokens": 1715,
"allowance_tokens": 32505,
"length_stops": 0,
"errors": 0,
"seconds": 11.353
},
"aime_2025:aime_2025/II-13#1": {
"requests": 1,
"prompt_tokens": 175,
"completion_tokens": 1778,
"allowance_tokens": 32593,
"length_stops": 0,
"errors": 0,
"seconds": 11.792
},
"aime_2025:aime_2025/II-13#2": {
"requests": 1,
"prompt_tokens": 175,
"completion_tokens": 1274,
"allowance_tokens": 32593,
"length_stops": 0,
"errors": 0,
"seconds": 8.408
},
"aime_2025:aime_2025/II-13#3": {
"requests": 1,
"prompt_tokens": 175,
"completion_tokens": 1372,
"allowance_tokens": 32593,
"length_stops": 0,
"errors": 0,
"seconds": 9.066
},
"aime_2025:aime_2025/II-13#4": {
"requests": 1,
"prompt_tokens": 175,
"completion_tokens": 1129,
"allowance_tokens": 32593,
"length_stops": 0,
"errors": 0,
"seconds": 7.449
},
"aime_2025:aime_2025/II-14#1": {
"requests": 1,
"prompt_tokens": 138,
"completion_tokens": 1742,
"allowance_tokens": 32630,
"length_stops": 0,
"errors": 0,
"seconds": 11.534
},
"aime_2025:aime_2025/II-14#2": {
"requests": 1,
"prompt_tokens": 138,
"completion_tokens": 1601,
"allowance_tokens": 32630,
"length_stops": 0,
"errors": 0,
"seconds": 10.578
},
"aime_2025:aime_2025/II-14#3": {
"requests": 1,
"prompt_tokens": 138,
"completion_tokens": 1036,
"allowance_tokens": 32630,
"length_stops": 0,
"errors": 0,
"seconds": 6.85
},
"aime_2025:aime_2025/II-14#4": {
"requests": 1,
"prompt_tokens": 138,
"completion_tokens": 1440,
"allowance_tokens": 32630,
"length_stops": 0,
"errors": 0,
"seconds": 9.544
},
"aime_2025:aime_2025/II-15#1": {
"requests": 1,
"prompt_tokens": 125,
"completion_tokens": 1691,
"allowance_tokens": 32643,
"length_stops": 0,
"errors": 0,
"seconds": 11.203
},
"aime_2025:aime_2025/II-15#2": {
"requests": 1,
"prompt_tokens": 125,
"completion_tokens": 1544,
"allowance_tokens": 32643,
"length_stops": 0,
"errors": 0,
"seconds": 10.2
},
"aime_2025:aime_2025/II-15#3": {
"requests": 1,
"prompt_tokens": 125,
"completion_tokens": 1632,
"allowance_tokens": 32643,
"length_stops": 0,
"errors": 0,
"seconds": 10.784
},
"aime_2025:aime_2025/II-15#4": {
"requests": 1,
"prompt_tokens": 125,
"completion_tokens": 1821,
"allowance_tokens": 32643,
"length_stops": 0,
"errors": 0,
"seconds": 12.045
},
"bfcl:simple_python_3#1": {
"requests": 1,
"prompt_tokens": 457,
"completion_tokens": 20,
"allowance_tokens": 32311,
"length_stops": 0,
"errors": 0,
"seconds": 0.239,
"budget_s": 0.1787231954781454,
"last_prompt_tokens": 457
},
"bfcl:multiple_0#1": {
"requests": 1,
"prompt_tokens": 887,
"completion_tokens": 23,
"allowance_tokens": 31881,
"length_stops": 0,
"errors": 0,
"seconds": 0.302,
"budget_s": 0.25187690311915145,
"last_prompt_tokens": 887
},
"bfcl:parallel_0#1": {
"requests": 1,
"prompt_tokens": 441,
"completion_tokens": 34,
"allowance_tokens": 32327,
"length_stops": 0,
"errors": 0,
"seconds": 0.302,
"budget_s": 0.26091251526575404,
"last_prompt_tokens": 441
},
"bfcl:parallel_multiple_0#1": {
"requests": 1,
"prompt_tokens": 630,
"completion_tokens": 50,
"allowance_tokens": 32138,
"length_stops": 0,
"errors": 0,
"seconds": 0.433,
"budget_s": 0.38182961064539506,
"last_prompt_tokens": 630
},
"bfcl:multi_turn_base_0#1": {
"requests": 4,
"prompt_tokens": 35789,
"completion_tokens": 113,
"allowance_tokens": 95283,
"length_stops": 0,
"errors": 0,
"seconds": 2.792,
"budget_s": 1.8397382826374635,
"last_prompt_tokens": 9042
},
"bfcl:simple_python_6#1": {
"requests": 1,
"prompt_tokens": 453,
"completion_tokens": 18,
"allowance_tokens": 32315,
"length_stops": 0,
"errors": 0,
"seconds": 0.256,
"budget_s": 0.15177010999817753,
"last_prompt_tokens": 453
},
"bfcl:multiple_9#1": {
"requests": 1,
"prompt_tokens": 662,
"completion_tokens": 37,
"allowance_tokens": 32106,
"length_stops": 0,
"errors": 0,
"seconds": 0.347,
"budget_s": 0.2933714606943676,
"last_prompt_tokens": 662
},
"bfcl:parallel_5#1": {
"requests": 1,
"prompt_tokens": 523,
"completion_tokens": 73,
"allowance_tokens": 32245,
"length_stops": 0,
"errors": 0,
"seconds": 0.548,
"budget_s": 0.48831972184981354,
"last_prompt_tokens": 523
},
"bfcl:parallel_multiple_1#1": {
"requests": 1,
"prompt_tokens": 698,
"completion_tokens": 27,
"allowance_tokens": 32070,
"length_stops": 0,
"errors": 0,
"seconds": 0.282,
"budget_s": 0.23356086059962602,
"last_prompt_tokens": 698
},
"bfcl:multi_turn_base_5#1": {
"requests": 4,
"prompt_tokens": 35950,
"completion_tokens": 133,
"allowance_tokens": 95122,
"length_stops": 0,
"errors": 0,
"seconds": 2.89,
"budget_s": 1.961093505310731,
"last_prompt_tokens": 9108
},
"bfcl:simple_python_11#1": {
"requests": 1,
"prompt_tokens": 428,
"completion_tokens": 17,
"allowance_tokens": 32340,
"length_stops": 0,
"errors": 0,
"seconds": 0.239,
"budget_s": 0.13388359557953422,
"last_prompt_tokens": 428
},
"bfcl:multiple_11#1": {
"requests": 1,
"prompt_tokens": 840,
"completion_tokens": 22,
"allowance_tokens": 31928,
"length_stops": 0,
"errors": 0,
"seconds": 0.265,
"budget_s": 0.21700947777604251,
"last_prompt_tokens": 840
},
"bfcl:parallel_8#1": {
"requests": 1,
"prompt_tokens": 480,
"completion_tokens": 91,
"allowance_tokens": 32288,
"length_stops": 0,
"errors": 0,
"seconds": 0.661,
"budget_s": 0.5867509660977649,
"last_prompt_tokens": 480
},
"bfcl:parallel_multiple_2#1": {
"requests": 1,
"prompt_tokens": 666,
"completion_tokens": 40,
"allowance_tokens": 32102,
"length_stops": 0,
"errors": 0,
"seconds": 0.348,
"budget_s": 0.3034674340716948,
"last_prompt_tokens": 666
},
"bfcl:multi_turn_base_7#1": {
"requests": 3,
"prompt_tokens": 26833,
"completion_tokens": 99,
"allowance_tokens": 71471,
"length_stops": 0,
"errors": 0,
"seconds": 2.422,
"budget_s": 1.7381688374588324,
"last_prompt_tokens": 9002
},
"bfcl:simple_python_17#1": {
"requests": 1,
"prompt_tokens": 419,
"completion_tokens": 18,
"allowance_tokens": 32349,
"length_stops": 0,
"errors": 0,
"seconds": 0.224,
"budget_s": 0.13913800144858432,
"last_prompt_tokens": 419
},
"bfcl:multiple_12#1": {
"requests": 1,
"prompt_tokens": 909,
"completion_tokens": 65,
"allowance_tokens": 31859,
"length_stops": 0,
"errors": 0,
"seconds": 0.552,
"budget_s": 0.48630330548915296,
"last_prompt_tokens": 909
},
"bfcl:parallel_10#1": {
"requests": 1,
"prompt_tokens": 440,
"completion_tokens": 33,
"allowance_tokens": 32328,
"length_stops": 0,
"errors": 0,
"seconds": 0.282,
"budget_s": 0.23235017477097666,
"last_prompt_tokens": 440
},
"bfcl:parallel_multiple_4#1": {
"requests": 1,
"prompt_tokens": 616,
"completion_tokens": 31,
"allowance_tokens": 32152,
"length_stops": 0,
"errors": 0,
"seconds": 0.285,
"budget_s": 0.24308263565890398,
"last_prompt_tokens": 616
},
"bfcl:multi_turn_base_11#1": {
"requests": 2,
"prompt_tokens": 17803,
"completion_tokens": 112,
"allowance_tokens": 47733,
"length_stops": 0,
"errors": 0,
"seconds": 2.278,
"budget_s": 1.8164044612372323,
"last_prompt_tokens": 8941
},
"bfcl:simple_python_22#1": {
"requests": 1,
"prompt_tokens": 420,
"completion_tokens": 18,
"allowance_tokens": 32348,
"length_stops": 0,
"errors": 0,
"seconds": 0.205,
"budget_s": 0.13964808106936896,
"last_prompt_tokens": 420
},
"bfcl:multiple_17#1": {
"requests": 1,
"prompt_tokens": 861,
"completion_tokens": 34,
"allowance_tokens": 31907,
"length_stops": 0,
"errors": 0,
"seconds": 0.349,
"budget_s": 0.2930157725283009,
"last_prompt_tokens": 861
},
"bfcl:parallel_14#1": {
"requests": 1,
"prompt_tokens": 485,
"completion_tokens": 99,
"allowance_tokens": 32283,
"length_stops": 0,
"errors": 0,
"seconds": 0.717,
"budget_s": 0.6363989040928605,
"last_prompt_tokens": 485
},
"bfcl:parallel_multiple_5#1": {
"requests": 1,
"prompt_tokens": 725,
"completion_tokens": 31,
"allowance_tokens": 32043,
"length_stops": 0,
"errors": 0,
"seconds": 0.306,
"budget_s": 0.25749722276653825,
"last_prompt_tokens": 725
},
"bfcl:multi_turn_base_13#1": {
"requests": 2,
"prompt_tokens": 17901,
"completion_tokens": 247,
"allowance_tokens": 47635,
"length_stops": 0,
"errors": 0,
"seconds": 3.156,
"budget_s": 2.6619404140664242,
"last_prompt_tokens": 9019
},
"bfcl:simple_python_33#1": {
"requests": 1,
"prompt_tokens": 487,
"completion_tokens": 25,
"allowance_tokens": 32281,
"length_stops": 0,
"errors": 0,
"seconds": 0.257,
"budget_s": 0.18623097665649296,
"last_prompt_tokens": 487
},
"bfcl:multiple_18#1": {
"requests": 1,
"prompt_tokens": 613,
"completion_tokens": 16,
"allowance_tokens": 32155,
"length_stops": 0,
"errors": 0,
"seconds": 0.188,
"budget_s": 0.14823662439463847,
"last_prompt_tokens": 613
},
"bfcl:parallel_16#1": {
"requests": 1,
"prompt_tokens": 470,
"completion_tokens": 19,
"allowance_tokens": 32298,
"length_stops": 0,
"errors": 0,
"seconds": 0.193,
"budget_s": 0.1481359435147749,
"last_prompt_tokens": 470
},
"bfcl:parallel_multiple_7#1": {
"requests": 1,
"prompt_tokens": 633,
"completion_tokens": 44,
"allowance_tokens": 32135,
"length_stops": 0,
"errors": 0,
"seconds": 0.371,
"budget_s": 0.32003571543123266,
"last_prompt_tokens": 633
},
"bfcl:multi_turn_base_14#1": {
"requests": 4,
"prompt_tokens": 29077,
"completion_tokens": 101,
"allowance_tokens": 101995,
"length_stops": 0,
"errors": 0,
"seconds": 2.354,
"budget_s": 1.5353949191949163,
"last_prompt_tokens": 7376
},
"bfcl:simple_python_47#1": {
"requests": 1,
"prompt_tokens": 438,
"completion_tokens": 25,
"allowance_tokens": 32330,
"length_stops": 0,
"errors": 0,
"seconds": 0.292,
"budget_s": 0.18044568178662196,
"last_prompt_tokens": 438
},
"bfcl:multiple_20#1": {
"requests": 1,
"prompt_tokens": 770,
"completion_tokens": 18,
"allowance_tokens": 31998,
"length_stops": 0,
"errors": 0,
"seconds": 0.229,
"budget_s": 0.18093259058160993,
"last_prompt_tokens": 770
},
"bfcl:parallel_19#1": {
"requests": 1,
"prompt_tokens": 497,
"completion_tokens": 108,
"allowance_tokens": 32271,
"length_stops": 0,
"errors": 0,
"seconds": 0.773,
"budget_s": 0.6885489749727909,
"last_prompt_tokens": 497
},
"bfcl:parallel_multiple_10#1": {
"requests": 1,
"prompt_tokens": 777,
"completion_tokens": 55,
"allowance_tokens": 31991,
"length_stops": 0,
"errors": 0,
"seconds": 0.467,
"budget_s": 0.40550175568632674,
"last_prompt_tokens": 777
},
"bfcl:multi_turn_base_16#1": {
"requests": 3,
"prompt_tokens": 14942,
"completion_tokens": 95,
"allowance_tokens": 83362,
"length_stops": 0,
"errors": 0,
"seconds": 1.729,
"budget_s": 1.1995723548335344,
"last_prompt_tokens": 5050
},
"bfcl:simple_python_59#1": {
"requests": 1,
"prompt_tokens": 429,
"completion_tokens": 27,
"allowance_tokens": 32339,
"length_stops": 0,
"errors": 0,
"seconds": 0.289,
"budget_s": 0.1915956895185637,
"last_prompt_tokens": 429
},
"bfcl:multiple_23#1": {
"requests": 1,
"prompt_tokens": 909,
"completion_tokens": 32,
"allowance_tokens": 31859,
"length_stops": 0,
"errors": 0,
"seconds": 0.34,
"budget_s": 0.2836690053322629,
"last_prompt_tokens": 909
},
"bfcl:parallel_20#1": {
"requests": 1,
"prompt_tokens": 516,
"completion_tokens": 75,
"allowance_tokens": 32252,
"length_stops": 0,
"errors": 0,
"seconds": 0.56,
"budget_s": 0.4923350042194612,
"last_prompt_tokens": 516
},
"bfcl:parallel_multiple_12#1": {
"requests": 1,
"prompt_tokens": 723,
"completion_tokens": 34,
"allowance_tokens": 32045,
"length_stops": 0,
"errors": 0,
"seconds": 0.329,
"budget_s": 0.2719200677138465,
"last_prompt_tokens": 723
},
"bfcl:multi_turn_base_17#1": {
"requests": 3,
"prompt_tokens": 22462,
"completion_tokens": 1448,
"allowance_tokens": 75842,
"length_stops": 0,
"errors": 0,
"seconds": 11.141,
"budget_s": 9.967193192826727,
"last_prompt_tokens": 7907
},
"bfcl:simple_python_67#1": {
"requests": 1,
"prompt_tokens": 459,
"completion_tokens": 18,
"allowance_tokens": 32309,
"length_stops": 0,
"errors": 0,
"seconds": 0.229,
"budget_s": 0.14069877367416847,
"last_prompt_tokens": 459
},
"bfcl:multiple_28#1": {
"requests": 1,
"prompt_tokens": 745,
"completion_tokens": 20,
"allowance_tokens": 32023,
"length_stops": 0,
"errors": 0,
"seconds": 0.236,
"budget_s": 0.18955494803990858,
"last_prompt_tokens": 745
},
"bfcl:parallel_21#1": {
"requests": 1,
"prompt_tokens": 473,
"completion_tokens": 49,
"allowance_tokens": 32295,
"length_stops": 0,
"errors": 0,
"seconds": 0.387,
"budget_s": 0.3265528884965195,
"last_prompt_tokens": 473
},
"bfcl:parallel_multiple_15#1": {
"requests": 1,
"prompt_tokens": 677,
"completion_tokens": 41,
"allowance_tokens": 32091,
"length_stops": 0,
"errors": 0,
"seconds": 0.352,
"budget_s": 0.3048142197896082,
"last_prompt_tokens": 677
},
"bfcl:multi_turn_base_18#1": {
"requests": 3,
"prompt_tokens": 26824,
"completion_tokens": 155,
"allowance_tokens": 71480,
"length_stops": 0,
"errors": 0,
"seconds": 2.791,
"budget_s": 2.0860996641072163,
"last_prompt_tokens": 9015
},
"bfcl:simple_python_90#1": {
"requests": 1,
"prompt_tokens": 528,
"completion_tokens": 34,
"allowance_tokens": 32240,
"length_stops": 0,
"errors": 0,
"seconds": 0.342,
"budget_s": 0.24347893234035892,
"last_prompt_tokens": 528
},
"bfcl:multiple_29#1": {
"requests": 1,
"prompt_tokens": 579,
"completion_tokens": 22,
"allowance_tokens": 32189,
"length_stops": 0,
"errors": 0,
"seconds": 0.225,
"budget_s": 0.17778716853329013,
"last_prompt_tokens": 579
},
"bfcl:parallel_22#1": {
"requests": 1,
"prompt_tokens": 507,
"completion_tokens": 66,
"allowance_tokens": 32261,
"length_stops": 0,
"errors": 0,
"seconds": 0.499,
"budget_s": 0.43396596110160296,
"last_prompt_tokens": 507
},
"bfcl:parallel_multiple_17#1": {
"requests": 1,
"prompt_tokens": 724,
"completion_tokens": 31,
"allowance_tokens": 32044,
"length_stops": 0,
"errors": 0,
"seconds": 0.296,
"budget_s": 0.25100279847711393,
"last_prompt_tokens": 724
},
"bfcl:multi_turn_base_20#1": {
"requests": 2,
"prompt_tokens": 17793,
"completion_tokens": 59,
"allowance_tokens": 47743,
"length_stops": 0,
"errors": 0,
"seconds": 1.9080000000000001,
"budget_s": 1.4751842061681149,
"last_prompt_tokens": 8925
},
"bfcl:simple_python_91#1": {
"requests": 1,
"prompt_tokens": 462,
"completion_tokens": 21,
"allowance_tokens": 32306,
"length_stops": 0,
"errors": 0,
"seconds": 0.226,
"budget_s": 0.1571279879422749,
"last_prompt_tokens": 462
},
"bfcl:multiple_30#1": {
"requests": 1,
"prompt_tokens": 717,
"completion_tokens": 15,
"allowance_tokens": 32051,
"length_stops": 0,
"errors": 0,
"seconds": 0.191,
"budget_s": 0.1537324138890906,
"last_prompt_tokens": 717
},
"bfcl:parallel_23#1": {
"requests": 1,
"prompt_tokens": 533,
"completion_tokens": 72,
"allowance_tokens": 32235,
"length_stops": 0,
"errors": 0,
"seconds": 0.539,
"budget_s": 0.474047315987155,
"last_prompt_tokens": 533
},
"bfcl:parallel_multiple_20#1": {
"requests": 1,
"prompt_tokens": 717,
"completion_tokens": 72,
"allowance_tokens": 32051,
"length_stops": 0,
"errors": 0,
"seconds": 0.564,
"budget_s": 0.4982094184885887,
"last_prompt_tokens": 717
},
"bfcl:multi_turn_base_24#1": {
"requests": 4,
"prompt_tokens": 29867,
"completion_tokens": 122,
"allowance_tokens": 101205,
"length_stops": 0,
"errors": 0,
"seconds": 2.54,
"budget_s": 1.6905266092000704,
"last_prompt_tokens": 7583
},
"bfcl:simple_python_93#1": {
"requests": 1,
"prompt_tokens": 493,
"completion_tokens": 23,
"allowance_tokens": 32275,
"length_stops": 0,
"errors": 0,
"seconds": 0.288,
"budget_s": 0.1736712163567265,
"last_prompt_tokens": 493
},
"bfcl:multiple_31#1": {
"requests": 1,
"prompt_tokens": 667,
"completion_tokens": 17,
"allowance_tokens": 32101,
"length_stops": 0,
"errors": 0,
"seconds": 0.201,
"budget_s": 0.15991484494668212,
"last_prompt_tokens": 667
},
"bfcl:parallel_26#1": {
"requests": 1,
"prompt_tokens": 530,
"completion_tokens": 42,
"allowance_tokens": 32238,
"length_stops": 0,
"errors": 0,
"seconds": 0.348,
"budget_s": 0.29325295386667066,
"last_prompt_tokens": 530
},
"bfcl:parallel_multiple_21#1": {
"requests": 1,
"prompt_tokens": 687,
"completion_tokens": 29,
"allowance_tokens": 32081,
"length_stops": 0,
"errors": 0,
"seconds": 0.281,
"budget_s": 0.23518268793328587,
"last_prompt_tokens": 687
},
"bfcl:multi_turn_base_25#1": {
"requests": 4,
"prompt_tokens": 19785,
"completion_tokens": 85,
"allowance_tokens": 111287,
"length_stops": 0,
"errors": 0,
"seconds": 1.8639999999999999,
"budget_s": 1.1322911406981786,
"last_prompt_tokens": 5022
},
"bfcl:simple_python_96#1": {
"requests": 1,
"prompt_tokens": 588,
"completion_tokens": 47,
"allowance_tokens": 32180,
"length_stops": 0,
"errors": 0,
"seconds": 0.449,
"budget_s": 0.3311689350364031,
"last_prompt_tokens": 588
},
"bfcl:multiple_32#1": {
"requests": 1,
"prompt_tokens": 649,
"completion_tokens": 18,
"allowance_tokens": 32119,
"length_stops": 0,
"errors": 0,
"seconds": 0.208,
"budget_s": 0.16415423829571185,
"last_prompt_tokens": 649
},
"bfcl:parallel_31#1": {
"requests": 1,
"prompt_tokens": 470,
"completion_tokens": 37,
"allowance_tokens": 32298,
"length_stops": 0,
"errors": 0,
"seconds": 0.307,
"budget_s": 0.25586955878357637,
"last_prompt_tokens": 470
},
"bfcl:parallel_multiple_24#1": {
"requests": 1,
"prompt_tokens": 570,
"completion_tokens": 36,
"allowance_tokens": 32198,
"length_stops": 0,
"errors": 0,
"seconds": 0.312,
"budget_s": 0.2628658450428625,
"last_prompt_tokens": 570
},
"bfcl:multi_turn_base_26#1": {
"requests": 3,
"prompt_tokens": 15478,
"completion_tokens": 811,
"allowance_tokens": 82826,
"length_stops": 0,
"errors": 0,
"seconds": 6.484,
"budget_s": 5.646226211223889,
"last_prompt_tokens": 5431
},
"bfcl:simple_python_104#1": {
"requests": 1,
"prompt_tokens": 453,
"completion_tokens": 17,
"allowance_tokens": 32315,
"length_stops": 0,
"errors": 0,
"seconds": 0.22,
"budget_s": 0.13339382539946204,
"last_prompt_tokens": 453
},
"bfcl:multiple_34#1": {
"requests": 1,
"prompt_tokens": 714,
"completion_tokens": 19,
"allowance_tokens": 32054,
"length_stops": 0,
"errors": 0,
"seconds": 0.217,
"budget_s": 0.17903156899125217,
"last_prompt_tokens": 714
},
"bfcl:parallel_32#1": {
"requests": 1,
"prompt_tokens": 480,
"completion_tokens": 40,
"allowance_tokens": 32288,
"length_stops": 0,
"errors": 0,
"seconds": 0.335,
"budget_s": 0.2757532142645514,
"last_prompt_tokens": 480
},
"bfcl:parallel_multiple_25#1": {
"requests": 1,
"prompt_tokens": 620,
"completion_tokens": 49,
"allowance_tokens": 32148,
"length_stops": 0,
"errors": 0,
"seconds": 0.4,
"budget_s": 0.34827234507470445,
"last_prompt_tokens": 620
},
"bfcl:multi_turn_base_27#1": {
"requests": 3,
"prompt_tokens": 22514,
"completion_tokens": 164,
"allowance_tokens": 75790,
"length_stops": 0,
"errors": 0,
"seconds": 2.5839999999999996,
"budget_s": 1.9586759395137412,
"last_prompt_tokens": 7642
},
"bfcl:simple_python_105#1": {
"requests": 1,
"prompt_tokens": 448,
"completion_tokens": 14,
"allowance_tokens": 32320,
"length_stops": 0,
"errors": 0,
"seconds": 0.203,
"budget_s": 0.11518272471305553,
"last_prompt_tokens": 448
},
"bfcl:multiple_44#1": {
"requests": 1,
"prompt_tokens": 812,
"completion_tokens": 23,
"allowance_tokens": 31956,
"length_stops": 0,
"errors": 0,
"seconds": 0.267,
"budget_s": 0.21627681210634192,
"last_prompt_tokens": 812
},
"bfcl:parallel_33#1": {
"requests": 1,
"prompt_tokens": 456,
"completion_tokens": 86,
"allowance_tokens": 32312,
"length_stops": 0,
"errors": 0,
"seconds": 0.623,
"budget_s": 0.5502774439354484,
"last_prompt_tokens": 456
},
"bfcl:parallel_multiple_28#1": {
"requests": 1,
"prompt_tokens": 599,
"completion_tokens": 40,
"allowance_tokens": 32169,
"length_stops": 0,
"errors": 0,
"seconds": 0.343,
"budget_s": 0.2916313503411238,
"last_prompt_tokens": 599
},
"bfcl:multi_turn_base_28#1": {
"requests": 3,
"prompt_tokens": 26129,
"completion_tokens": 198,
"allowance_tokens": 72175,
"length_stops": 0,
"errors": 0,
"seconds": 3.0380000000000003,
"budget_s": 2.324764658097772,
"last_prompt_tokens": 8847
},
"bfcl:simple_python_107#1": {
"requests": 1,
"prompt_tokens": 478,
"completion_tokens": 18,
"allowance_tokens": 32290,
"length_stops": 0,
"errors": 0,
"seconds": 0.232,
"budget_s": 0.14351536025787656,
"last_prompt_tokens": 478
},
"bfcl:multiple_47#1": {
"requests": 1,
"prompt_tokens": 752,
"completion_tokens": 34,
"allowance_tokens": 32016,
"length_stops": 0,
"errors": 0,
"seconds": 0.332,
"budget_s": 0.27540294562551904,
"last_prompt_tokens": 752
},
"bfcl:parallel_40#1": {
"requests": 1,
"prompt_tokens": 534,
"completion_tokens": 67,
"allowance_tokens": 32234,
"length_stops": 0,
"errors": 0,
"seconds": 0.508,
"budget_s": 0.4421153521672158,
"last_prompt_tokens": 534
},
"bfcl:parallel_multiple_31#1": {
"requests": 1,
"prompt_tokens": 561,
"completion_tokens": 72,
"allowance_tokens": 32207,
"length_stops": 0,
"errors": 0,
"seconds": 0.543,
"budget_s": 0.47592228244662077,
"last_prompt_tokens": 561
},
"bfcl:multi_turn_base_29#1": {
"requests": 3,
"prompt_tokens": 14799,
"completion_tokens": 111,
"allowance_tokens": 83505,
"length_stops": 0,
"errors": 0,
"seconds": 1.831,
"budget_s": 1.2881244347861651,
"last_prompt_tokens": 4993
},
"bfcl:simple_python_110#1": {
"requests": 1,
"prompt_tokens": 520,
"completion_tokens": 28,
"allowance_tokens": 32248,
"length_stops": 0,
"errors": 0,
"seconds": 0.298,
"budget_s": 0.20537714386096234,
"last_prompt_tokens": 520
},
"bfcl:multiple_50#1": {
"requests": 1,
"prompt_tokens": 828,
"completion_tokens": 17,
"allowance_tokens": 31940,
"length_stops": 0,
"errors": 0,
"seconds": 0.229,
"budget_s": 0.1786287082967343,
"last_prompt_tokens": 828
},
"bfcl:parallel_47#1": {
"requests": 1,
"prompt_tokens": 436,
"completion_tokens": 34,
"allowance_tokens": 32332,
"length_stops": 0,
"errors": 0,
"seconds": 0.283,
"budget_s": 0.23085869680672208,
"last_prompt_tokens": 436
},
"bfcl:parallel_multiple_37#1": {
"requests": 1,
"prompt_tokens": 595,
"completion_tokens": 34,
"allowance_tokens": 32173,
"length_stops": 0,
"errors": 0,
"seconds": 0.302,
"budget_s": 0.25147700891525976,
"last_prompt_tokens": 595
},
"bfcl:multi_turn_base_30#1": {
"requests": 2,
"prompt_tokens": 17873,
"completion_tokens": 71,
"allowance_tokens": 47663,
"length_stops": 0,
"errors": 0,
"seconds": 1.996,
"budget_s": 1.5553588818214015,
"last_prompt_tokens": 8986
},
"bfcl:simple_python_111#1": {
"requests": 1,
"prompt_tokens": 424,
"completion_tokens": 16,
"allowance_tokens": 32344,
"length_stops": 0,
"errors": 0,
"seconds": 0.188,
"budget_s": 0.12110732489394366,
"last_prompt_tokens": 424
},
"bfcl:multiple_51#1": {
"requests": 1,
"prompt_tokens": 812,
"completion_tokens": 22,
"allowance_tokens": 31956,
"length_stops": 0,
"errors": 0,
"seconds": 0.258,
"budget_s": 0.20717791816983722,
"last_prompt_tokens": 812
},
"bfcl:parallel_49#1": {
"requests": 1,
"prompt_tokens": 495,
"completion_tokens": 83,
"allowance_tokens": 32273,
"length_stops": 0,
"errors": 0,
"seconds": 0.612,
"budget_s": 0.5342546409099059,
"last_prompt_tokens": 495
},
"bfcl:parallel_multiple_38#1": {
"requests": 1,
"prompt_tokens": 540,
"completion_tokens": 63,
"allowance_tokens": 32228,
"length_stops": 0,
"errors": 0,
"seconds": 0.479,
"budget_s": 0.4196654825378307,
"last_prompt_tokens": 540
},
"bfcl:multi_turn_base_32#1": {
"requests": 2,
"prompt_tokens": 27168,
"completion_tokens": 10181,
"allowance_tokens": 38368,
"length_stops": 0,
"errors": 0,
"seconds": 71.611,
"budget_s": 65.08233118856646,
"last_prompt_tokens": 18540
},
"bfcl:simple_python_113#1": {
"requests": 1,
"prompt_tokens": 492,
"completion_tokens": 28,
"allowance_tokens": 32276,
"length_stops": 0,
"errors": 0,
"seconds": 0.287,
"budget_s": 0.20254026425279262,
"last_prompt_tokens": 492
},
"bfcl:multiple_55#1": {
"requests": 1,
"prompt_tokens": 615,
"completion_tokens": 14,
"allowance_tokens": 32153,
"length_stops": 0,
"errors": 0,
"seconds": 0.176,
"budget_s": 0.1340051696127743,
"last_prompt_tokens": 615
},
"bfcl:parallel_51#1": {
"requests": 1,
"prompt_tokens": 450,
"completion_tokens": 28,
"allowance_tokens": 32318,
"length_stops": 0,
"errors": 0,
"seconds": 0.245,
"budget_s": 0.19739377792876878,
"last_prompt_tokens": 450
},
"bfcl:parallel_multiple_39#1": {
"requests": 1,
"prompt_tokens": 563,
"completion_tokens": 33,
"allowance_tokens": 32205,
"length_stops": 0,
"errors": 0,
"seconds": 0.291,
"budget_s": 0.24221227441254678,
"last_prompt_tokens": 563
},
"bfcl:multi_turn_base_35#1": {
"requests": 3,
"prompt_tokens": 26039,
"completion_tokens": 99,
"allowance_tokens": 72265,
"length_stops": 0,
"errors": 0,
"seconds": 2.404,
"budget_s": 1.6986126124863479,
"last_prompt_tokens": 8738
},
"bfcl:simple_python_115#1": {
"requests": 1,
"prompt_tokens": 495,
"completion_tokens": 35,
"allowance_tokens": 32273,
"length_stops": 0,
"errors": 0,
"seconds": 0.351,
"budget_s": 0.24563681337039817,
"last_prompt_tokens": 495
},
"bfcl:multiple_57#1": {
"requests": 1,
"prompt_tokens": 765,
"completion_tokens": 33,
"allowance_tokens": 32003,
"length_stops": 0,
"errors": 0,
"seconds": 0.327,
"budget_s": 0.26849091246161316,
"last_prompt_tokens": 765
},
"bfcl:parallel_55#1": {
"requests": 1,
"prompt_tokens": 497,
"completion_tokens": 45,
"allowance_tokens": 32271,
"length_stops": 0,
"errors": 0,
"seconds": 0.366,
"budget_s": 0.303000584842152,
"last_prompt_tokens": 497
},
"bfcl:parallel_multiple_48#1": {
"requests": 1,
"prompt_tokens": 513,
"completion_tokens": 28,
"allowance_tokens": 32255,
"length_stops": 0,
"errors": 0,
"seconds": 0.25,
"budget_s": 0.2026945607159512,
"last_prompt_tokens": 513
},
"bfcl:multi_turn_base_37#1": {
"requests": 3,
"prompt_tokens": 14843,
"completion_tokens": 278,
"allowance_tokens": 83461,
"length_stops": 0,
"errors": 0,
"seconds": 2.956,
"budget_s": 2.3264724792344067,
"last_prompt_tokens": 5026
},
"bfcl:simple_python_116#1": {
"requests": 1,
"prompt_tokens": 471,
"completion_tokens": 27,
"allowance_tokens": 32297,
"length_stops": 0,
"errors": 0,
"seconds": 0.284,
"budget_s": 0.1913930588807499,
"last_prompt_tokens": 471
},
"bfcl:multiple_58#1": {
"requests": 1,
"prompt_tokens": 598,
"completion_tokens": 16,
"allowance_tokens": 32170,
"length_stops": 0,
"errors": 0,
"seconds": 0.185,
"budget_s": 0.1414779694822016,
"last_prompt_tokens": 598
},
"bfcl:parallel_65#1": {
"requests": 1,
"prompt_tokens": 463,
"completion_tokens": 58,
"allowance_tokens": 32305,
"length_stops": 0,
"errors": 0,
"seconds": 0.448,
"budget_s": 0.377466118548054,
"last_prompt_tokens": 463
},
"bfcl:parallel_multiple_49#1": {
"requests": 1,
"prompt_tokens": 663,
"completion_tokens": 47,
"allowance_tokens": 32105,
"length_stops": 0,
"errors": 0,
"seconds": 0.391,
"budget_s": 0.3371874260518976,
"last_prompt_tokens": 663
},
"bfcl:multi_turn_base_39#1": {
"requests": 4,
"prompt_tokens": 19928,
"completion_tokens": 96,
"allowance_tokens": 111144,
"length_stops": 0,
"errors": 0,
"seconds": 1.939,
"budget_s": 1.201198173565162,
"last_prompt_tokens": 5065
},
"bfcl:simple_python_124#1": {
"requests": 1,
"prompt_tokens": 530,
"completion_tokens": 43,
"allowance_tokens": 32238,
"length_stops": 0,
"errors": 0,
"seconds": 0.415,
"budget_s": 0.2959731905592413,
"last_prompt_tokens": 530
},
"bfcl:multiple_59#1": {
"requests": 1,
"prompt_tokens": 676,
"completion_tokens": 18,
"allowance_tokens": 32092,
"length_stops": 0,
"errors": 0,
"seconds": 0.206,
"budget_s": 0.16404928115804385,
"last_prompt_tokens": 676
},
"bfcl:parallel_66#1": {
"requests": 1,
"prompt_tokens": 461,
"completion_tokens": 46,
"allowance_tokens": 32307,
"length_stops": 0,
"errors": 0,
"seconds": 0.364,
"budget_s": 0.30538871441630017,
"last_prompt_tokens": 461
},
"bfcl:parallel_multiple_50#1": {
"requests": 1,
"prompt_tokens": 701,
"completion_tokens": 70,
"allowance_tokens": 32067,
"length_stops": 0,
"errors": 0,
"seconds": 0.554,
"budget_s": 0.48149659065672945,
"last_prompt_tokens": 701
},
"bfcl:multi_turn_base_40#1": {
"requests": 4,
"prompt_tokens": 29129,
"completion_tokens": 133,
"allowance_tokens": 101943,
"length_stops": 0,
"errors": 0,
"seconds": 2.583,
"budget_s": 1.7316908682630368,
"last_prompt_tokens": 7397
},
"bfcl:simple_python_125#1": {
"requests": 1,
"prompt_tokens": 514,
"completion_tokens": 31,
"allowance_tokens": 32254,
"length_stops": 0,
"errors": 0,
"seconds": 0.338,
"budget_s": 0.2220579550793755,
"last_prompt_tokens": 514
},
"bfcl:multiple_63#1": {
"requests": 1,
"prompt_tokens": 733,
"completion_tokens": 17,
"allowance_tokens": 32035,
"length_stops": 0,
"errors": 0,
"seconds": 0.207,
"budget_s": 0.16582291626115483,
"last_prompt_tokens": 733
},
"bfcl:parallel_68#1": {
"requests": 1,
"prompt_tokens": 538,
"completion_tokens": 32,
"allowance_tokens": 32230,
"length_stops": 0,
"errors": 0,
"seconds": 0.285,
"budget_s": 0.23143302334974297,
"last_prompt_tokens": 538
},
"bfcl:parallel_multiple_51#1": {
"requests": 1,
"prompt_tokens": 596,
"completion_tokens": 40,
"allowance_tokens": 32172,
"length_stops": 0,
"errors": 0,
"seconds": 0.341,
"budget_s": 0.28729717107067165,
"last_prompt_tokens": 596
},
"bfcl:multi_turn_base_42#1": {
"requests": 3,
"prompt_tokens": 26101,
"completion_tokens": 118,
"allowance_tokens": 72203,
"length_stops": 0,
"errors": 0,
"seconds": 2.5290000000000004,
"budget_s": 1.8183621556477925,
"last_prompt_tokens": 8779
},
"bfcl:simple_python_129#1": {
"requests": 1,
"prompt_tokens": 529,
"completion_tokens": 32,
"allowance_tokens": 32239,
"length_stops": 0,
"errors": 0,
"seconds": 0.329,
"budget_s": 0.2305286053925641,
"last_prompt_tokens": 529
},
"bfcl:multiple_65#1": {
"requests": 1,
"prompt_tokens": 790,
"completion_tokens": 23,
"allowance_tokens": 31978,
"length_stops": 0,
"errors": 0,
"seconds": 0.261,
"budget_s": 0.20988780310311245,
"last_prompt_tokens": 790
},
"bfcl:parallel_70#1": {
"requests": 1,
"prompt_tokens": 465,
"completion_tokens": 58,
"allowance_tokens": 32303,
"length_stops": 0,
"errors": 0,
"seconds": 0.447,
"budget_s": 0.3791265162098147,
"last_prompt_tokens": 465
},
"bfcl:parallel_multiple_61#1": {
"requests": 1,
"prompt_tokens": 857,
"completion_tokens": 26,
"allowance_tokens": 31911,
"length_stops": 0,
"errors": 0,
"seconds": 0.29,
"budget_s": 0.23675251865213578,
"last_prompt_tokens": 857
},
"bfcl:multi_turn_base_44#1": {
"requests": 3,
"prompt_tokens": 26239,
"completion_tokens": 132,
"allowance_tokens": 72065,
"length_stops": 0,
"errors": 0,
"seconds": 2.63,
"budget_s": 1.910343227268196,
"last_prompt_tokens": 8828
},
"bfcl:simple_python_131#1": {
"requests": 1,
"prompt_tokens": 550,
"completion_tokens": 32,
"allowance_tokens": 32218,
"length_stops": 0,
"errors": 0,
"seconds": 0.33,
"budget_s": 0.2334876745438636,
"last_prompt_tokens": 550
},
"bfcl:multiple_66#1": {
"requests": 1,
"prompt_tokens": 781,
"completion_tokens": 26,
"allowance_tokens": 31987,
"length_stops": 0,
"errors": 0,
"seconds": 0.284,
"budget_s": 0.22711287843766453,
"last_prompt_tokens": 781
},
"bfcl:parallel_72#1": {
"requests": 1,
"prompt_tokens": 596,
"completion_tokens": 122,
"allowance_tokens": 32172,
"length_stops": 0,
"errors": 0,
"seconds": 0.882,
"budget_s": 0.7825471487529148,
"last_prompt_tokens": 596
},
"bfcl:parallel_multiple_64#1": {
"requests": 1,
"prompt_tokens": 788,
"completion_tokens": 104,
"allowance_tokens": 31980,
"length_stops": 0,
"errors": 0,
"seconds": 0.792,
"budget_s": 0.6994360272129581,
"last_prompt_tokens": 788
},
"bfcl:multi_turn_base_46#1": {
"requests": 1,
"prompt_tokens": 7161,
"completion_tokens": 39,
"allowance_tokens": 25607,
"length_stops": 0,
"errors": 0,
"seconds": 1.289,
"budget_s": 1.1263193332287043,
"last_prompt_tokens": 7161
},
"bfcl:simple_python_138#1": {
"requests": 1,
"prompt_tokens": 554,
"completion_tokens": 36,
"allowance_tokens": 32214,
"length_stops": 0,
"errors": 0,
"seconds": 0.323,
"budget_s": 0.2583834106874143,
"last_prompt_tokens": 554
},
"bfcl:multiple_68#1": {
"requests": 1,
"prompt_tokens": 738,
"completion_tokens": 16,
"allowance_tokens": 32030,
"length_stops": 0,
"errors": 0,
"seconds": 0.205,
"budget_s": 0.16144195229494307,
"last_prompt_tokens": 738
},
"bfcl:parallel_73#1": {
"requests": 1,
"prompt_tokens": 536,
"completion_tokens": 65,
"allowance_tokens": 32232,
"length_stops": 0,
"errors": 0,
"seconds": 0.502,
"budget_s": 0.4312059924527969,
"last_prompt_tokens": 536
},
"bfcl:parallel_multiple_67#1": {
"requests": 1,
"prompt_tokens": 1057,
"completion_tokens": 64,
"allowance_tokens": 31711,
"length_stops": 0,
"errors": 0,
"seconds": 0.568,
"budget_s": 0.49317006168946304,
"last_prompt_tokens": 1057
},
"bfcl:multi_turn_base_49#1": {
"requests": 4,
"prompt_tokens": 34810,
"completion_tokens": 49,
"allowance_tokens": 96262,
"length_stops": 0,
"errors": 0,
"seconds": 2.3200000000000003,
"budget_s": 1.3915307822737748,
"last_prompt_tokens": 8771
},
"bfcl:simple_python_140#1": {
"requests": 1,
"prompt_tokens": 502,
"completion_tokens": 34,
"allowance_tokens": 32266,
"length_stops": 0,
"errors": 0,
"seconds": 0.361,
"budget_s": 0.24000088969356037,
"last_prompt_tokens": 502
},
"bfcl:multiple_69#1": {
"requests": 1,
"prompt_tokens": 887,
"completion_tokens": 33,
"allowance_tokens": 31881,
"length_stops": 0,
"errors": 0,
"seconds": 0.342,
"budget_s": 0.2836995876854881,
"last_prompt_tokens": 887
},
"bfcl:parallel_74#1": {
"requests": 1,
"prompt_tokens": 441,
"completion_tokens": 37,
"allowance_tokens": 32327,
"length_stops": 0,
"errors": 0,
"seconds": 0.305,
"budget_s": 0.2503458673862042,
"last_prompt_tokens": 441
},
"bfcl:parallel_multiple_68#1": {
"requests": 1,
"prompt_tokens": 1043,
"completion_tokens": 143,
"allowance_tokens": 31725,
"length_stops": 0,
"errors": 0,
"seconds": 1.09,
"budget_s": 0.9698696239302568,
"last_prompt_tokens": 1043
},
"bfcl:multi_turn_base_50#1": {
"requests": 1,
"prompt_tokens": 6690,
"completion_tokens": 106,
"allowance_tokens": 26078,
"length_stops": 0,
"errors": 0,
"seconds": 1.666,
"budget_s": 1.4851040861326859,
"last_prompt_tokens": 6690
},
"bfcl:simple_python_143#1": {
"requests": 1,
"prompt_tokens": 459,
"completion_tokens": 20,
"allowance_tokens": 32309,
"length_stops": 0,
"errors": 0,
"seconds": 0.204,
"budget_s": 0.15033408253837832,
"last_prompt_tokens": 459
},
"bfcl:multiple_72#1": {
"requests": 1,
"prompt_tokens": 671,
"completion_tokens": 25,
"allowance_tokens": 32097,
"length_stops": 0,
"errors": 0,
"seconds": 0.251,
"budget_s": 0.2078310084053972,
"last_prompt_tokens": 671
},
"bfcl:parallel_75#1": {
"requests": 1,
"prompt_tokens": 388,
"completion_tokens": 44,
"allowance_tokens": 32380,
"length_stops": 0,
"errors": 0,
"seconds": 0.344,
"budget_s": 0.2860574762650432,
"last_prompt_tokens": 388
},
"bfcl:parallel_multiple_69#1": {
"requests": 1,
"prompt_tokens": 1087,
"completion_tokens": 147,
"allowance_tokens": 31681,
"length_stops": 0,
"errors": 0,
"seconds": 1.124,
"budget_s": 1.0003688760134184,
"last_prompt_tokens": 1087
},
"bfcl:multi_turn_base_52#1": {
"requests": 4,
"prompt_tokens": 43091,
"completion_tokens": 208,
"allowance_tokens": 87981,
"length_stops": 0,
"errors": 0,
"seconds": 3.807,
"budget_s": 2.6583187293838244,
"last_prompt_tokens": 10916
},
"bfcl:simple_python_149#1": {
"requests": 1,
"prompt_tokens": 404,
"completion_tokens": 18,
"allowance_tokens": 32364,
"length_stops": 0,
"errors": 0,
"seconds": 0.24,
"budget_s": 0.13161412230748915,
"last_prompt_tokens": 404
},
"bfcl:multiple_73#1": {
"requests": 1,
"prompt_tokens": 513,
"completion_tokens": 14,
"allowance_tokens": 32255,
"length_stops": 0,
"errors": 0,
"seconds": 0.161,
"budget_s": 0.12159015306908053,
"last_prompt_tokens": 513
},
"bfcl:parallel_81#1": {
"requests": 1,
"prompt_tokens": 615,
"completion_tokens": 99,
"allowance_tokens": 32153,
"length_stops": 0,
"errors": 0,
"seconds": 0.728,
"budget_s": 0.6476842722325828,
"last_prompt_tokens": 615
},
"bfcl:parallel_multiple_71#1": {
"requests": 1,
"prompt_tokens": 855,
"completion_tokens": 65,
"allowance_tokens": 31913,
"length_stops": 0,
"errors": 0,
"seconds": 0.547,
"budget_s": 0.4739191883307736,
"last_prompt_tokens": 855
},
"bfcl:multi_turn_base_57#1": {
"requests": 2,
"prompt_tokens": 20884,
"completion_tokens": 75,
"allowance_tokens": 44652,
"length_stops": 0,
"errors": 0,
"seconds": 2.291,
"budget_s": 1.7772812422171185,
"last_prompt_tokens": 10493
},
"bfcl:simple_python_160#1": {
"requests": 1,
"prompt_tokens": 483,
"completion_tokens": 32,
"allowance_tokens": 32285,
"length_stops": 0,
"errors": 0,
"seconds": 0.308,
"budget_s": 0.22651432940753624,
"last_prompt_tokens": 483
},
"bfcl:multiple_74#1": {
"requests": 1,
"prompt_tokens": 681,
"completion_tokens": 25,
"allowance_tokens": 32087,
"length_stops": 0,
"errors": 0,
"seconds": 0.256,
"budget_s": 0.2098795296806621,
"last_prompt_tokens": 681
},
"bfcl:parallel_84#1": {
"requests": 1,
"prompt_tokens": 594,
"completion_tokens": 66,
"allowance_tokens": 32174,
"length_stops": 0,
"errors": 0,
"seconds": 0.51,
"budget_s": 0.4461971335461144,
"last_prompt_tokens": 594
},
"bfcl:parallel_multiple_72#1": {
"requests": 1,
"prompt_tokens": 774,
"completion_tokens": 116,
"allowance_tokens": 31994,
"length_stops": 0,
"errors": 0,
"seconds": 0.867,
"budget_s": 0.7718683019872878,
"last_prompt_tokens": 774
},
"bfcl:multi_turn_base_64#1": {
"requests": 8,
"prompt_tokens": 55422,
"completion_tokens": 138,
"allowance_tokens": 206722,
"length_stops": 0,
"errors": 0,
"seconds": 3.432,
"budget_s": 1.7405231529554603,
"last_prompt_tokens": 7207
},
"bfcl:simple_python_167#1": {
"requests": 1,
"prompt_tokens": 442,
"completion_tokens": 28,
"allowance_tokens": 32326,
"length_stops": 0,
"errors": 0,
"seconds": 0.382,
"budget_s": 0.19751082126750608,
"last_prompt_tokens": 442
},
"bfcl:multiple_76#1": {
"requests": 1,
"prompt_tokens": 732,
"completion_tokens": 18,
"allowance_tokens": 32036,
"length_stops": 0,
"errors": 0,
"seconds": 0.22,
"budget_s": 0.17453540011324872,
"last_prompt_tokens": 732
},
"bfcl:parallel_85#1": {
"requests": 1,
"prompt_tokens": 591,
"completion_tokens": 51,
"allowance_tokens": 32177,
"length_stops": 0,
"errors": 0,
"seconds": 0.412,
"budget_s": 0.35556281605960655,
"last_prompt_tokens": 591
},
"bfcl:parallel_multiple_74#1": {
"requests": 1,
"prompt_tokens": 998,
"completion_tokens": 129,
"allowance_tokens": 31770,
"length_stops": 0,
"errors": 0,
"seconds": 0.994,
"budget_s": 0.8805159659901799,
"last_prompt_tokens": 998
},
"bfcl:multi_turn_base_68#1": {
"requests": 5,
"prompt_tokens": 57588,
"completion_tokens": 2189,
"allowance_tokens": 106252,
"length_stops": 0,
"errors": 0,
"seconds": 17.694,
"budget_s": 15.22852762165857,
"last_prompt_tokens": 12596
},
"bfcl:simple_python_183#1": {
"requests": 1,
"prompt_tokens": 450,
"completion_tokens": 17,
"allowance_tokens": 32318,
"length_stops": 0,
"errors": 0,
"seconds": 0.262,
"budget_s": 0.1325002912166368,
"last_prompt_tokens": 450
},
"bfcl:multiple_78#1": {
"requests": 1,
"prompt_tokens": 787,
"completion_tokens": 32,
"allowance_tokens": 31981,
"length_stops": 0,
"errors": 0,
"seconds": 0.322,
"budget_s": 0.266428877158096,
"last_prompt_tokens": 787
},
"bfcl:parallel_87#1": {
"requests": 1,
"prompt_tokens": 552,
"completion_tokens": 47,
"allowance_tokens": 32216,
"length_stops": 0,
"errors": 0,
"seconds": 0.386,
"budget_s": 0.32663523254448906,
"last_prompt_tokens": 552
},
"bfcl:parallel_multiple_77#1": {
"requests": 1,
"prompt_tokens": 955,
"completion_tokens": 73,
"allowance_tokens": 31813,
"length_stops": 0,
"errors": 0,
"seconds": 0.617,
"budget_s": 0.536241596548398,
"last_prompt_tokens": 955
},
"bfcl:multi_turn_base_69#1": {
"requests": 2,
"prompt_tokens": 21487,
"completion_tokens": 291,
"allowance_tokens": 44049,
"length_stops": 0,
"errors": 0,
"seconds": 3.819,
"budget_s": 3.1713389317917837,
"last_prompt_tokens": 10855
},
"bfcl:simple_python_192#1": {
"requests": 1,
"prompt_tokens": 561,
"completion_tokens": 53,
"allowance_tokens": 32207,
"length_stops": 0,
"errors": 0,
"seconds": 0.455,
"budget_s": 0.36411868909033523,
"last_prompt_tokens": 561
},
"bfcl:multiple_80#1": {
"requests": 1,
"prompt_tokens": 861,
"completion_tokens": 32,
"allowance_tokens": 31907,
"length_stops": 0,
"errors": 0,
"seconds": 0.337,
"budget_s": 0.27621391488318225,
"last_prompt_tokens": 861
},
"bfcl:parallel_91#1": {
"requests": 1,
"prompt_tokens": 494,
"completion_tokens": 71,
"allowance_tokens": 32274,
"length_stops": 0,
"errors": 0,
"seconds": 0.54,
"budget_s": 0.4640702880999512,
"last_prompt_tokens": 494
},
"bfcl:parallel_multiple_79#1": {
"requests": 1,
"prompt_tokens": 846,
"completion_tokens": 59,
"allowance_tokens": 31922,
"length_stops": 0,
"errors": 0,
"seconds": 0.509,
"budget_s": 0.437622596461941,
"last_prompt_tokens": 846
},
"bfcl:multi_turn_base_70#1": {
"requests": 2,
"prompt_tokens": 13458,
"completion_tokens": 42,
"allowance_tokens": 52078,
"length_stops": 0,
"errors": 0,
"seconds": 1.463,
"budget_s": 1.0945850482358883,
"last_prompt_tokens": 6770
},
"bfcl:simple_python_198#1": {
"requests": 1,
"prompt_tokens": 468,
"completion_tokens": 27,
"allowance_tokens": 32300,
"length_stops": 0,
"errors": 0,
"seconds": 0.267,
"budget_s": 0.19559069466918821,
"last_prompt_tokens": 468
},
"bfcl:multiple_81#1": {
"requests": 1,
"prompt_tokens": 758,
"completion_tokens": 29,
"allowance_tokens": 32010,
"length_stops": 0,
"errors": 0,
"seconds": 0.301,
"budget_s": 0.24508064303992655,
"last_prompt_tokens": 758
},
"bfcl:parallel_92#1": {
"requests": 1,
"prompt_tokens": 470,
"completion_tokens": 72,
"allowance_tokens": 32298,
"length_stops": 0,
"errors": 0,
"seconds": 0.538,
"budget_s": 0.4631448182821841,
"last_prompt_tokens": 470
},
"bfcl:parallel_multiple_84#1": {
"requests": 1,
"prompt_tokens": 823,
"completion_tokens": 93,
"allowance_tokens": 31945,
"length_stops": 0,
"errors": 0,
"seconds": 0.722,
"budget_s": 0.6363040732348191,
"last_prompt_tokens": 823
},
"bfcl:multi_turn_base_71#1": {
"requests": 5,
"prompt_tokens": 33663,
"completion_tokens": 115,
"allowance_tokens": 130177,
"length_stops": 0,
"errors": 0,
"seconds": 2.596,
"budget_s": 1.545462845319439,
"last_prompt_tokens": 6825
},
"bfcl:simple_python_199#1": {
"requests": 1,
"prompt_tokens": 436,
"completion_tokens": 21,
"allowance_tokens": 32332,
"length_stops": 0,
"errors": 0,
"seconds": 0.28,
"budget_s": 0.15150115162598496,
"last_prompt_tokens": 436
},
"bfcl:multiple_86#1": {
"requests": 1,
"prompt_tokens": 730,
"completion_tokens": 18,
"allowance_tokens": 32038,
"length_stops": 0,
"errors": 0,
"seconds": 0.213,
"budget_s": 0.17122358705784646,
"last_prompt_tokens": 730
},
"bfcl:parallel_93#1": {
"requests": 1,
"prompt_tokens": 559,
"completion_tokens": 63,
"allowance_tokens": 32209,
"length_stops": 0,
"errors": 0,
"seconds": 0.488,
"budget_s": 0.42086627410996835,
"last_prompt_tokens": 559
},
"bfcl:parallel_multiple_85#1": {
"requests": 1,
"prompt_tokens": 775,
"completion_tokens": 86,
"allowance_tokens": 31993,
"length_stops": 0,
"errors": 0,
"seconds": 0.675,
"budget_s": 0.588077557838033,
"last_prompt_tokens": 775
},
"bfcl:multi_turn_base_72#1": {
"requests": 5,
"prompt_tokens": 48678,
"completion_tokens": 1850,
"allowance_tokens": 115162,
"length_stops": 0,
"errors": 0,
"seconds": 14.915000000000001,
"budget_s": 12.781293566057325,
"last_prompt_tokens": 10551
},
"bfcl:simple_python_203#1": {
"requests": 1,
"prompt_tokens": 445,
"completion_tokens": 16,
"allowance_tokens": 32323,
"length_stops": 0,
"errors": 0,
"seconds": 0.252,
"budget_s": 0.12290511899574558,
"last_prompt_tokens": 445
},
"bfcl:multiple_87#1": {
"requests": 1,
"prompt_tokens": 783,
"completion_tokens": 22,
"allowance_tokens": 31985,
"length_stops": 0,
"errors": 0,
"seconds": 0.256,
"budget_s": 0.20256521706375685,
"last_prompt_tokens": 783
},
"bfcl:parallel_95#1": {
"requests": 1,
"prompt_tokens": 592,
"completion_tokens": 37,
"allowance_tokens": 32176,
"length_stops": 0,
"errors": 0,
"seconds": 0.326,
"budget_s": 0.26855353133368665,
"last_prompt_tokens": 592
},
"bfcl:parallel_multiple_90#1": {
"requests": 1,
"prompt_tokens": 778,
"completion_tokens": 61,
"allowance_tokens": 31990,
"length_stops": 0,
"errors": 0,
"seconds": 0.512,
"budget_s": 0.4376810455908351,
"last_prompt_tokens": 778
},
"bfcl:multi_turn_base_73#1": {
"requests": 3,
"prompt_tokens": 20272,
"completion_tokens": 129,
"allowance_tokens": 78032,
"length_stops": 0,
"errors": 0,
"seconds": 2.255,
"budget_s": 1.6389799825825861,
"last_prompt_tokens": 6868
},
"bfcl:simple_python_204#1": {
"requests": 1,
"prompt_tokens": 468,
"completion_tokens": 25,
"allowance_tokens": 32300,
"length_stops": 0,
"errors": 0,
"seconds": 0.273,
"budget_s": 0.18035275903772593,
"last_prompt_tokens": 468
},
"bfcl:multiple_91#1": {
"requests": 1,
"prompt_tokens": 610,
"completion_tokens": 16,
"allowance_tokens": 32158,
"length_stops": 0,
"errors": 0,
"seconds": 0.186,
"budget_s": 0.1444150081124047,
"last_prompt_tokens": 610
},
"bfcl:parallel_96#1": {
"requests": 1,
"prompt_tokens": 662,
"completion_tokens": 72,
"allowance_tokens": 32106,
"length_stops": 0,
"errors": 0,
"seconds": 0.559,
"budget_s": 0.48924253632716874,
"last_prompt_tokens": 662
},
"bfcl:parallel_multiple_93#1": {
"requests": 1,
"prompt_tokens": 622,
"completion_tokens": 50,
"allowance_tokens": 32146,
"length_stops": 0,
"errors": 0,
"seconds": 0.414,
"budget_s": 0.3513832545444418,
"last_prompt_tokens": 622
},
"bfcl:multi_turn_base_74#1": {
"requests": 2,
"prompt_tokens": 20848,
"completion_tokens": 47,
"allowance_tokens": 44688,
"length_stops": 0,
"errors": 0,
"seconds": 2.107,
"budget_s": 1.5963781138427573,
"last_prompt_tokens": 10464
},
"bfcl:simple_python_208#1": {
"requests": 1,
"prompt_tokens": 507,
"completion_tokens": 30,
"allowance_tokens": 32261,
"length_stops": 0,
"errors": 0,
"seconds": 0.302,
"budget_s": 0.21601865965721864,
"last_prompt_tokens": 507
},
"bfcl:multiple_93#1": {
"requests": 1,
"prompt_tokens": 788,
"completion_tokens": 29,
"allowance_tokens": 31980,
"length_stops": 0,
"errors": 0,
"seconds": 0.301,
"budget_s": 0.2462665544532348,
"last_prompt_tokens": 788
},
"bfcl:parallel_97#1": {
"requests": 1,
"prompt_tokens": 557,
"completion_tokens": 68,
"allowance_tokens": 32211,
"length_stops": 0,
"errors": 0,
"seconds": 0.522,
"budget_s": 0.45178252797469504,
"last_prompt_tokens": 557
},
"bfcl:parallel_multiple_95#1": {
"requests": 1,
"prompt_tokens": 936,
"completion_tokens": 145,
"allowance_tokens": 31832,
"length_stops": 0,
"errors": 0,
"seconds": 1.092,
"budget_s": 0.9671286004414381,
"last_prompt_tokens": 936
},
"bfcl:multi_turn_base_85#1": {
"requests": 2,
"prompt_tokens": 13348,
"completion_tokens": 63,
"allowance_tokens": 52188,
"length_stops": 0,
"errors": 0,
"seconds": 1.5999999999999999,
"budget_s": 1.2146271232788952,
"last_prompt_tokens": 6703
},
"bfcl:simple_python_216#1": {
"requests": 1,
"prompt_tokens": 425,
"completion_tokens": 26,
"allowance_tokens": 32343,
"length_stops": 0,
"errors": 0,
"seconds": 0.255,
"budget_s": 0.18161844012086453,
"last_prompt_tokens": 425
},
"bfcl:multiple_96#1": {
"requests": 1,
"prompt_tokens": 923,
"completion_tokens": 21,
"allowance_tokens": 31845,
"length_stops": 0,
"errors": 0,
"seconds": 0.271,
"budget_s": 0.21550124701222428,
"last_prompt_tokens": 923
},
"bfcl:parallel_98#1": {
"requests": 1,
"prompt_tokens": 537,
"completion_tokens": 25,
"allowance_tokens": 32231,
"length_stops": 0,
"errors": 0,
"seconds": 0.241,
"budget_s": 0.19010775629078555,
"last_prompt_tokens": 537
},
"bfcl:parallel_multiple_97#1": {
"requests": 1,
"prompt_tokens": 1070,
"completion_tokens": 114,
"allowance_tokens": 31698,
"length_stops": 0,
"errors": 0,
"seconds": 0.901,
"budget_s": 0.7976278876544268,
"last_prompt_tokens": 1070
},
"bfcl:multi_turn_base_86#1": {
"requests": 3,
"prompt_tokens": 32086,
"completion_tokens": 79,
"allowance_tokens": 66218,
"length_stops": 0,
"errors": 0,
"seconds": 2.657,
"budget_s": 1.8375717491551886,
"last_prompt_tokens": 10774
},
"bfcl:simple_python_217#1": {
"requests": 1,
"prompt_tokens": 525,
"completion_tokens": 39,
"allowance_tokens": 32243,
"length_stops": 0,
"errors": 0,
"seconds": 0.377,
"budget_s": 0.2731158075097675,
"last_prompt_tokens": 525
},
"bfcl:multiple_97#1": {
"requests": 1,
"prompt_tokens": 697,
"completion_tokens": 13,
"allowance_tokens": 32071,
"length_stops": 0,
"errors": 0,
"seconds": 0.179,
"budget_s": 0.13845962572015724,
"last_prompt_tokens": 697
},
"bfcl:parallel_105#1": {
"requests": 1,
"prompt_tokens": 544,
"completion_tokens": 48,
"allowance_tokens": 32224,
"length_stops": 0,
"errors": 0,
"seconds": 0.391,
"budget_s": 0.3301027277480308,
"last_prompt_tokens": 544
},
"bfcl:parallel_multiple_100#1": {
"requests": 1,
"prompt_tokens": 689,
"completion_tokens": 68,
"allowance_tokens": 32079,
"length_stops": 0,
"errors": 0,
"seconds": 0.537,
"budget_s": 0.4697644710949525,
"last_prompt_tokens": 689
},
"bfcl:multi_turn_base_88#1": {
"requests": 3,
"prompt_tokens": 32130,
"completion_tokens": 85,
"allowance_tokens": 66174,
"length_stops": 0,
"errors": 0,
"seconds": 2.699,
"budget_s": 1.8793279189506376,
"last_prompt_tokens": 10807
},
"bfcl:simple_python_221#1": {
"requests": 1,
"prompt_tokens": 465,
"completion_tokens": 22,
"allowance_tokens": 32303,
"length_stops": 0,
"errors": 0,
"seconds": 0.254,
"budget_s": 0.1634129139836826,
"last_prompt_tokens": 465
},
"bfcl:multiple_102#1": {
"requests": 1,
"prompt_tokens": 867,
"completion_tokens": 24,
"allowance_tokens": 31901,
"length_stops": 0,
"errors": 0,
"seconds": 0.282,
"budget_s": 0.2272189829225318,
"last_prompt_tokens": 867
},
"bfcl:parallel_107#1": {
"requests": 1,
"prompt_tokens": 436,
"completion_tokens": 83,
"allowance_tokens": 32332,
"length_stops": 0,
"errors": 0,
"seconds": 0.605,
"budget_s": 0.5274097378579138,
"last_prompt_tokens": 436
},
"bfcl:parallel_multiple_104#1": {
"requests": 1,
"prompt_tokens": 773,
"completion_tokens": 56,
"allowance_tokens": 31995,
"length_stops": 0,
"errors": 0,
"seconds": 0.476,
"budget_s": 0.4086049721594889,
"last_prompt_tokens": 773
},
"bfcl:multi_turn_base_89#1": {
"requests": 4,
"prompt_tokens": 27051,
"completion_tokens": 174,
"allowance_tokens": 104021,
"length_stops": 0,
"errors": 0,
"seconds": 2.785,
"budget_s": 1.9184107295153914,
"last_prompt_tokens": 6881
},
"bfcl:simple_python_225#1": {
"requests": 1,
"prompt_tokens": 519,
"completion_tokens": 28,
"allowance_tokens": 32249,
"length_stops": 0,
"errors": 0,
"seconds": 0.321,
"budget_s": 0.20677630894658763,
"last_prompt_tokens": 519
},
"bfcl:multiple_103#1": {
"requests": 1,
"prompt_tokens": 1014,
"completion_tokens": 26,
"allowance_tokens": 31754,
"length_stops": 0,
"errors": 0,
"seconds": 0.319,
"budget_s": 0.2584570882505224,
"last_prompt_tokens": 1014
},
"bfcl:parallel_108#1": {
"requests": 1,
"prompt_tokens": 452,
"completion_tokens": 21,
"allowance_tokens": 32316,
"length_stops": 0,
"errors": 0,
"seconds": 0.204,
"budget_s": 0.15609984858097936,
"last_prompt_tokens": 452
},
"bfcl:parallel_multiple_105#1": {
"requests": 1,
"prompt_tokens": 891,
"completion_tokens": 86,
"allowance_tokens": 31877,
"length_stops": 0,
"errors": 0,
"seconds": 0.687,
"budget_s": 0.6055976906756521,
"last_prompt_tokens": 891
},
"bfcl:multi_turn_base_90#1": {
"requests": 4,
"prompt_tokens": 43539,
"completion_tokens": 600,
"allowance_tokens": 87533,
"length_stops": 0,
"errors": 0,
"seconds": 6.5,
"budget_s": 5.138577048721855,
"last_prompt_tokens": 11174
},
"bfcl:simple_python_230#1": {
"requests": 1,
"prompt_tokens": 449,
"completion_tokens": 23,
"allowance_tokens": 32319,
"length_stops": 0,
"errors": 0,
"seconds": 0.28,
"budget_s": 0.1678956839155883,
"last_prompt_tokens": 449
},
"bfcl:multiple_108#1": {
"requests": 1,
"prompt_tokens": 719,
"completion_tokens": 23,
"allowance_tokens": 32049,
"length_stops": 0,
"errors": 0,
"seconds": 0.246,
"budget_s": 0.20268301265353883,
"last_prompt_tokens": 719
},
"bfcl:parallel_110#1": {
"requests": 1,
"prompt_tokens": 512,
"completion_tokens": 37,
"allowance_tokens": 32256,
"length_stops": 0,
"errors": 0,
"seconds": 0.318,
"budget_s": 0.26065146358355923,
"last_prompt_tokens": 512
},
"bfcl:parallel_multiple_106#1": {
"requests": 1,
"prompt_tokens": 867,
"completion_tokens": 84,
"allowance_tokens": 31901,
"length_stops": 0,
"errors": 0,
"seconds": 0.673,
"budget_s": 0.5907370376477916,
"last_prompt_tokens": 867
},
"bfcl:multi_turn_base_92#1": {
"requests": 5,
"prompt_tokens": 35727,
"completion_tokens": 806,
"allowance_tokens": 128113,
"length_stops": 0,
"errors": 0,
"seconds": 7.284999999999999,
"budget_s": 5.878797292916089,
"last_prompt_tokens": 7567
},
"bfcl:simple_python_235#1": {
"requests": 1,
"prompt_tokens": 411,
"completion_tokens": 20,
"allowance_tokens": 32357,
"length_stops": 0,
"errors": 0,
"seconds": 0.273,
"budget_s": 0.1453248412906813,
"last_prompt_tokens": 411
},
"bfcl:multiple_110#1": {
"requests": 1,
"prompt_tokens": 675,
"completion_tokens": 21,
"allowance_tokens": 32093,
"length_stops": 0,
"errors": 0,
"seconds": 0.231,
"budget_s": 0.18534246272315727,
"last_prompt_tokens": 675
},
"bfcl:parallel_114#1": {
"requests": 1,
"prompt_tokens": 532,
"completion_tokens": 91,
"allowance_tokens": 32236,
"length_stops": 0,
"errors": 0,
"seconds": 0.669,
"budget_s": 0.5892597709235784,
"last_prompt_tokens": 532
},
"bfcl:parallel_multiple_108#1": {
"requests": 1,
"prompt_tokens": 952,
"completion_tokens": 84,
"allowance_tokens": 31816,
"length_stops": 0,
"errors": 0,
"seconds": 0.689,
"budget_s": 0.602270327883169,
"last_prompt_tokens": 952
},
"bfcl:multi_turn_base_93#1": {
"requests": 3,
"prompt_tokens": 20170,
"completion_tokens": 212,
"allowance_tokens": 78134,
"length_stops": 0,
"errors": 0,
"seconds": 2.822,
"budget_s": 2.15060316262223,
"last_prompt_tokens": 6846
},
"bfcl:simple_python_239#1": {
"requests": 1,
"prompt_tokens": 434,
"completion_tokens": 23,
"allowance_tokens": 32334,
"length_stops": 0,
"errors": 0,
"seconds": 0.258,
"budget_s": 0.1667340438787424,
"last_prompt_tokens": 434
},
"bfcl:multiple_111#1": {
"requests": 1,
"prompt_tokens": 788,
"completion_tokens": 21,
"allowance_tokens": 31980,
"length_stops": 0,
"errors": 0,
"seconds": 0.251,
"budget_s": 0.20022012332615918,
"last_prompt_tokens": 788
},
"bfcl:parallel_116#1": {
"requests": 1,
"prompt_tokens": 501,
"completion_tokens": 21,
"allowance_tokens": 32267,
"length_stops": 0,
"errors": 0,
"seconds": 0.208,
"budget_s": 0.15589242246632012,
"last_prompt_tokens": 501
},
"bfcl:parallel_multiple_109#1": {
"requests": 1,
"prompt_tokens": 911,
"completion_tokens": 65,
"allowance_tokens": 31857,
"length_stops": 0,
"errors": 0,
"seconds": 0.555,
"budget_s": 0.4821079573120586,
"last_prompt_tokens": 911
},
"bfcl:multi_turn_base_94#1": {
"requests": 3,
"prompt_tokens": 20433,
"completion_tokens": 321,
"allowance_tokens": 77871,
"length_stops": 0,
"errors": 0,
"seconds": 3.567,
"budget_s": 2.8422996502758027,
"last_prompt_tokens": 6962
},
"bfcl:simple_python_240#1": {
"requests": 1,
"prompt_tokens": 451,
"completion_tokens": 21,
"allowance_tokens": 32317,
"length_stops": 0,
"errors": 0,
"seconds": 0.24,
"budget_s": 0.15699008131127218,
"last_prompt_tokens": 451
},
"bfcl:multiple_115#1": {
"requests": 1,
"prompt_tokens": 1114,
"completion_tokens": 28,
"allowance_tokens": 31654,
"length_stops": 0,
"errors": 0,
"seconds": 0.357,
"budget_s": 0.2845692601530915,
"last_prompt_tokens": 1114
},
"bfcl:parallel_117#1": {
"requests": 1,
"prompt_tokens": 556,
"completion_tokens": 70,
"allowance_tokens": 32212,
"length_stops": 0,
"errors": 0,
"seconds": 0.552,
"budget_s": 0.46613422257871634,
"last_prompt_tokens": 556
},
"bfcl:parallel_multiple_110#1": {
"requests": 1,
"prompt_tokens": 734,
"completion_tokens": 53,
"allowance_tokens": 32034,
"length_stops": 0,
"errors": 0,
"seconds": 0.461,
"budget_s": 0.3868202681539281,
"last_prompt_tokens": 734
},
"bfcl:multi_turn_base_96#1": {
"requests": 2,
"prompt_tokens": 13438,
"completion_tokens": 63,
"allowance_tokens": 52098,
"length_stops": 0,
"errors": 0,
"seconds": 1.601,
"budget_s": 1.222815807103869,
"last_prompt_tokens": 6764
},
"bfcl:simple_python_247#1": {
"requests": 1,
"prompt_tokens": 481,
"completion_tokens": 28,
"allowance_tokens": 32287,
"length_stops": 0,
"errors": 0,
"seconds": 0.286,
"budget_s": 0.20239882018307856,
"last_prompt_tokens": 481
},
"bfcl:multiple_116#1": {
"requests": 1,
"prompt_tokens": 640,
"completion_tokens": 17,
"allowance_tokens": 32128,
"length_stops": 0,
"errors": 0,
"seconds": 0.199,
"budget_s": 0.1565835780586574,
"last_prompt_tokens": 640
},
"bfcl:parallel_118#1": {
"requests": 1,
"prompt_tokens": 470,
"completion_tokens": 64,
"allowance_tokens": 32298,
"length_stops": 0,
"errors": 0,
"seconds": 0.486,
"budget_s": 0.41823386367587684,
"last_prompt_tokens": 470
},
"bfcl:parallel_multiple_112#1": {
"requests": 1,
"prompt_tokens": 783,
"completion_tokens": 59,
"allowance_tokens": 31985,
"length_stops": 0,
"errors": 0,
"seconds": 0.498,
"budget_s": 0.4289211980752503,
"last_prompt_tokens": 783
},
"bfcl:multi_turn_base_97#1": {
"requests": 6,
"prompt_tokens": 54939,
"completion_tokens": 182,
"allowance_tokens": 141669,
"length_stops": 0,
"errors": 0,
"seconds": 3.823,
"budget_s": 2.2889134047837376,
"last_prompt_tokens": 9355
},
"bfcl:simple_python_249#1": {
"requests": 1,
"prompt_tokens": 472,
"completion_tokens": 20,
"allowance_tokens": 32296,
"length_stops": 0,
"errors": 0,
"seconds": 0.299,
"budget_s": 0.15340153494687492,
"last_prompt_tokens": 472
},
"bfcl:multiple_117#1": {
"requests": 1,
"prompt_tokens": 962,
"completion_tokens": 23,
"allowance_tokens": 31806,
"length_stops": 0,
"errors": 0,
"seconds": 0.289,
"budget_s": 0.2345146007269424,
"last_prompt_tokens": 962
},
"bfcl:parallel_120#1": {
"requests": 1,
"prompt_tokens": 503,
"completion_tokens": 53,
"allowance_tokens": 32265,
"length_stops": 0,
"errors": 0,
"seconds": 0.424,
"budget_s": 0.3564660563580286,
"last_prompt_tokens": 503
},
"bfcl:parallel_multiple_120#1": {
"requests": 1,
"prompt_tokens": 699,
"completion_tokens": 48,
"allowance_tokens": 32069,
"length_stops": 0,
"errors": 0,
"seconds": 0.408,
"budget_s": 0.3518334578643208,
"last_prompt_tokens": 699
},
"bfcl:multi_turn_base_98#1": {
"requests": 3,
"prompt_tokens": 31414,
"completion_tokens": 76,
"allowance_tokens": 66890,
"length_stops": 0,
"errors": 0,
"seconds": 2.593,
"budget_s": 1.7880753114046097,
"last_prompt_tokens": 10535
},
"bfcl:simple_python_252#1": {
"requests": 1,
"prompt_tokens": 543,
"completion_tokens": 29,
"allowance_tokens": 32225,
"length_stops": 0,
"errors": 0,
"seconds": 0.326,
"budget_s": 0.21716681326422124,
"last_prompt_tokens": 543
},
"bfcl:multiple_121#1": {
"requests": 1,
"prompt_tokens": 863,
"completion_tokens": 17,
"allowance_tokens": 31905,
"length_stops": 0,
"errors": 0,
"seconds": 0.241,
"budget_s": 0.1859130343384559,
"last_prompt_tokens": 863
},
"bfcl:parallel_121#1": {
"requests": 1,
"prompt_tokens": 497,
"completion_tokens": 52,
"allowance_tokens": 32271,
"length_stops": 0,
"errors": 0,
"seconds": 0.415,
"budget_s": 0.35004064579670807,
"last_prompt_tokens": 497
},
"bfcl:parallel_multiple_121#1": {
"requests": 1,
"prompt_tokens": 898,
"completion_tokens": 96,
"allowance_tokens": 31870,
"length_stops": 0,
"errors": 0,
"seconds": 0.758,
"budget_s": 0.6682631508715471,
"last_prompt_tokens": 898
},
"bfcl:multi_turn_base_99#1": {
"requests": 4,
"prompt_tokens": 27541,
"completion_tokens": 393,
"allowance_tokens": 103531,
"length_stops": 0,
"errors": 0,
"seconds": 4.277,
"budget_s": 3.296460353034531,
"last_prompt_tokens": 7158
},
"bfcl:simple_python_260#1": {
"requests": 1,
"prompt_tokens": 680,
"completion_tokens": 39,
"allowance_tokens": 32088,
"length_stops": 0,
"errors": 0,
"seconds": 0.414,
"budget_s": 0.2954188940638342,
"last_prompt_tokens": 680
},
"bfcl:multiple_122#1": {
"requests": 1,
"prompt_tokens": 1138,
"completion_tokens": 29,
"allowance_tokens": 31630,
"length_stops": 0,
"errors": 0,
"seconds": 0.355,
"budget_s": 0.29409629540738924,
"last_prompt_tokens": 1138
},
"bfcl:parallel_122#1": {
"requests": 1,
"prompt_tokens": 557,
"completion_tokens": 81,
"allowance_tokens": 32211,
"length_stops": 0,
"errors": 0,
"seconds": 0.612,
"budget_s": 0.5329946414601371,
"last_prompt_tokens": 557
},
"bfcl:parallel_multiple_126#1": {
"requests": 1,
"prompt_tokens": 663,
"completion_tokens": 28,
"allowance_tokens": 32105,
"length_stops": 0,
"errors": 0,
"seconds": 0.275,
"budget_s": 0.2270771424655232,
"last_prompt_tokens": 663
},
"bfcl:multi_turn_base_102#1": {
"requests": 5,
"prompt_tokens": 38870,
"completion_tokens": 207,
"allowance_tokens": 124970,
"length_stops": 0,
"errors": 0,
"seconds": 3.449,
"budget_s": 2.265343786587814,
"last_prompt_tokens": 7942
},
"bfcl:simple_python_261#1": {
"requests": 1,
"prompt_tokens": 451,
"completion_tokens": 20,
"allowance_tokens": 32317,
"length_stops": 0,
"errors": 0,
"seconds": 0.278,
"budget_s": 0.15172719589979,
"last_prompt_tokens": 451
},
"bfcl:multiple_123#1": {
"requests": 1,
"prompt_tokens": 877,
"completion_tokens": 27,
"allowance_tokens": 31891,
"length_stops": 0,
"errors": 0,
"seconds": 0.303,
"budget_s": 0.24880662055615504,
"last_prompt_tokens": 877
},
"bfcl:parallel_123#1": {
"requests": 1,
"prompt_tokens": 520,
"completion_tokens": 73,
"allowance_tokens": 32248,
"length_stops": 0,
"errors": 0,
"seconds": 0.553,
"budget_s": 0.4802889083413693,
"last_prompt_tokens": 520
},
"bfcl:parallel_multiple_132#1": {
"requests": 1,
"prompt_tokens": 737,
"completion_tokens": 70,
"allowance_tokens": 32031,
"length_stops": 0,
"errors": 0,
"seconds": 0.566,
"budget_s": 0.49065188348321226,
"last_prompt_tokens": 737
},
"bfcl:multi_turn_base_103#1": {
"requests": 5,
"prompt_tokens": 37653,
"completion_tokens": 152,
"allowance_tokens": 126187,
"length_stops": 0,
"errors": 0,
"seconds": 3.0370000000000004,
"budget_s": 1.8912240356291048,
"last_prompt_tokens": 7728
},
"bfcl:simple_python_263#1": {
"requests": 1,
"prompt_tokens": 429,
"completion_tokens": 22,
"allowance_tokens": 32339,
"length_stops": 0,
"errors": 0,
"seconds": 0.29,
"budget_s": 0.15917524955169882,
"last_prompt_tokens": 429
},
"bfcl:multiple_126#1": {
"requests": 1,
"prompt_tokens": 927,
"completion_tokens": 44,
"allowance_tokens": 31841,
"length_stops": 0,
"errors": 0,
"seconds": 0.42,
"budget_s": 0.356279679461138,
"last_prompt_tokens": 927
},
"bfcl:parallel_126#1": {
"requests": 1,
"prompt_tokens": 494,
"completion_tokens": 47,
"allowance_tokens": 32274,
"length_stops": 0,
"errors": 0,
"seconds": 0.383,
"budget_s": 0.31836202522144147,
"last_prompt_tokens": 494
},
"bfcl:parallel_multiple_134#1": {
"requests": 1,
"prompt_tokens": 687,
"completion_tokens": 43,
"allowance_tokens": 32081,
"length_stops": 0,
"errors": 0,
"seconds": 0.373,
"budget_s": 0.31932726189525656,
"last_prompt_tokens": 687
},
"bfcl:multi_turn_base_105#1": {
"requests": 2,
"prompt_tokens": 10283,
"completion_tokens": 40,
"allowance_tokens": 55253,
"length_stops": 0,
"errors": 0,
"seconds": 1.209,
"budget_s": 0.8775014792317823,
"last_prompt_tokens": 5174
},
"bfcl:simple_python_269#1": {
"requests": 1,
"prompt_tokens": 534,
"completion_tokens": 40,
"allowance_tokens": 32234,
"length_stops": 0,
"errors": 0,
"seconds": 0.364,
"budget_s": 0.2815814564601366,
"last_prompt_tokens": 534
},
"bfcl:multiple_130#1": {
"requests": 1,
"prompt_tokens": 855,
"completion_tokens": 33,
"allowance_tokens": 31913,
"length_stops": 0,
"errors": 0,
"seconds": 0.339,
"budget_s": 0.28084995517819467,
"last_prompt_tokens": 855
},
"bfcl:parallel_127#1": {
"requests": 1,
"prompt_tokens": 521,
"completion_tokens": 93,
"allowance_tokens": 32247,
"length_stops": 0,
"errors": 0,
"seconds": 0.683,
"budget_s": 0.5996186991564335,
"last_prompt_tokens": 521
},
"bfcl:parallel_multiple_136#1": {
"requests": 1,
"prompt_tokens": 1181,
"completion_tokens": 90,
"allowance_tokens": 31587,
"length_stops": 0,
"errors": 0,
"seconds": 0.759,
"budget_s": 0.6684234440464409,
"last_prompt_tokens": 1181
},
"bfcl:multi_turn_base_110#1": {
"requests": 6,
"prompt_tokens": 46895,
"completion_tokens": 203,
"allowance_tokens": 149713,
"length_stops": 0,
"errors": 0,
"seconds": 3.6750000000000003,
"budget_s": 2.2544669696026443,
"last_prompt_tokens": 8044
},
"bfcl:simple_python_270#1": {
"requests": 1,
"prompt_tokens": 446,
"completion_tokens": 20,
"allowance_tokens": 32322,
"length_stops": 0,
"errors": 0,
"seconds": 0.296,
"budget_s": 0.14994021945046584,
"last_prompt_tokens": 446
},
"bfcl:multiple_131#1": {
"requests": 1,
"prompt_tokens": 954,
"completion_tokens": 28,
"allowance_tokens": 31814,
"length_stops": 0,
"errors": 0,
"seconds": 0.318,
"budget_s": 0.2636098760127929,
"last_prompt_tokens": 954
},
"bfcl:parallel_130#1": {
"requests": 1,
"prompt_tokens": 554,
"completion_tokens": 63,
"allowance_tokens": 32214,
"length_stops": 0,
"errors": 0,
"seconds": 0.491,
"budget_s": 0.4232707667820894,
"last_prompt_tokens": 554
},
"bfcl:parallel_multiple_138#1": {
"requests": 1,
"prompt_tokens": 992,
"completion_tokens": 74,
"allowance_tokens": 31776,
"length_stops": 0,
"errors": 0,
"seconds": 0.634,
"budget_s": 0.5470014049359185,
"last_prompt_tokens": 992
},
"bfcl:multi_turn_base_111#1": {
"requests": 4,
"prompt_tokens": 30176,
"completion_tokens": 128,
"allowance_tokens": 100896,
"length_stops": 0,
"errors": 0,
"seconds": 2.65,
"budget_s": 1.7398481354410615,
"last_prompt_tokens": 7691
},
"bfcl:simple_python_272#1": {
"requests": 1,
"prompt_tokens": 384,
"completion_tokens": 12,
"allowance_tokens": 32384,
"length_stops": 0,
"errors": 0,
"seconds": 0.199,
"budget_s": 0.09405499833975349,
"last_prompt_tokens": 384
},
"bfcl:multiple_132#1": {
"requests": 1,
"prompt_tokens": 917,
"completion_tokens": 35,
"allowance_tokens": 31851,
"length_stops": 0,
"errors": 0,
"seconds": 0.36,
"budget_s": 0.30144351870340147,
"last_prompt_tokens": 917
},
"bfcl:parallel_131#1": {
"requests": 1,
"prompt_tokens": 510,
"completion_tokens": 81,
"allowance_tokens": 32258,
"length_stops": 0,
"errors": 0,
"seconds": 0.609,
"budget_s": 0.5263268622265797,
"last_prompt_tokens": 510
},
"bfcl:parallel_multiple_139#1": {
"requests": 1,
"prompt_tokens": 1344,
"completion_tokens": 109,
"allowance_tokens": 31424,
"length_stops": 0,
"errors": 0,
"seconds": 0.922,
"budget_s": 0.805657769055057,
"last_prompt_tokens": 1344
},
"bfcl:multi_turn_base_116#1": {
"requests": 6,
"prompt_tokens": 31881,
"completion_tokens": 287,
"allowance_tokens": 164727,
"length_stops": 0,
"errors": 0,
"seconds": 3.6900000000000004,
"budget_s": 2.4356412553343136,
"last_prompt_tokens": 5528
},
"bfcl:simple_python_280#1": {
"requests": 1,
"prompt_tokens": 449,
"completion_tokens": 17,
"allowance_tokens": 32319,
"length_stops": 0,
"errors": 0,
"seconds": 0.277,
"budget_s": 0.13275413906043293,
"last_prompt_tokens": 449
},
"bfcl:multiple_135#1": {
"requests": 1,
"prompt_tokens": 671,
"completion_tokens": 33,
"allowance_tokens": 32097,
"length_stops": 0,
"errors": 0,
"seconds": 0.308,
"budget_s": 0.2579083757829169,
"last_prompt_tokens": 671
},
"bfcl:parallel_134#1": {
"requests": 1,
"prompt_tokens": 538,
"completion_tokens": 79,
"allowance_tokens": 32230,
"length_stops": 0,
"errors": 0,
"seconds": 0.592,
"budget_s": 0.5183161778807179,
"last_prompt_tokens": 538
},
"bfcl:parallel_multiple_145#1": {
"requests": 1,
"prompt_tokens": 1416,
"completion_tokens": 123,
"allowance_tokens": 31352,
"length_stops": 0,
"errors": 0,
"seconds": 1.024,
"budget_s": 0.9005700925352966,
"last_prompt_tokens": 1416
},
"bfcl:multi_turn_base_117#1": {
"requests": 6,
"prompt_tokens": 32052,
"completion_tokens": 293,
"allowance_tokens": 164556,
"length_stops": 0,
"errors": 0,
"seconds": 3.7299999999999995,
"budget_s": 2.4763713743315936,
"last_prompt_tokens": 5580
},
"bfcl:simple_python_282#1": {
"requests": 1,
"prompt_tokens": 473,
"completion_tokens": 28,
"allowance_tokens": 32295,
"length_stops": 0,
"errors": 0,
"seconds": 0.353,
"budget_s": 0.20251585867565428,
"last_prompt_tokens": 473
},
"bfcl:multiple_139#1": {
"requests": 1,
"prompt_tokens": 1132,
"completion_tokens": 23,
"allowance_tokens": 31636,
"length_stops": 0,
"errors": 0,
"seconds": 0.313,
"budget_s": 0.2571027287429413,
"last_prompt_tokens": 1132
},
"bfcl:parallel_137#1": {
"requests": 1,
"prompt_tokens": 554,
"completion_tokens": 238,
"allowance_tokens": 32214,
"length_stops": 0,
"errors": 0,
"seconds": 1.647,
"budget_s": 1.479673408917109,
"last_prompt_tokens": 554
},
"bfcl:parallel_multiple_151#1": {
"requests": 1,
"prompt_tokens": 1424,
"completion_tokens": 136,
"allowance_tokens": 31344,
"length_stops": 0,
"errors": 0,
"seconds": 1.112,
"budget_s": 0.9808213198463924,
"last_prompt_tokens": 1424
},
"bfcl:multi_turn_base_118#1": {
"requests": 5,
"prompt_tokens": 28592,
"completion_tokens": 1434,
"allowance_tokens": 135248,
"length_stops": 0,
"errors": 0,
"seconds": 11.213,
"budget_s": 9.5791808708901,
"last_prompt_tokens": 6481
},
"bfcl:simple_python_285#1": {
"requests": 1,
"prompt_tokens": 494,
"completion_tokens": 22,
"allowance_tokens": 32274,
"length_stops": 0,
"errors": 0,
"seconds": 0.301,
"budget_s": 0.16929860957312104,
"last_prompt_tokens": 494
},
"bfcl:multiple_140#1": {
"requests": 1,
"prompt_tokens": 843,
"completion_tokens": 24,
"allowance_tokens": 31925,
"length_stops": 0,
"errors": 0,
"seconds": 0.28,
"budget_s": 0.22630121273405596,
"last_prompt_tokens": 843
},
"bfcl:parallel_139#1": {
"requests": 1,
"prompt_tokens": 595,
"completion_tokens": 82,
"allowance_tokens": 32173,
"length_stops": 0,
"errors": 0,
"seconds": 0.622,
"budget_s": 0.5443892840742413,
"last_prompt_tokens": 595
},
"bfcl:parallel_multiple_154#1": {
"requests": 1,
"prompt_tokens": 1023,
"completion_tokens": 78,
"allowance_tokens": 31745,
"length_stops": 0,
"errors": 0,
"seconds": 0.661,
"budget_s": 0.5764124752824812,
"last_prompt_tokens": 1023
},
"bfcl:multi_turn_base_122#1": {
"requests": 4,
"prompt_tokens": 21154,
"completion_tokens": 200,
"allowance_tokens": 109918,
"length_stops": 0,
"errors": 0,
"seconds": 2.6999999999999997,
"budget_s": 1.887481391392354,
"last_prompt_tokens": 5425
},
"bfcl:simple_python_289#1": {
"requests": 1,
"prompt_tokens": 422,
"completion_tokens": 18,
"allowance_tokens": 32346,
"length_stops": 0,
"errors": 0,
"seconds": 0.243,
"budget_s": 0.13621226207884896,
"last_prompt_tokens": 422
},
"bfcl:multiple_143#1": {
"requests": 1,
"prompt_tokens": 1094,
"completion_tokens": 53,
"allowance_tokens": 31674,
"length_stops": 0,
"errors": 0,
"seconds": 0.506,
"budget_s": 0.43442545953260675,
"last_prompt_tokens": 1094
},
"bfcl:parallel_140#1": {
"requests": 1,
"prompt_tokens": 498,
"completion_tokens": 58,
"allowance_tokens": 32270,
"length_stops": 0,
"errors": 0,
"seconds": 0.457,
"budget_s": 0.38723508078740587,
"last_prompt_tokens": 498
},
"bfcl:parallel_multiple_157#1": {
"requests": 1,
"prompt_tokens": 1106,
"completion_tokens": 131,
"allowance_tokens": 31662,
"length_stops": 0,
"errors": 0,
"seconds": 1.024,
"budget_s": 0.9085522178226886,
"last_prompt_tokens": 1106
},
"bfcl:multi_turn_base_124#1": {
"requests": 3,
"prompt_tokens": 27014,
"completion_tokens": 93,
"allowance_tokens": 71290,
"length_stops": 0,
"errors": 0,
"seconds": 2.439,
"budget_s": 1.7087982294244535,
"last_prompt_tokens": 9101
},
"bfcl:simple_python_291#1": {
"requests": 1,
"prompt_tokens": 544,
"completion_tokens": 32,
"allowance_tokens": 32224,
"length_stops": 0,
"errors": 0,
"seconds": 0.336,
"budget_s": 0.23653438581925051,
"last_prompt_tokens": 544
},
"bfcl:multiple_144#1": {
"requests": 1,
"prompt_tokens": 619,
"completion_tokens": 23,
"allowance_tokens": 32149,
"length_stops": 0,
"errors": 0,
"seconds": 0.235,
"budget_s": 0.19201016547959562,
"last_prompt_tokens": 619
},
"bfcl:parallel_143#1": {
"requests": 1,
"prompt_tokens": 505,
"completion_tokens": 48,
"allowance_tokens": 32263,
"length_stops": 0,
"errors": 0,
"seconds": 0.388,
"budget_s": 0.3282344032435598,
"last_prompt_tokens": 505
},
"bfcl:parallel_multiple_159#1": {
"requests": 1,
"prompt_tokens": 1330,
"completion_tokens": 133,
"allowance_tokens": 31438,
"length_stops": 0,
"errors": 0,
"seconds": 1.079,
"budget_s": 0.9509484653201008,
"last_prompt_tokens": 1330
},
"bfcl:multi_turn_base_126#1": {
"requests": 4,
"prompt_tokens": 21097,
"completion_tokens": 146,
"allowance_tokens": 109975,
"length_stops": 0,
"errors": 0,
"seconds": 2.3320000000000003,
"budget_s": 1.5572861106099414,
"last_prompt_tokens": 5405
},
"bfcl:simple_python_293#1": {
"requests": 1,
"prompt_tokens": 543,
"completion_tokens": 29,
"allowance_tokens": 32225,
"length_stops": 0,
"errors": 0,
"seconds": 0.335,
"budget_s": 0.21869457724241786,
"last_prompt_tokens": 543
},
"bfcl:multiple_146#1": {
"requests": 1,
"prompt_tokens": 663,
"completion_tokens": 25,
"allowance_tokens": 32105,
"length_stops": 0,
"errors": 0,
"seconds": 0.258,
"budget_s": 0.21011345352480787,
"last_prompt_tokens": 663
},
"bfcl:parallel_147#1": {
"requests": 1,
"prompt_tokens": 538,
"completion_tokens": 38,
"allowance_tokens": 32230,
"length_stops": 0,
"errors": 0,
"seconds": 0.325,
"budget_s": 0.2725828118848435,
"last_prompt_tokens": 538
},
"bfcl:parallel_multiple_160#1": {
"requests": 1,
"prompt_tokens": 887,
"completion_tokens": 60,
"allowance_tokens": 31881,
"length_stops": 0,
"errors": 0,
"seconds": 0.523,
"budget_s": 0.45078329735629147,
"last_prompt_tokens": 887
},
"bfcl:multi_turn_base_127#1": {
"requests": 4,
"prompt_tokens": 22076,
"completion_tokens": 1196,
"allowance_tokens": 108996,
"length_stops": 0,
"errors": 0,
"seconds": 9.399999999999999,
"budget_s": 8.086129964720694,
"last_prompt_tokens": 6041
},
"bfcl:simple_python_303#1": {
"requests": 1,
"prompt_tokens": 465,
"completion_tokens": 31,
"allowance_tokens": 32303,
"length_stops": 0,
"errors": 0,
"seconds": 0.335,
"budget_s": 0.22109706765328224,
"last_prompt_tokens": 465
},
"bfcl:multiple_150#1": {
"requests": 1,
"prompt_tokens": 1223,
"completion_tokens": 34,
"allowance_tokens": 31545,
"length_stops": 0,
"errors": 0,
"seconds": 0.396,
"budget_s": 0.33699920371332237,
"last_prompt_tokens": 1223
},
"bfcl:parallel_149#1": {
"requests": 1,
"prompt_tokens": 475,
"completion_tokens": 40,
"allowance_tokens": 32293,
"length_stops": 0,
"errors": 0,
"seconds": 0.339,
"budget_s": 0.27676254925258115,
"last_prompt_tokens": 475
},
"bfcl:parallel_multiple_161#1": {
"requests": 1,
"prompt_tokens": 935,
"completion_tokens": 70,
"allowance_tokens": 31833,
"length_stops": 0,
"errors": 0,
"seconds": 0.594,
"budget_s": 0.5177723904652279,
"last_prompt_tokens": 935
},
"bfcl:multi_turn_base_128#1": {
"requests": 4,
"prompt_tokens": 20983,
"completion_tokens": 115,
"allowance_tokens": 110089,
"length_stops": 0,
"errors": 0,
"seconds": 2.1239999999999997,
"budget_s": 1.3616074259191138,
"last_prompt_tokens": 5355
},
"bfcl:simple_python_304#1": {
"requests": 1,
"prompt_tokens": 514,
"completion_tokens": 31,
"allowance_tokens": 32254,
"length_stops": 0,
"errors": 0,
"seconds": 0.342,
"budget_s": 0.22765975633276303,
"last_prompt_tokens": 514
},
"bfcl:multiple_151#1": {
"requests": 1,
"prompt_tokens": 916,
"completion_tokens": 28,
"allowance_tokens": 31852,
"length_stops": 0,
"errors": 0,
"seconds": 0.319,
"budget_s": 0.2613963946427026,
"last_prompt_tokens": 916
},
"bfcl:parallel_150#1": {
"requests": 1,
"prompt_tokens": 594,
"completion_tokens": 58,
"allowance_tokens": 32174,
"length_stops": 0,
"errors": 0,
"seconds": 0.462,
"budget_s": 0.4009860091512626,
"last_prompt_tokens": 594
},
"bfcl:parallel_multiple_162#1": {
"requests": 1,
"prompt_tokens": 1083,
"completion_tokens": 107,
"allowance_tokens": 31685,
"length_stops": 0,
"errors": 0,
"seconds": 0.858,
"budget_s": 0.7614163430811769,
"last_prompt_tokens": 1083
},
"bfcl:multi_turn_base_130#1": {
"requests": 5,
"prompt_tokens": 26391,
"completion_tokens": 139,
"allowance_tokens": 137449,
"length_stops": 0,
"errors": 0,
"seconds": 2.495,
"budget_s": 1.5145841909785671,
"last_prompt_tokens": 5407
},
"bfcl:simple_python_306#1": {
"requests": 1,
"prompt_tokens": 481,
"completion_tokens": 29,
"allowance_tokens": 32287,
"length_stops": 0,
"errors": 0,
"seconds": 0.342,
"budget_s": 0.21160980511370553,
"last_prompt_tokens": 481
},
"bfcl:multiple_156#1": {
"requests": 1,
"prompt_tokens": 845,
"completion_tokens": 28,
"allowance_tokens": 31923,
"length_stops": 0,
"errors": 0,
"seconds": 0.308,
"budget_s": 0.25251999699985583,
"last_prompt_tokens": 845
},
"bfcl:parallel_151#1": {
"requests": 1,
"prompt_tokens": 525,
"completion_tokens": 65,
"allowance_tokens": 32243,
"length_stops": 0,
"errors": 0,
"seconds": 0.497,
"budget_s": 0.4344833230340587,
"last_prompt_tokens": 525
},
"bfcl:parallel_multiple_163#1": {
"requests": 1,
"prompt_tokens": 740,
"completion_tokens": 48,
"allowance_tokens": 32028,
"length_stops": 0,
"errors": 0,
"seconds": 0.425,
"budget_s": 0.35994482758415647,
"last_prompt_tokens": 740
},
"bfcl:multi_turn_base_131#1": {
"requests": 7,
"prompt_tokens": 52936,
"completion_tokens": 148,
"allowance_tokens": 176440,
"length_stops": 0,
"errors": 0,
"seconds": 3.476,
"budget_s": 1.866885247906325,
"last_prompt_tokens": 7737
},
"bfcl:simple_python_316#1": {
"requests": 1,
"prompt_tokens": 472,
"completion_tokens": 20,
"allowance_tokens": 32296,
"length_stops": 0,
"errors": 0,
"seconds": 0.316,
"budget_s": 0.15225571196322746,
"last_prompt_tokens": 472
},
"bfcl:multiple_158#1": {
"requests": 1,
"prompt_tokens": 1136,
"completion_tokens": 23,
"allowance_tokens": 31632,
"length_stops": 0,
"errors": 0,
"seconds": 0.315,
"budget_s": 0.25570654110652535,
"last_prompt_tokens": 1136
},
"bfcl:parallel_158#1": {
"requests": 1,
"prompt_tokens": 492,
"completion_tokens": 60,
"allowance_tokens": 32276,
"length_stops": 0,
"errors": 0,
"seconds": 0.467,
"budget_s": 0.3960935675114615,
"last_prompt_tokens": 492
},
"bfcl:parallel_multiple_164#1": {
"requests": 1,
"prompt_tokens": 791,
"completion_tokens": 76,
"allowance_tokens": 31977,
"length_stops": 0,
"errors": 0,
"seconds": 0.61,
"budget_s": 0.5317910299387381,
"last_prompt_tokens": 791
},
"bfcl:multi_turn_base_135#1": {
"requests": 4,
"prompt_tokens": 21022,
"completion_tokens": 239,
"allowance_tokens": 110050,
"length_stops": 0,
"errors": 0,
"seconds": 2.946,
"budget_s": 2.1217318566803196,
"last_prompt_tokens": 5398
},
"bfcl:simple_python_332#1": {
"requests": 1,
"prompt_tokens": 433,
"completion_tokens": 19,
"allowance_tokens": 32335,
"length_stops": 0,
"errors": 0,
"seconds": 0.25,
"budget_s": 0.14148339792625528,
"last_prompt_tokens": 433
},
"bfcl:multiple_160#1": {
"requests": 1,
"prompt_tokens": 1071,
"completion_tokens": 22,
"allowance_tokens": 31697,
"length_stops": 0,
"errors": 0,
"seconds": 0.298,
"budget_s": 0.24156121292384422,
"last_prompt_tokens": 1071
},
"bfcl:parallel_162#1": {
"requests": 1,
"prompt_tokens": 600,
"completion_tokens": 79,
"allowance_tokens": 32168,
"length_stops": 0,
"errors": 0,
"seconds": 0.601,
"budget_s": 0.5250341673745004,
"last_prompt_tokens": 600
},
"bfcl:parallel_multiple_168#1": {
"requests": 1,
"prompt_tokens": 944,
"completion_tokens": 98,
"allowance_tokens": 31824,
"length_stops": 0,
"errors": 0,
"seconds": 0.78,
"budget_s": 0.6851493025419759,
"last_prompt_tokens": 944
},
"bfcl:multi_turn_base_150#1": {
"requests": 2,
"prompt_tokens": 13415,
"completion_tokens": 187,
"allowance_tokens": 52121,
"length_stops": 0,
"errors": 0,
"seconds": 2.44,
"budget_s": 1.9901976872752325,
"last_prompt_tokens": 6797
},
"bfcl:simple_python_334#1": {
"requests": 1,
"prompt_tokens": 528,
"completion_tokens": 34,
"allowance_tokens": 32240,
"length_stops": 0,
"errors": 0,
"seconds": 0.328,
"budget_s": 0.24437012799430693,
"last_prompt_tokens": 528
},
"bfcl:multiple_163#1": {
"requests": 1,
"prompt_tokens": 728,
"completion_tokens": 24,
"allowance_tokens": 32040,
"length_stops": 0,
"errors": 0,
"seconds": 0.253,
"budget_s": 0.20987786227053337,
"last_prompt_tokens": 728
},
"bfcl:parallel_163#1": {
"requests": 1,
"prompt_tokens": 593,
"completion_tokens": 87,
"allowance_tokens": 32175,
"length_stops": 0,
"errors": 0,
"seconds": 0.648,
"budget_s": 0.5727657844143618,
"last_prompt_tokens": 593
},
"bfcl:parallel_multiple_170#1": {
"requests": 1,
"prompt_tokens": 1284,
"completion_tokens": 121,
"allowance_tokens": 31484,
"length_stops": 0,
"errors": 0,
"seconds": 0.987,
"budget_s": 0.8698610044359799,
"last_prompt_tokens": 1284
},
"bfcl:multi_turn_base_153#1": {
"requests": 5,
"prompt_tokens": 34843,
"completion_tokens": 440,
"allowance_tokens": 128997,
"length_stops": 0,
"errors": 0,
"seconds": 4.839,
"budget_s": 3.6022647122442972,
"last_prompt_tokens": 7277
},
"bfcl:simple_python_338#1": {
"requests": 1,
"prompt_tokens": 481,
"completion_tokens": 24,
"allowance_tokens": 32287,
"length_stops": 0,
"errors": 0,
"seconds": 0.311,
"budget_s": 0.17841356061039207,
"last_prompt_tokens": 481
},
"bfcl:multiple_166#1": {
"requests": 1,
"prompt_tokens": 907,
"completion_tokens": 26,
"allowance_tokens": 31861,
"length_stops": 0,
"errors": 0,
"seconds": 0.303,
"budget_s": 0.24534253617181667,
"last_prompt_tokens": 907
},
"bfcl:parallel_164#1": {
"requests": 1,
"prompt_tokens": 572,
"completion_tokens": 63,
"allowance_tokens": 32196,
"length_stops": 0,
"errors": 0,
"seconds": 0.496,
"budget_s": 0.4254871852965605,
"last_prompt_tokens": 572
},
"bfcl:parallel_multiple_172#1": {
"requests": 1,
"prompt_tokens": 1085,
"completion_tokens": 95,
"allowance_tokens": 31683,
"length_stops": 0,
"errors": 0,
"seconds": 0.779,
"budget_s": 0.6862082270415395,
"last_prompt_tokens": 1085
},
"bfcl:multi_turn_base_156#1": {
"requests": 5,
"prompt_tokens": 46586,
"completion_tokens": 230,
"allowance_tokens": 117254,
"length_stops": 0,
"errors": 0,
"seconds": 3.9510000000000005,
"budget_s": 2.6043071134463585,
"last_prompt_tokens": 9490
},
"bfcl:simple_python_342#1": {
"requests": 1,
"prompt_tokens": 480,
"completion_tokens": 30,
"allowance_tokens": 32288,
"length_stops": 0,
"errors": 0,
"seconds": 0.349,
"budget_s": 0.21470835791909598,
"last_prompt_tokens": 480
},
"bfcl:multiple_167#1": {
"requests": 1,
"prompt_tokens": 999,
"completion_tokens": 32,
"allowance_tokens": 31769,
"length_stops": 0,
"errors": 0,
"seconds": 0.358,
"budget_s": 0.2937325448865752,
"last_prompt_tokens": 999
},
"bfcl:parallel_166#1": {
"requests": 1,
"prompt_tokens": 598,
"completion_tokens": 89,
"allowance_tokens": 32170,
"length_stops": 0,
"errors": 0,
"seconds": 0.665,
"budget_s": 0.5859986281754159,
"last_prompt_tokens": 598
},
"bfcl:parallel_multiple_175#1": {
"requests": 1,
"prompt_tokens": 774,
"completion_tokens": 91,
"allowance_tokens": 31994,
"length_stops": 0,
"errors": 0,
"seconds": 0.703,
"budget_s": 0.621337195224026,
"last_prompt_tokens": 774
},
"bfcl:multi_turn_base_160#1": {
"requests": 3,
"prompt_tokens": 28869,
"completion_tokens": 1246,
"allowance_tokens": 69435,
"length_stops": 0,
"errors": 0,
"seconds": 10.357000000000001,
"budget_s": 8.99836133066536,
"last_prompt_tokens": 10292
},
"bfcl:simple_python_344#1": {
"requests": 1,
"prompt_tokens": 466,
"completion_tokens": 31,
"allowance_tokens": 32302,
"length_stops": 0,
"errors": 0,
"seconds": 0.315,
"budget_s": 0.21918878444213,
"last_prompt_tokens": 466
},
"bfcl:multiple_168#1": {
"requests": 1,
"prompt_tokens": 849,
"completion_tokens": 23,
"allowance_tokens": 31919,
"length_stops": 0,
"errors": 0,
"seconds": 0.274,
"budget_s": 0.22026278306838165,
"last_prompt_tokens": 849
},
"bfcl:parallel_169#1": {
"requests": 1,
"prompt_tokens": 634,
"completion_tokens": 79,
"allowance_tokens": 32134,
"length_stops": 0,
"errors": 0,
"seconds": 0.607,
"budget_s": 0.5306321282379522,
"last_prompt_tokens": 634
},
"bfcl:parallel_multiple_180#1": {
"requests": 1,
"prompt_tokens": 935,
"completion_tokens": 66,
"allowance_tokens": 31833,
"length_stops": 0,
"errors": 0,
"seconds": 0.569,
"budget_s": 0.4915392024098275,
"last_prompt_tokens": 935
},
"bfcl:multi_turn_base_166#1": {
"requests": 5,
"prompt_tokens": 45874,
"completion_tokens": 388,
"allowance_tokens": 117966,
"length_stops": 0,
"errors": 0,
"seconds": 4.9590000000000005,
"budget_s": 3.573201021010521,
"last_prompt_tokens": 9424
},
"bfcl:simple_python_347#1": {
"requests": 1,
"prompt_tokens": 438,
"completion_tokens": 25,
"allowance_tokens": 32330,
"length_stops": 0,
"errors": 0,
"seconds": 0.308,
"budget_s": 0.17968179979752366,
"last_prompt_tokens": 438
},
"bfcl:multiple_170#1": {
"requests": 1,
"prompt_tokens": 1049,
"completion_tokens": 31,
"allowance_tokens": 31719,
"length_stops": 0,
"errors": 0,
"seconds": 0.357,
"budget_s": 0.2946271788121029,
"last_prompt_tokens": 1049
},
"bfcl:parallel_170#1": {
"requests": 1,
"prompt_tokens": 598,
"completion_tokens": 93,
"allowance_tokens": 32170,
"length_stops": 0,
"errors": 0,
"seconds": 0.695,
"budget_s": 0.6106419442823777,
"last_prompt_tokens": 598
},
"bfcl:parallel_multiple_181#1": {
"requests": 1,
"prompt_tokens": 974,
"completion_tokens": 83,
"allowance_tokens": 31794,
"length_stops": 0,
"errors": 0,
"seconds": 0.686,
"budget_s": 0.5997423531119317,
"last_prompt_tokens": 974
},
"bfcl:multi_turn_base_171#1": {
"requests": 3,
"prompt_tokens": 27263,
"completion_tokens": 122,
"allowance_tokens": 71041,
"length_stops": 0,
"errors": 0,
"seconds": 2.677,
"budget_s": 1.901793555396297,
"last_prompt_tokens": 9223
},
"bfcl:simple_python_357#1": {
"requests": 1,
"prompt_tokens": 437,
"completion_tokens": 20,
"allowance_tokens": 32331,
"length_stops": 0,
"errors": 0,
"seconds": 0.24,
"budget_s": 0.1496773397452018,
"last_prompt_tokens": 437
},
"bfcl:multiple_172#1": {
"requests": 1,
"prompt_tokens": 1178,
"completion_tokens": 20,
"allowance_tokens": 31590,
"length_stops": 0,
"errors": 0,
"seconds": 0.303,
"budget_s": 0.244827329693602,
"last_prompt_tokens": 1178
},
"bfcl:parallel_171#1": {
"requests": 1,
"prompt_tokens": 581,
"completion_tokens": 78,
"allowance_tokens": 32187,
"length_stops": 0,
"errors": 0,
"seconds": 0.594,
"budget_s": 0.5181685792521548,
"last_prompt_tokens": 581
},
"bfcl:parallel_multiple_182#1": {
"requests": 1,
"prompt_tokens": 973,
"completion_tokens": 66,
"allowance_tokens": 31795,
"length_stops": 0,
"errors": 0,
"seconds": 0.573,
"budget_s": 0.4970022613325641,
"last_prompt_tokens": 973
},
"bfcl:multi_turn_base_175#1": {
"requests": 2,
"prompt_tokens": 21676,
"completion_tokens": 515,
"allowance_tokens": 43860,
"length_stops": 0,
"errors": 0,
"seconds": 5.386,
"budget_s": 4.594515716534833,
"last_prompt_tokens": 11040
},
"bfcl:simple_python_358#1": {
"requests": 1,
"prompt_tokens": 520,
"completion_tokens": 23,
"allowance_tokens": 32248,
"length_stops": 0,
"errors": 0,
"seconds": 0.255,
"budget_s": 0.17866494816028078,
"last_prompt_tokens": 520
},
"bfcl:multiple_173#1": {
"requests": 1,
"prompt_tokens": 841,
"completion_tokens": 19,
"allowance_tokens": 31927,
"length_stops": 0,
"errors": 0,
"seconds": 0.247,
"budget_s": 0.19582085453777578,
"last_prompt_tokens": 841
},
"bfcl:parallel_175#1": {
"requests": 1,
"prompt_tokens": 633,
"completion_tokens": 112,
"allowance_tokens": 32135,
"length_stops": 0,
"errors": 0,
"seconds": 0.821,
"budget_s": 0.730424164238907,
"last_prompt_tokens": 633
},
"bfcl:parallel_multiple_185#1": {
"requests": 1,
"prompt_tokens": 801,
"completion_tokens": 111,
"allowance_tokens": 31967,
"length_stops": 0,
"errors": 0,
"seconds": 0.846,
"budget_s": 0.7467635909096152,
"last_prompt_tokens": 801
},
"bfcl:multi_turn_base_178#1": {
"requests": 5,
"prompt_tokens": 47443,
"completion_tokens": 434,
"allowance_tokens": 116397,
"length_stops": 0,
"errors": 0,
"seconds": 5.363,
"budget_s": 3.8995827224567687,
"last_prompt_tokens": 9740
},
"bfcl:simple_python_363#1": {
"requests": 1,
"prompt_tokens": 512,
"completion_tokens": 24,
"allowance_tokens": 32256,
"length_stops": 0,
"errors": 0,
"seconds": 0.316,
"budget_s": 0.18392218658099452,
"last_prompt_tokens": 512
},
"bfcl:multiple_176#1": {
"requests": 1,
"prompt_tokens": 1381,
"completion_tokens": 30,
"allowance_tokens": 31387,
"length_stops": 0,
"errors": 0,
"seconds": 0.402,
"budget_s": 0.332066938769194,
"last_prompt_tokens": 1381
},
"bfcl:parallel_176#1": {
"requests": 1,
"prompt_tokens": 552,
"completion_tokens": 65,
"allowance_tokens": 32216,
"length_stops": 0,
"errors": 0,
"seconds": 0.509,
"budget_s": 0.4364736075523018,
"last_prompt_tokens": 552
},
"bfcl:parallel_multiple_187#1": {
"requests": 1,
"prompt_tokens": 967,
"completion_tokens": 92,
"allowance_tokens": 31801,
"length_stops": 0,
"errors": 0,
"seconds": 0.744,
"budget_s": 0.654042874861408,
"last_prompt_tokens": 967
},
"bfcl:multi_turn_base_180#1": {
"requests": 6,
"prompt_tokens": 56931,
"completion_tokens": 544,
"allowance_tokens": 139677,
"length_stops": 0,
"errors": 0,
"seconds": 6.345000000000001,
"budget_s": 4.587902369366815,
"last_prompt_tokens": 9811
},
"bfcl:simple_python_366#1": {
"requests": 1,
"prompt_tokens": 535,
"completion_tokens": 27,
"allowance_tokens": 32233,
"length_stops": 0,
"errors": 0,
"seconds": 0.357,
"budget_s": 0.20522223167913045,
"last_prompt_tokens": 535
},
"bfcl:multiple_184#1": {
"requests": 1,
"prompt_tokens": 837,
"completion_tokens": 25,
"allowance_tokens": 31931,
"length_stops": 0,
"errors": 0,
"seconds": 0.287,
"budget_s": 0.2320844621435491,
"last_prompt_tokens": 837
},
"bfcl:parallel_184#1": {
"requests": 1,
"prompt_tokens": 562,
"completion_tokens": 38,
"allowance_tokens": 32206,
"length_stops": 0,
"errors": 0,
"seconds": 0.331,
"budget_s": 0.2751709663301748,
"last_prompt_tokens": 562
},
"bfcl:parallel_multiple_188#1": {
"requests": 1,
"prompt_tokens": 1265,
"completion_tokens": 98,
"allowance_tokens": 31503,
"length_stops": 0,
"errors": 0,
"seconds": 0.833,
"budget_s": 0.7298928508209757,
"last_prompt_tokens": 1265
},
"bfcl:multi_turn_base_181#1": {
"requests": 5,
"prompt_tokens": 46645,
"completion_tokens": 270,
"allowance_tokens": 117195,
"length_stops": 0,
"errors": 0,
"seconds": 4.231,
"budget_s": 2.8623316090255457,
"last_prompt_tokens": 9541
},
"bfcl:simple_python_373#1": {
"requests": 1,
"prompt_tokens": 526,
"completion_tokens": 37,
"allowance_tokens": 32242,
"length_stops": 0,
"errors": 0,
"seconds": 0.403,
"budget_s": 0.2646219637258241,
"last_prompt_tokens": 526
},
"bfcl:multiple_185#1": {
"requests": 1,
"prompt_tokens": 727,
"completion_tokens": 20,
"allowance_tokens": 32041,
"length_stops": 0,
"errors": 0,
"seconds": 0.236,
"budget_s": 0.18801060510450107,
"last_prompt_tokens": 727
},
"bfcl:parallel_188#1": {
"requests": 1,
"prompt_tokens": 530,
"completion_tokens": 80,
"allowance_tokens": 32238,
"length_stops": 0,
"errors": 0,
"seconds": 0.594,
"budget_s": 0.5231717310449753,
"last_prompt_tokens": 530
},
"bfcl:parallel_multiple_189#1": {
"requests": 1,
"prompt_tokens": 1488,
"completion_tokens": 123,
"allowance_tokens": 31280,
"length_stops": 0,
"errors": 0,
"seconds": 1.033,
"budget_s": 0.9100216949548134,
"last_prompt_tokens": 1488
},
"bfcl:multi_turn_base_182#1": {
"requests": 2,
"prompt_tokens": 13395,
"completion_tokens": 154,
"allowance_tokens": 52141,
"length_stops": 0,
"errors": 0,
"seconds": 2.229,
"budget_s": 1.7866660182132772,
"last_prompt_tokens": 6773
},
"bfcl:simple_python_380#1": {
"requests": 1,
"prompt_tokens": 635,
"completion_tokens": 44,
"allowance_tokens": 32133,
"length_stops": 0,
"errors": 0,
"seconds": 0.404,
"budget_s": 0.3197851344391518,
"last_prompt_tokens": 635
},
"bfcl:multiple_188#1": {
"requests": 1,
"prompt_tokens": 1099,
"completion_tokens": 32,
"allowance_tokens": 31669,
"length_stops": 0,
"errors": 0,
"seconds": 0.369,
"budget_s": 0.30724855415819974,
"last_prompt_tokens": 1099
},
"bfcl:parallel_194#1": {
"requests": 1,
"prompt_tokens": 516,
"completion_tokens": 78,
"allowance_tokens": 32252,
"length_stops": 0,
"errors": 0,
"seconds": 0.585,
"budget_s": 0.5095330565571852,
"last_prompt_tokens": 516
},
"bfcl:parallel_multiple_192#1": {
"requests": 1,
"prompt_tokens": 989,
"completion_tokens": 70,
"allowance_tokens": 31779,
"length_stops": 0,
"errors": 0,
"seconds": 0.606,
"budget_s": 0.5231669312821625,
"last_prompt_tokens": 989
},
"bfcl:multi_turn_base_190#1": {
"requests": 5,
"prompt_tokens": 46901,
"completion_tokens": 289,
"allowance_tokens": 116939,
"length_stops": 0,
"errors": 0,
"seconds": 4.359999999999999,
"budget_s": 2.98582213912748,
"last_prompt_tokens": 9614
},
"bfcl:simple_python_382#1": {
"requests": 1,
"prompt_tokens": 564,
"completion_tokens": 43,
"allowance_tokens": 32204,
"length_stops": 0,
"errors": 0,
"seconds": 0.448,
"budget_s": 0.30482496878720733,
"last_prompt_tokens": 564
},
"bfcl:multiple_190#1": {
"requests": 1,
"prompt_tokens": 901,
"completion_tokens": 44,
"allowance_tokens": 31867,
"length_stops": 0,
"errors": 0,
"seconds": 0.421,
"budget_s": 0.3545717742770266,
"last_prompt_tokens": 901
},
"bfcl:parallel_196#1": {
"requests": 1,
"prompt_tokens": 542,
"completion_tokens": 69,
"allowance_tokens": 32226,
"length_stops": 0,
"errors": 0,
"seconds": 0.531,
"budget_s": 0.458912453486152,
"last_prompt_tokens": 542
},
"bfcl:parallel_multiple_193#1": {
"requests": 1,
"prompt_tokens": 1218,
"completion_tokens": 130,
"allowance_tokens": 31550,
"length_stops": 0,
"errors": 0,
"seconds": 1.035,
"budget_s": 0.9170473597551534,
"last_prompt_tokens": 1218
},
"bfcl:multi_turn_base_193#1": {
"requests": 4,
"prompt_tokens": 37614,
"completion_tokens": 368,
"allowance_tokens": 93458,
"length_stops": 0,
"errors": 0,
"seconds": 4.619999999999999,
"budget_s": 3.481848243815187,
"last_prompt_tokens": 9644
},
"bfcl:simple_python_386#1": {
"requests": 1,
"prompt_tokens": 611,
"completion_tokens": 49,
"allowance_tokens": 32157,
"length_stops": 0,
"errors": 0,
"seconds": 0.48,
"budget_s": 0.3473608774665529,
"last_prompt_tokens": 611
},
"bfcl:multiple_195#1": {
"requests": 1,
"prompt_tokens": 999,
"completion_tokens": 23,
"allowance_tokens": 31769,
"length_stops": 0,
"errors": 0,
"seconds": 0.297,
"budget_s": 0.24041036665671378,
"last_prompt_tokens": 999
},
"bfcl:parallel_197#1": {
"requests": 1,
"prompt_tokens": 555,
"completion_tokens": 59,
"allowance_tokens": 32213,
"length_stops": 0,
"errors": 0,
"seconds": 0.469,
"budget_s": 0.40054795077791566,
"last_prompt_tokens": 555
},
"bfcl:parallel_multiple_194#1": {
"requests": 1,
"prompt_tokens": 1247,
"completion_tokens": 99,
"allowance_tokens": 31521,
"length_stops": 0,
"errors": 0,
"seconds": 0.837,
"budget_s": 0.7332008678332915,
"last_prompt_tokens": 1247
},
"bfcl:multi_turn_base_195#1": {
"requests": 4,
"prompt_tokens": 43402,
"completion_tokens": 377,
"allowance_tokens": 87670,
"length_stops": 0,
"errors": 0,
"seconds": 5,
"budget_s": 3.721043333084083,
"last_prompt_tokens": 11039
},
"bfcl:simple_python_396#1": {
"requests": 1,
"prompt_tokens": 511,
"completion_tokens": 21,
"allowance_tokens": 32257,
"length_stops": 0,
"errors": 0,
"seconds": 0.278,
"budget_s": 0.1658325255516836,
"last_prompt_tokens": 511
},
"bfcl:multiple_197#1": {
"requests": 1,
"prompt_tokens": 1143,
"completion_tokens": 21,
"allowance_tokens": 31625,
"length_stops": 0,
"errors": 0,
"seconds": 0.305,
"budget_s": 0.24703413891352313,
"last_prompt_tokens": 1143
},
"bfcl:parallel_199#1": {
"requests": 1,
"prompt_tokens": 476,
"completion_tokens": 93,
"allowance_tokens": 32292,
"length_stops": 0,
"errors": 0,
"seconds": 0.682,
"budget_s": 0.5954799412978613,
"last_prompt_tokens": 476
},
"bfcl:parallel_multiple_197#1": {
"requests": 1,
"prompt_tokens": 920,
"completion_tokens": 92,
"allowance_tokens": 31848,
"length_stops": 0,
"errors": 0,
"seconds": 0.735,
"budget_s": 0.6479866829675919,
"last_prompt_tokens": 920
},
"bfcl:multi_turn_base_196#1": {
"requests": 5,
"prompt_tokens": 47113,
"completion_tokens": 410,
"allowance_tokens": 116727,
"length_stops": 0,
"errors": 0,
"seconds": 5.186999999999999,
"budget_s": 3.7444877867373476,
"last_prompt_tokens": 9705
},
"livecodebench:livecodebench/3692#1": {
"requests": 1,
"prompt_tokens": 409,
"completion_tokens": 730,
"allowance_tokens": 32359,
"length_stops": 0,
"errors": 0,
"seconds": 4.963,
"budget_s": 4.450078768935507,
"last_prompt_tokens": 409
},
"livecodebench:livecodebench/3723#1": {
"requests": 1,
"prompt_tokens": 369,
"completion_tokens": 467,
"allowance_tokens": 32399,
"length_stops": 0,
"errors": 0,
"seconds": 3.14,
"budget_s": 2.859596998665562,
"last_prompt_tokens": 369
},
"livecodebench:livecodebench/3763#1": {
"requests": 1,
"prompt_tokens": 545,
"completion_tokens": 233,
"allowance_tokens": 32223,
"length_stops": 0,
"errors": 0,
"seconds": 1.628,
"budget_s": 1.4745004326391626,
"last_prompt_tokens": 545
},
"livecodebench:livecodebench/3709#1": {
"requests": 1,
"prompt_tokens": 385,
"completion_tokens": 127,
"allowance_tokens": 32383,
"length_stops": 0,
"errors": 0,
"seconds": 0.898,
"budget_s": 0.8104662899482671,
"last_prompt_tokens": 385
},
"livecodebench:livecodebench/3733#1": {
"requests": 1,
"prompt_tokens": 764,
"completion_tokens": 403,
"allowance_tokens": 32004,
"length_stops": 0,
"errors": 0,
"seconds": 2.786,
"budget_s": 2.528415111480793,
"last_prompt_tokens": 764
},
"livecodebench:livecodebench/3771#1": {
"requests": 1,
"prompt_tokens": 416,
"completion_tokens": 397,
"allowance_tokens": 32352,
"length_stops": 0,
"errors": 0,
"seconds": 2.685,
"budget_s": 2.4415147136345703,
"last_prompt_tokens": 416
},
"livecodebench:livecodebench/3779#1": {
"requests": 1,
"prompt_tokens": 594,
"completion_tokens": 226,
"allowance_tokens": 32174,
"length_stops": 0,
"errors": 0,
"seconds": 1.587,
"budget_s": 1.4358470905573104,
"last_prompt_tokens": 594
},
"livecodebench:livecodebench/abc394_a#1": {
"requests": 1,
"prompt_tokens": 350,
"completion_tokens": 44,
"allowance_tokens": 32418,
"length_stops": 0,
"errors": 0,
"seconds": 0.351,
"budget_s": 0.30622377806489043,
"last_prompt_tokens": 350
},
"livecodebench:livecodebench/abc394_b#1": {
"requests": 1,
"prompt_tokens": 394,
"completion_tokens": 71,
"allowance_tokens": 32374,
"length_stops": 0,
"errors": 0,
"seconds": 0.533,
"budget_s": 0.4746937404785322,
"last_prompt_tokens": 394
},
"livecodebench:livecodebench/abc394_c#1": {
"requests": 1,
"prompt_tokens": 636,
"completion_tokens": 74,
"allowance_tokens": 32132,
"length_stops": 0,
"errors": 0,
"seconds": 0.593,
"budget_s": 0.524525365062769,
"last_prompt_tokens": 636
},
"livecodebench:livecodebench/abc394_d#1": {
"requests": 1,
"prompt_tokens": 628,
"completion_tokens": 247,
"allowance_tokens": 32140,
"length_stops": 0,
"errors": 0,
"seconds": 1.731,
"budget_s": 1.5675163589790548,
"last_prompt_tokens": 628
},
"livecodebench:livecodebench/abc394_e#1": {
"requests": 1,
"prompt_tokens": 865,
"completion_tokens": 494,
"allowance_tokens": 31903,
"length_stops": 0,
"errors": 0,
"seconds": 3.407,
"budget_s": 3.093701223970667,
"last_prompt_tokens": 865
},
"livecodebench:livecodebench/abc394_f#1": {
"requests": 1,
"prompt_tokens": 663,
"completion_tokens": 481,
"allowance_tokens": 32105,
"length_stops": 0,
"errors": 0,
"seconds": 3.284,
"budget_s": 2.984911843711799,
"last_prompt_tokens": 663
},
"livecodebench:livecodebench/abc394_g#1": {
"requests": 1,
"prompt_tokens": 1476,
"completion_tokens": 365,
"allowance_tokens": 31292,
"length_stops": 0,
"errors": 0,
"seconds": 2.651,
"budget_s": 2.401919188067377,
"last_prompt_tokens": 1476
},
"livecodebench:livecodebench/3764#1": {
"requests": 1,
"prompt_tokens": 476,
"completion_tokens": 294,
"allowance_tokens": 32292,
"length_stops": 0,
"errors": 0,
"seconds": 2.019,
"budget_s": 1.82975247711355,
"last_prompt_tokens": 476
},
"livecodebench:livecodebench/3768#1": {
"requests": 1,
"prompt_tokens": 544,
"completion_tokens": 622,
"allowance_tokens": 32224,
"length_stops": 0,
"errors": 0,
"seconds": 4.194,
"budget_s": 3.817634042451788,
"last_prompt_tokens": 544
},
"livecodebench:livecodebench/3781#1": {
"requests": 1,
"prompt_tokens": 612,
"completion_tokens": 699,
"allowance_tokens": 32156,
"length_stops": 0,
"errors": 0,
"seconds": 4.71,
"budget_s": 4.292355504262014,
"last_prompt_tokens": 612
},
"livecodebench:livecodebench/arc193_a#1": {
"requests": 1,
"prompt_tokens": 1122,
"completion_tokens": 562,
"allowance_tokens": 31646,
"length_stops": 0,
"errors": 0,
"seconds": 3.917,
"budget_s": 3.5438317874244083,
"last_prompt_tokens": 1122
},
"livecodebench:livecodebench/arc193_b#1": {
"requests": 1,
"prompt_tokens": 884,
"completion_tokens": 31884,
"allowance_tokens": 31884,
"length_stops": 1,
"errors": 0,
"seconds": 222.629,
"budget_s": 192.90169021988288,
"last_prompt_tokens": 884
},
"livecodebench:livecodebench/arc193_d#1": {
"requests": 1,
"prompt_tokens": 1025,
"completion_tokens": 512,
"allowance_tokens": 31743,
"length_stops": 0,
"errors": 0,
"seconds": 3.552,
"budget_s": 3.2262987479888374,
"last_prompt_tokens": 1025
},
"livecodebench:livecodebench/abc395_a#1": {
"requests": 1,
"prompt_tokens": 475,
"completion_tokens": 104,
"allowance_tokens": 32293,
"length_stops": 0,
"errors": 0,
"seconds": 0.767,
"budget_s": 0.6839348018987903,
"last_prompt_tokens": 475
},
"livecodebench:livecodebench/abc395_b#1": {
"requests": 1,
"prompt_tokens": 796,
"completion_tokens": 162,
"allowance_tokens": 31972,
"length_stops": 0,
"errors": 0,
"seconds": 1.194,
"budget_s": 1.0769373183425839,
"last_prompt_tokens": 796
},
"livecodebench:livecodebench/abc395_c#1": {
"requests": 1,
"prompt_tokens": 468,
"completion_tokens": 103,
"allowance_tokens": 32300,
"length_stops": 0,
"errors": 0,
"seconds": 0.753,
"budget_s": 0.6771099813898718,
"last_prompt_tokens": 468
},
"livecodebench:livecodebench/abc395_e#1": {
"requests": 1,
"prompt_tokens": 1033,
"completion_tokens": 326,
"allowance_tokens": 31735,
"length_stops": 0,
"errors": 0,
"seconds": 2.301,
"budget_s": 2.101796852351776,
"last_prompt_tokens": 1033
},
"livecodebench:livecodebench/abc395_f#1": {
"requests": 1,
"prompt_tokens": 954,
"completion_tokens": 507,
"allowance_tokens": 31814,
"length_stops": 0,
"errors": 0,
"seconds": 3.488,
"budget_s": 3.1855839075085792,
"last_prompt_tokens": 954
},
"livecodebench:livecodebench/3776#1": {
"requests": 1,
"prompt_tokens": 561,
"completion_tokens": 655,
"allowance_tokens": 32207,
"length_stops": 0,
"errors": 0,
"seconds": 4.38,
"budget_s": 4.019079259035838,
"last_prompt_tokens": 561
},
"livecodebench:livecodebench/3778#1": {
"requests": 1,
"prompt_tokens": 425,
"completion_tokens": 91,
"allowance_tokens": 32343,
"length_stops": 0,
"errors": 0,
"seconds": 0.664,
"budget_s": 0.5968338726438625,
"last_prompt_tokens": 425
},
"livecodebench:livecodebench/3783#1": {
"requests": 1,
"prompt_tokens": 569,
"completion_tokens": 231,
"allowance_tokens": 32199,
"length_stops": 0,
"errors": 0,
"seconds": 1.603,
"budget_s": 1.4626928917119078,
"last_prompt_tokens": 569
},
"livecodebench:livecodebench/3785#1": {
"requests": 1,
"prompt_tokens": 553,
"completion_tokens": 217,
"allowance_tokens": 32215,
"length_stops": 0,
"errors": 0,
"seconds": 1.509,
"budget_s": 1.3760458430044653,
"last_prompt_tokens": 553
},
"livecodebench:livecodebench/3705#1": {
"requests": 1,
"prompt_tokens": 710,
"completion_tokens": 151,
"allowance_tokens": 32058,
"length_stops": 0,
"errors": 0,
"seconds": 1.102,
"budget_s": 0.9976480994573992,
"last_prompt_tokens": 710
},
"livecodebench:livecodebench/3722#1": {
"requests": 1,
"prompt_tokens": 410,
"completion_tokens": 248,
"allowance_tokens": 32358,
"length_stops": 0,
"errors": 0,
"seconds": 1.682,
"budget_s": 1.5417875364730218,
"last_prompt_tokens": 410
},
"livecodebench:livecodebench/3770#1": {
"requests": 1,
"prompt_tokens": 561,
"completion_tokens": 246,
"allowance_tokens": 32207,
"length_stops": 0,
"errors": 0,
"seconds": 1.699,
"budget_s": 1.5507935940168927,
"last_prompt_tokens": 561
},
"livecodebench:livecodebench/3786#1": {
"requests": 1,
"prompt_tokens": 462,
"completion_tokens": 276,
"allowance_tokens": 32306,
"length_stops": 0,
"errors": 0,
"seconds": 1.875,
"budget_s": 1.71674413630826,
"last_prompt_tokens": 462
},
"livecodebench:livecodebench/abc396_a#1": {
"requests": 1,
"prompt_tokens": 554,
"completion_tokens": 93,
"allowance_tokens": 32214,
"length_stops": 0,
"errors": 0,
"seconds": 0.691,
"budget_s": 0.6259862228681461,
"last_prompt_tokens": 554
},
"livecodebench:livecodebench/abc396_b#1": {
"requests": 1,
"prompt_tokens": 764,
"completion_tokens": 107,
"allowance_tokens": 32004,
"length_stops": 0,
"errors": 0,
"seconds": 0.825,
"budget_s": 0.7383239117586194,
"last_prompt_tokens": 764
},
"livecodebench:livecodebench/abc396_c#1": {
"requests": 1,
"prompt_tokens": 564,
"completion_tokens": 221,
"allowance_tokens": 32204,
"length_stops": 0,
"errors": 0,
"seconds": 1.542,
"budget_s": 1.399519404826546,
"last_prompt_tokens": 564
},
"livecodebench:livecodebench/abc396_d#1": {
"requests": 1,
"prompt_tokens": 1002,
"completion_tokens": 328,
"allowance_tokens": 31766,
"length_stops": 0,
"errors": 0,
"seconds": 2.305,
"budget_s": 2.107322661633815,
"last_prompt_tokens": 1002
},
"livecodebench:livecodebench/abc396_e#1": {
"requests": 1,
"prompt_tokens": 887,
"completion_tokens": 517,
"allowance_tokens": 31881,
"length_stops": 0,
"errors": 0,
"seconds": 3.544,
"budget_s": 3.233789045241944,
"last_prompt_tokens": 887
},
"livecodebench:livecodebench/abc396_f#1": {
"requests": 1,
"prompt_tokens": 603,
"completion_tokens": 343,
"allowance_tokens": 32165,
"length_stops": 0,
"errors": 0,
"seconds": 2.343,
"budget_s": 2.140997552473165,
"last_prompt_tokens": 603
},
"livecodebench:livecodebench/abc396_g#1": {
"requests": 1,
"prompt_tokens": 726,
"completion_tokens": 251,
"allowance_tokens": 32042,
"length_stops": 0,
"errors": 0,
"seconds": 1.763,
"budget_s": 1.6029728145156115,
"last_prompt_tokens": 726
},
"livecodebench:livecodebench/3759#1": {
"requests": 1,
"prompt_tokens": 667,
"completion_tokens": 215,
"allowance_tokens": 32101,
"length_stops": 0,
"errors": 0,
"seconds": 1.517,
"budget_s": 1.377458957550284,
"last_prompt_tokens": 667
},
"livecodebench:livecodebench/3789#1": {
"requests": 1,
"prompt_tokens": 564,
"completion_tokens": 237,
"allowance_tokens": 32204,
"length_stops": 0,
"errors": 0,
"seconds": 1.64,
"budget_s": 1.4960306570817492,
"last_prompt_tokens": 564
},
"livecodebench:livecodebench/3791#1": {
"requests": 1,
"prompt_tokens": 534,
"completion_tokens": 166,
"allowance_tokens": 32234,
"length_stops": 0,
"errors": 0,
"seconds": 1.165,
"budget_s": 1.063713875622583,
"last_prompt_tokens": 534
},
"livecodebench:livecodebench/arc194_a#1": {
"requests": 1,
"prompt_tokens": 694,
"completion_tokens": 235,
"allowance_tokens": 32074,
"length_stops": 0,
"errors": 0,
"seconds": 1.647,
"budget_s": 1.5020494568609841,
"last_prompt_tokens": 694
},
"livecodebench:livecodebench/arc194_b#1": {
"requests": 1,
"prompt_tokens": 538,
"completion_tokens": 187,
"allowance_tokens": 32230,
"length_stops": 0,
"errors": 0,
"seconds": 1.303,
"budget_s": 1.1910268744827794,
"last_prompt_tokens": 538
},
"livecodebench:livecodebench/arc194_c#1": {
"requests": 1,
"prompt_tokens": 938,
"completion_tokens": 503,
"allowance_tokens": 31830,
"length_stops": 0,
"errors": 0,
"seconds": 3.462,
"budget_s": 3.157065681753192,
"last_prompt_tokens": 938
},
"livecodebench:livecodebench/arc194_d#1": {
"requests": 1,
"prompt_tokens": 540,
"completion_tokens": 201,
"allowance_tokens": 32228,
"length_stops": 0,
"errors": 0,
"seconds": 1.398,
"budget_s": 1.2758579029018196,
"last_prompt_tokens": 540
},
"livecodebench:livecodebench/arc194_e#1": {
"requests": 1,
"prompt_tokens": 812,
"completion_tokens": 208,
"allowance_tokens": 31956,
"length_stops": 0,
"errors": 0,
"seconds": 1.494,
"budget_s": 1.3554335867920186,
"last_prompt_tokens": 812
},
"livecodebench:livecodebench/abc397_a#1": {
"requests": 1,
"prompt_tokens": 470,
"completion_tokens": 53,
"allowance_tokens": 32298,
"length_stops": 0,
"errors": 0,
"seconds": 0.427,
"budget_s": 0.3743370425873771,
"last_prompt_tokens": 470
},
"livecodebench:livecodebench/abc397_b#1": {
"requests": 1,
"prompt_tokens": 457,
"completion_tokens": 348,
"allowance_tokens": 32311,
"length_stops": 0,
"errors": 0,
"seconds": 2.351,
"budget_s": 2.150755226739786,
"last_prompt_tokens": 457
},
"livecodebench:livecodebench/abc397_c#1": {
"requests": 1,
"prompt_tokens": 596,
"completion_tokens": 146,
"allowance_tokens": 32172,
"length_stops": 0,
"errors": 0,
"seconds": 1.055,
"budget_s": 0.9520481625697141,
"last_prompt_tokens": 596
},
"livecodebench:livecodebench/abc397_d#1": {
"requests": 1,
"prompt_tokens": 435,
"completion_tokens": 139,
"allowance_tokens": 32333,
"length_stops": 0,
"errors": 0,
"seconds": 0.983,
"budget_s": 0.8882895422787163,
"last_prompt_tokens": 435
},
"livecodebench:livecodebench/abc397_e#1": {
"requests": 1,
"prompt_tokens": 570,
"completion_tokens": 187,
"allowance_tokens": 32198,
"length_stops": 0,
"errors": 0,
"seconds": 1.319,
"budget_s": 1.1960122959369504,
"last_prompt_tokens": 570
},
"livecodebench:livecodebench/abc397_f#1": {
"requests": 1,
"prompt_tokens": 575,
"completion_tokens": 201,
"allowance_tokens": 32193,
"length_stops": 0,
"errors": 0,
"seconds": 1.415,
"budget_s": 1.2811462361974917,
"last_prompt_tokens": 575
},
"livecodebench:livecodebench/abc397_g#1": {
"requests": 1,
"prompt_tokens": 580,
"completion_tokens": 421,
"allowance_tokens": 32188,
"length_stops": 0,
"errors": 0,
"seconds": 2.844,
"budget_s": 2.609020394991202,
"last_prompt_tokens": 580
},
"livecodebench:livecodebench/3784#1": {
"requests": 1,
"prompt_tokens": 624,
"completion_tokens": 260,
"allowance_tokens": 32144,
"length_stops": 0,
"errors": 0,
"seconds": 1.797,
"budget_s": 1.6438851524171447,
"last_prompt_tokens": 624
},
"livecodebench:livecodebench/3799#1": {
"requests": 1,
"prompt_tokens": 459,
"completion_tokens": 495,
"allowance_tokens": 32309,
"length_stops": 0,
"errors": 0,
"seconds": 3.307,
"budget_s": 3.0372851330791604,
"last_prompt_tokens": 459
},
"livecodebench:livecodebench/3750#1": {
"requests": 1,
"prompt_tokens": 532,
"completion_tokens": 140,
"allowance_tokens": 32236,
"length_stops": 0,
"errors": 0,
"seconds": 0.997,
"budget_s": 0.9075426229708761,
"last_prompt_tokens": 532
},
"livecodebench:livecodebench/3788#1": {
"requests": 1,
"prompt_tokens": 423,
"completion_tokens": 521,
"allowance_tokens": 32345,
"length_stops": 0,
"errors": 0,
"seconds": 3.48,
"budget_s": 3.1888020679197364,
"last_prompt_tokens": 423
},
"livecodebench:livecodebench/3795#1": {
"requests": 1,
"prompt_tokens": 932,
"completion_tokens": 213,
"allowance_tokens": 31836,
"length_stops": 0,
"errors": 0,
"seconds": 1.541,
"budget_s": 1.402871489718604,
"last_prompt_tokens": 932
},
"livecodebench:livecodebench/3801#1": {
"requests": 1,
"prompt_tokens": 290,
"completion_tokens": 318,
"allowance_tokens": 32478,
"length_stops": 0,
"errors": 0,
"seconds": 2.128,
"budget_s": 1.9468869273522618,
"last_prompt_tokens": 290
},
"livecodebench:livecodebench/abc398_a#1": {
"requests": 1,
"prompt_tokens": 267,
"completion_tokens": 119,
"allowance_tokens": 32501,
"length_stops": 0,
"errors": 0,
"seconds": 0.819,
"budget_s": 0.7461014347037044,
"last_prompt_tokens": 267
},
"livecodebench:livecodebench/abc398_b#1": {
"requests": 1,
"prompt_tokens": 470,
"completion_tokens": 273,
"allowance_tokens": 32298,
"length_stops": 0,
"errors": 0,
"seconds": 1.853,
"budget_s": 1.7014360739471721,
"last_prompt_tokens": 470
},
"livecodebench:livecodebench/abc398_c#1": {
"requests": 1,
"prompt_tokens": 499,
"completion_tokens": 179,
"allowance_tokens": 32269,
"length_stops": 0,
"errors": 0,
"seconds": 1.252,
"budget_s": 1.1388955877854525,
"last_prompt_tokens": 499
},
"livecodebench:livecodebench/abc398_d#1": {
"requests": 1,
"prompt_tokens": 898,
"completion_tokens": 275,
"allowance_tokens": 31870,
"length_stops": 0,
"errors": 0,
"seconds": 1.948,
"budget_s": 1.773652343944773,
"last_prompt_tokens": 898
},
"livecodebench:livecodebench/abc398_f#1": {
"requests": 1,
"prompt_tokens": 302,
"completion_tokens": 260,
"allowance_tokens": 32466,
"length_stops": 0,
"errors": 0,
"seconds": 1.751,
"budget_s": 1.599941814737448,
"last_prompt_tokens": 302
},
"livecodebench:livecodebench/abc398_g#1": {
"requests": 1,
"prompt_tokens": 693,
"completion_tokens": 332,
"allowance_tokens": 32075,
"length_stops": 0,
"errors": 0,
"seconds": 2.291,
"budget_s": 2.0892411858152307,
"last_prompt_tokens": 693
},
"livecodebench:livecodebench/3744#1": {
"requests": 1,
"prompt_tokens": 675,
"completion_tokens": 212,
"allowance_tokens": 32093,
"length_stops": 0,
"errors": 0,
"seconds": 1.502,
"budget_s": 1.3622278221656234,
"last_prompt_tokens": 675
},
"livecodebench:livecodebench/3794#1": {
"requests": 1,
"prompt_tokens": 695,
"completion_tokens": 177,
"allowance_tokens": 32073,
"length_stops": 0,
"errors": 0,
"seconds": 1.274,
"budget_s": 1.1536400336719401,
"last_prompt_tokens": 695
},
"livecodebench:livecodebench/3809#1": {
"requests": 1,
"prompt_tokens": 474,
"completion_tokens": 278,
"allowance_tokens": 32294,
"length_stops": 0,
"errors": 0,
"seconds": 1.893,
"budget_s": 1.7325164682933356,
"last_prompt_tokens": 474
},
"livecodebench:livecodebench/3817#1": {
"requests": 1,
"prompt_tokens": 369,
"completion_tokens": 320,
"allowance_tokens": 32399,
"length_stops": 0,
"errors": 0,
"seconds": 2.158,
"budget_s": 1.9709044048567126,
"last_prompt_tokens": 369
},
"livecodebench:livecodebench/arc195_a#1": {
"requests": 1,
"prompt_tokens": 507,
"completion_tokens": 358,
"allowance_tokens": 32261,
"length_stops": 0,
"errors": 0,
"seconds": 2.43,
"budget_s": 2.219740258153163,
"last_prompt_tokens": 507
},
"livecodebench:livecodebench/arc195_b#1": {
"requests": 1,
"prompt_tokens": 691,
"completion_tokens": 243,
"allowance_tokens": 32077,
"length_stops": 0,
"errors": 0,
"seconds": 1.704,
"budget_s": 1.5519744272250522,
"last_prompt_tokens": 691
},
"livecodebench:livecodebench/arc195_c#1": {
"requests": 1,
"prompt_tokens": 961,
"completion_tokens": 199,
"allowance_tokens": 31807,
"length_stops": 0,
"errors": 0,
"seconds": 1.46,
"budget_s": 1.3231640674914182,
"last_prompt_tokens": 961
},
"livecodebench:livecodebench/arc195_d#1": {
"requests": 1,
"prompt_tokens": 670,
"completion_tokens": 385,
"allowance_tokens": 32098,
"length_stops": 0,
"errors": 0,
"seconds": 2.639,
"budget_s": 2.406295324601673,
"last_prompt_tokens": 670
},
"livecodebench:livecodebench/arc195_e#1": {
"requests": 1,
"prompt_tokens": 949,
"completion_tokens": 31819,
"allowance_tokens": 31819,
"length_stops": 1,
"errors": 0,
"seconds": 222.239,
"budget_s": 192.6126959941963,
"last_prompt_tokens": 949
},
"livecodebench:livecodebench/abc399_a#1": {
"requests": 1,
"prompt_tokens": 405,
"completion_tokens": 71,
"allowance_tokens": 32363,
"length_stops": 0,
"errors": 0,
"seconds": 0.54,
"budget_s": 0.4761299740617035,
"last_prompt_tokens": 405
},
"livecodebench:livecodebench/abc399_b#1": {
"requests": 1,
"prompt_tokens": 790,
"completion_tokens": 249,
"allowance_tokens": 31978,
"length_stops": 0,
"errors": 0,
"seconds": 1.771,
"budget_s": 1.6020678696085033,
"last_prompt_tokens": 790
},
"livecodebench:livecodebench/abc399_c#1": {
"requests": 1,
"prompt_tokens": 505,
"completion_tokens": 306,
"allowance_tokens": 32263,
"length_stops": 0,
"errors": 0,
"seconds": 2.101,
"budget_s": 1.9061866863782402,
"last_prompt_tokens": 505
},
"livecodebench:livecodebench/abc399_d#1": {
"requests": 1,
"prompt_tokens": 813,
"completion_tokens": 406,
"allowance_tokens": 31955,
"length_stops": 0,
"errors": 0,
"seconds": 2.808,
"budget_s": 2.554203102367282,
"last_prompt_tokens": 813
},
"livecodebench:livecodebench/abc399_e#1": {
"requests": 1,
"prompt_tokens": 556,
"completion_tokens": 324,
"allowance_tokens": 32212,
"length_stops": 0,
"errors": 0,
"seconds": 2.219,
"budget_s": 2.02209199277351,
"last_prompt_tokens": 556
},
"livecodebench:livecodebench/abc399_f#1": {
"requests": 1,
"prompt_tokens": 517,
"completion_tokens": 307,
"allowance_tokens": 32251,
"length_stops": 0,
"errors": 0,
"seconds": 2.095,
"budget_s": 1.9120030672396269,
"last_prompt_tokens": 517
},
"livecodebench:livecodebench/3765#1": {
"requests": 1,
"prompt_tokens": 736,
"completion_tokens": 253,
"allowance_tokens": 32032,
"length_stops": 0,
"errors": 0,
"seconds": 1.772,
"budget_s": 1.616950517081229,
"last_prompt_tokens": 736
},
"livecodebench:livecodebench/3805#1": {
"requests": 1,
"prompt_tokens": 706,
"completion_tokens": 146,
"allowance_tokens": 32062,
"length_stops": 0,
"errors": 0,
"seconds": 1.073,
"budget_s": 0.9666640001112765,
"last_prompt_tokens": 706
},
"livecodebench:livecodebench/3811#1": {
"requests": 1,
"prompt_tokens": 424,
"completion_tokens": 460,
"allowance_tokens": 32344,
"length_stops": 0,
"errors": 0,
"seconds": 3.079,
"budget_s": 2.8208866996335775,
"last_prompt_tokens": 424
},
"livecodebench:livecodebench/3717#1": {
"requests": 1,
"prompt_tokens": 614,
"completion_tokens": 281,
"allowance_tokens": 32154,
"length_stops": 0,
"errors": 0,
"seconds": 1.939,
"budget_s": 1.7690844442029086,
"last_prompt_tokens": 614
},
"livecodebench:livecodebench/3793#1": {
"requests": 1,
"prompt_tokens": 410,
"completion_tokens": 513,
"allowance_tokens": 32358,
"length_stops": 0,
"errors": 0,
"seconds": 3.426,
"budget_s": 3.1382555447521048,
"last_prompt_tokens": 410
},
"livecodebench:livecodebench/3832#1": {
"requests": 1,
"prompt_tokens": 572,
"completion_tokens": 114,
"allowance_tokens": 32196,
"length_stops": 0,
"errors": 0,
"seconds": 0.837,
"budget_s": 0.7559246893079264,
"last_prompt_tokens": 572
},
"livecodebench:livecodebench/abc400_a#1": {
"requests": 1,
"prompt_tokens": 323,
"completion_tokens": 117,
"allowance_tokens": 32445,
"length_stops": 0,
"errors": 0,
"seconds": 0.818,
"budget_s": 0.7409847788710421,
"last_prompt_tokens": 323
},
"livecodebench:livecodebench/abc400_b#1": {
"requests": 1,
"prompt_tokens": 430,
"completion_tokens": 84,
"allowance_tokens": 32338,
"length_stops": 0,
"errors": 0,
"seconds": 0.621,
"budget_s": 0.5564564872139489,
"last_prompt_tokens": 430
},
"livecodebench:livecodebench/abc400_c#1": {
"requests": 1,
"prompt_tokens": 406,
"completion_tokens": 123,
"allowance_tokens": 32362,
"length_stops": 0,
"errors": 0,
"seconds": 0.87,
"budget_s": 0.7883993632261951,
"last_prompt_tokens": 406
},
"livecodebench:livecodebench/abc400_d#1": {
"requests": 1,
"prompt_tokens": 1405,
"completion_tokens": 330,
"allowance_tokens": 31363,
"length_stops": 0,
"errors": 0,
"seconds": 2.398,
"budget_s": 2.1780486245043575,
"last_prompt_tokens": 1405
},
"livecodebench:livecodebench/abc400_e#1": {
"requests": 1,
"prompt_tokens": 579,
"completion_tokens": 253,
"allowance_tokens": 32189,
"length_stops": 0,
"errors": 0,
"seconds": 1.761,
"budget_s": 1.595771769537586,
"last_prompt_tokens": 579
},
"livecodebench:livecodebench/abc400_g#1": {
"requests": 1,
"prompt_tokens": 1143,
"completion_tokens": 391,
"allowance_tokens": 31625,
"length_stops": 0,
"errors": 0,
"seconds": 2.761,
"budget_s": 2.510256467667305,
"last_prompt_tokens": 1143
},
"livecodebench:livecodebench/3773#1": {
"requests": 1,
"prompt_tokens": 364,
"completion_tokens": 413,
"allowance_tokens": 32404,
"length_stops": 0,
"errors": 0,
"seconds": 2.776,
"budget_s": 2.529549299487216,
"last_prompt_tokens": 364
},
"livecodebench:livecodebench/3777#1": {
"requests": 1,
"prompt_tokens": 657,
"completion_tokens": 220,
"allowance_tokens": 32111,
"length_stops": 0,
"errors": 0,
"seconds": 1.554,
"budget_s": 1.407541195866678,
"last_prompt_tokens": 657
},
"livecodebench:livecodebench/arc196_a#1": {
"requests": 1,
"prompt_tokens": 475,
"completion_tokens": 320,
"allowance_tokens": 32293,
"length_stops": 0,
"errors": 0,
"seconds": 2.178,
"budget_s": 1.9853177382857086,
"last_prompt_tokens": 475
},
"livecodebench:livecodebench/arc196_b#1": {
"requests": 1,
"prompt_tokens": 1065,
"completion_tokens": 472,
"allowance_tokens": 31703,
"length_stops": 0,
"errors": 0,
"seconds": 3.282,
"budget_s": 2.9896275233485436,
"last_prompt_tokens": 1065
},
"livecodebench:livecodebench/arc196_c#1": {
"requests": 1,
"prompt_tokens": 740,
"completion_tokens": 214,
"allowance_tokens": 32028,
"length_stops": 0,
"errors": 0,
"seconds": 1.528,
"budget_s": 1.3828356956742376,
"last_prompt_tokens": 740
},
"livecodebench:livecodebench/arc196_d#1": {
"requests": 1,
"prompt_tokens": 1061,
"completion_tokens": 237,
"allowance_tokens": 31707,
"length_stops": 0,
"errors": 0,
"seconds": 1.726,
"budget_s": 1.566136848676717,
"last_prompt_tokens": 1061
},
"gpqa_diamond:gpqa_diamond/rec06pnAkLOr2t2mp#1": {
"requests": 1,
"prompt_tokens": 150,
"completion_tokens": 469,
"allowance_tokens": 32618,
"length_stops": 0,
"errors": 0,
"seconds": 3.07,
"budget_s": 2.8391927060060995,
"last_prompt_tokens": 150
},
"gpqa_diamond:gpqa_diamond/rec0Arme2jcXQZnAW#1": {
"requests": 1,
"prompt_tokens": 155,
"completion_tokens": 415,
"allowance_tokens": 32613,
"length_stops": 0,
"errors": 0,
"seconds": 2.677,
"budget_s": 2.515094829989477,
"last_prompt_tokens": 155
},
"gpqa_diamond:gpqa_diamond/rec0VuKUjt1SZ7NYv#1": {
"requests": 1,
"prompt_tokens": 171,
"completion_tokens": 1521,
"allowance_tokens": 32597,
"length_stops": 0,
"errors": 0,
"seconds": 9.817,
"budget_s": 9.168870108297865,
"last_prompt_tokens": 171
},
"gpqa_diamond:gpqa_diamond/rec0wZvZgiz320KRs#1": {
"requests": 1,
"prompt_tokens": 180,
"completion_tokens": 5563,
"allowance_tokens": 32588,
"length_stops": 0,
"errors": 0,
"seconds": 36.254,
"budget_s": 33.48071144691561,
"last_prompt_tokens": 180
},
"gpqa_diamond:gpqa_diamond/rec0yTRmO1o1xCA6H#1": {
"requests": 1,
"prompt_tokens": 161,
"completion_tokens": 479,
"allowance_tokens": 32607,
"length_stops": 0,
"errors": 0,
"seconds": 3.116,
"budget_s": 2.900836331599141,
"last_prompt_tokens": 161
},
"gpqa_diamond:gpqa_diamond/rec1oj2DveQWl9Rpw#1": {
"requests": 1,
"prompt_tokens": 197,
"completion_tokens": 1459,
"allowance_tokens": 32571,
"length_stops": 0,
"errors": 0,
"seconds": 9.498,
"budget_s": 8.801047351823135,
"last_prompt_tokens": 197
},
"gpqa_diamond:gpqa_diamond/rec1zl5LvaatzGhFt#1": {
"requests": 1,
"prompt_tokens": 375,
"completion_tokens": 1081,
"allowance_tokens": 32393,
"length_stops": 0,
"errors": 0,
"seconds": 7.092,
"budget_s": 6.558785935576772,
"last_prompt_tokens": 375
},
"gpqa_diamond:gpqa_diamond/rec260hNUCEj109Dj#1": {
"requests": 1,
"prompt_tokens": 227,
"completion_tokens": 790,
"allowance_tokens": 32541,
"length_stops": 0,
"errors": 0,
"seconds": 5.165,
"budget_s": 4.78181876482259,
"last_prompt_tokens": 227
},
"gpqa_diamond:gpqa_diamond/rec2UlKqC6RFHdcro#1": {
"requests": 1,
"prompt_tokens": 187,
"completion_tokens": 683,
"allowance_tokens": 32581,
"length_stops": 0,
"errors": 0,
"seconds": 4.46,
"budget_s": 4.131711174922585,
"last_prompt_tokens": 187
},
"gpqa_diamond:gpqa_diamond/rec2ZTsqazBZ3LTWZ#1": {
"requests": 1,
"prompt_tokens": 474,
"completion_tokens": 638,
"allowance_tokens": 32294,
"length_stops": 0,
"errors": 0,
"seconds": 4.234,
"budget_s": 3.905969932054819,
"last_prompt_tokens": 474
},
"gpqa_diamond:gpqa_diamond/rec2fsnzUuvNtUYK8#1": {
"requests": 1,
"prompt_tokens": 259,
"completion_tokens": 319,
"allowance_tokens": 32509,
"length_stops": 0,
"errors": 0,
"seconds": 2.122,
"budget_s": 1.9525814085659545,
"last_prompt_tokens": 259
},
"gpqa_diamond:gpqa_diamond/rec32C1ZEapBnCC0E#1": {
"requests": 1,
"prompt_tokens": 241,
"completion_tokens": 1067,
"allowance_tokens": 32527,
"length_stops": 0,
"errors": 0,
"seconds": 6.973,
"budget_s": 6.450858168178848,
"last_prompt_tokens": 241
},
"gpqa_diamond:gpqa_diamond/rec47oNPXudlTIk4y#1": {
"requests": 1,
"prompt_tokens": 281,
"completion_tokens": 255,
"allowance_tokens": 32487,
"length_stops": 0,
"errors": 0,
"seconds": 1.708,
"budget_s": 1.5704883465038273,
"last_prompt_tokens": 281
},
"gpqa_diamond:gpqa_diamond/rec4JrWy7Il8ho5vk#1": {
"requests": 1,
"prompt_tokens": 160,
"completion_tokens": 324,
"allowance_tokens": 32608,
"length_stops": 0,
"errors": 0,
"seconds": 2.123,
"budget_s": 1.968601841767608,
"last_prompt_tokens": 160
},
"gpqa_diamond:gpqa_diamond/rec4L69T0Y1AS4AFS#1": {
"requests": 1,
"prompt_tokens": 370,
"completion_tokens": 374,
"allowance_tokens": 32398,
"length_stops": 0,
"errors": 0,
"seconds": 2.49,
"budget_s": 2.2996015300782946,
"last_prompt_tokens": 370
},
"gpqa_diamond:gpqa_diamond/rec4UqStf9WUVif1f#1": {
"requests": 1,
"prompt_tokens": 167,
"completion_tokens": 1730,
"allowance_tokens": 32601,
"length_stops": 0,
"errors": 0,
"seconds": 11.318,
"budget_s": 10.424961671031177,
"last_prompt_tokens": 167
},
"gpqa_diamond:gpqa_diamond/rec4r4KWNLrT7g2kR#1": {
"requests": 1,
"prompt_tokens": 314,
"completion_tokens": 613,
"allowance_tokens": 32454,
"length_stops": 0,
"errors": 0,
"seconds": 4.051,
"budget_s": 3.730410809167889,
"last_prompt_tokens": 314
},
"gpqa_diamond:gpqa_diamond/rec527dNeEtWJrYNl#1": {
"requests": 1,
"prompt_tokens": 153,
"completion_tokens": 283,
"allowance_tokens": 32615,
"length_stops": 0,
"errors": 0,
"seconds": 1.864,
"budget_s": 1.721068618762089,
"last_prompt_tokens": 153
},
"gpqa_diamond:gpqa_diamond/rec5eA17Qr1ucNLUf#1": {
"requests": 1,
"prompt_tokens": 311,
"completion_tokens": 632,
"allowance_tokens": 32457,
"length_stops": 0,
"errors": 0,
"seconds": 4.172,
"budget_s": 3.8443314626332676,
"last_prompt_tokens": 311
},
"gpqa_diamond:gpqa_diamond/rec5rjeLsEq5Fg7Oj#1": {
"requests": 1,
"prompt_tokens": 218,
"completion_tokens": 935,
"allowance_tokens": 32550,
"length_stops": 0,
"errors": 0,
"seconds": 6.139,
"budget_s": 5.652682921955063,
"last_prompt_tokens": 218
},
"gpqa_diamond:gpqa_diamond/rec6sE2CRtD4drtHg#1": {
"requests": 1,
"prompt_tokens": 810,
"completion_tokens": 474,
"allowance_tokens": 31958,
"length_stops": 0,
"errors": 0,
"seconds": 3.244,
"budget_s": 2.9675085215406134,
"last_prompt_tokens": 810
},
"gpqa_diamond:gpqa_diamond/rec7C06XOB2h298ue#1": {
"requests": 1,
"prompt_tokens": 163,
"completion_tokens": 1034,
"allowance_tokens": 32605,
"length_stops": 0,
"errors": 0,
"seconds": 6.787,
"budget_s": 6.238677456568361,
"last_prompt_tokens": 163
},
"gpqa_diamond:gpqa_diamond/rec7qmSnbud4FHSqL#1": {
"requests": 1,
"prompt_tokens": 245,
"completion_tokens": 1369,
"allowance_tokens": 32523,
"length_stops": 0,
"errors": 0,
"seconds": 9.022,
"budget_s": 8.268798108371241,
"last_prompt_tokens": 245
},
"gpqa_diamond:gpqa_diamond/rec8nshandHARTkrg#1": {
"requests": 1,
"prompt_tokens": 478,
"completion_tokens": 603,
"allowance_tokens": 32290,
"length_stops": 0,
"errors": 0,
"seconds": 4.032,
"budget_s": 3.6956160493441668,
"last_prompt_tokens": 478
},
"gpqa_diamond:gpqa_diamond/rec8y3ZrBOcLgNEkE#1": {
"requests": 1,
"prompt_tokens": 198,
"completion_tokens": 430,
"allowance_tokens": 32570,
"length_stops": 0,
"errors": 0,
"seconds": 2.847,
"budget_s": 2.611611992703111,
"last_prompt_tokens": 198
},
"gpqa_diamond:gpqa_diamond/rec9GugrngvsQR99A#1": {
"requests": 1,
"prompt_tokens": 251,
"completion_tokens": 279,
"allowance_tokens": 32517,
"length_stops": 0,
"errors": 0,
"seconds": 1.87,
"budget_s": 1.710741802117014,
"last_prompt_tokens": 251
},
"gpqa_diamond:gpqa_diamond/rec9W28HgpEUeUN8k#1": {
"requests": 1,
"prompt_tokens": 250,
"completion_tokens": 882,
"allowance_tokens": 32518,
"length_stops": 0,
"errors": 0,
"seconds": 5.818,
"budget_s": 5.339193422477597,
"last_prompt_tokens": 250
},
"gpqa_diamond:gpqa_diamond/rec9ubQihAh6g9bft#1": {
"requests": 1,
"prompt_tokens": 143,
"completion_tokens": 357,
"allowance_tokens": 32625,
"length_stops": 0,
"errors": 0,
"seconds": 2.353,
"budget_s": 2.1646044596238565,
"last_prompt_tokens": 143
},
"gpqa_diamond:gpqa_diamond/recA1i5ZAh0Uzclxp#1": {
"requests": 1,
"prompt_tokens": 249,
"completion_tokens": 480,
"allowance_tokens": 32519,
"length_stops": 0,
"errors": 0,
"seconds": 3.183,
"budget_s": 2.9199830412154353,
"last_prompt_tokens": 249
},
"gpqa_diamond:gpqa_diamond/recAAJoHMW45Lv5je#1": {
"requests": 1,
"prompt_tokens": 225,
"completion_tokens": 1911,
"allowance_tokens": 32543,
"length_stops": 0,
"errors": 0,
"seconds": 12.623,
"budget_s": 11.525901418902398,
"last_prompt_tokens": 225
},
"gpqa_diamond:gpqa_diamond/recAYkd96NNuNl1Ei#1": {
"requests": 1,
"prompt_tokens": 141,
"completion_tokens": 262,
"allowance_tokens": 32627,
"length_stops": 0,
"errors": 0,
"seconds": 1.736,
"budget_s": 1.5931215415529836,
"last_prompt_tokens": 141
},
"gpqa_diamond:gpqa_diamond/recBGxVySx1pkrO1i#1": {
"requests": 1,
"prompt_tokens": 361,
"completion_tokens": 379,
"allowance_tokens": 32407,
"length_stops": 0,
"errors": 0,
"seconds": 2.542,
"budget_s": 2.3284147831682014,
"last_prompt_tokens": 361
},
"gpqa_diamond:gpqa_diamond/recBXlqrZUCR897It#1": {
"requests": 1,
"prompt_tokens": 380,
"completion_tokens": 637,
"allowance_tokens": 32388,
"length_stops": 0,
"errors": 0,
"seconds": 4.243,
"budget_s": 3.88523003198009,
"last_prompt_tokens": 380
},
"gpqa_diamond:gpqa_diamond/recBhnXrUyTJ6WHIR#1": {
"requests": 1,
"prompt_tokens": 228,
"completion_tokens": 278,
"allowance_tokens": 32540,
"length_stops": 0,
"errors": 0,
"seconds": 1.856,
"budget_s": 1.7015037333075573,
"last_prompt_tokens": 228
},
"gpqa_diamond:gpqa_diamond/recBtVK8rBVtIlXDq#1": {
"requests": 1,
"prompt_tokens": 162,
"completion_tokens": 345,
"allowance_tokens": 32606,
"length_stops": 0,
"errors": 0,
"seconds": 2.281,
"budget_s": 2.095170494394879,
"last_prompt_tokens": 162
},
"gpqa_diamond:gpqa_diamond/recCJJOeBGERaHYax#1": {
"requests": 1,
"prompt_tokens": 301,
"completion_tokens": 695,
"allowance_tokens": 32467,
"length_stops": 0,
"errors": 0,
"seconds": 4.604,
"budget_s": 4.222022700197981,
"last_prompt_tokens": 301
},
"gpqa_diamond:gpqa_diamond/recD8oX1KevFbl7bL#1": {
"requests": 1,
"prompt_tokens": 387,
"completion_tokens": 584,
"allowance_tokens": 32381,
"length_stops": 0,
"errors": 0,
"seconds": 3.897,
"budget_s": 3.567062263012789,
"last_prompt_tokens": 387
},
"gpqa_diamond:gpqa_diamond/recDDxpS9s8cwkqfq#1": {
"requests": 1,
"prompt_tokens": 105,
"completion_tokens": 409,
"allowance_tokens": 32663,
"length_stops": 0,
"errors": 0,
"seconds": 2.687,
"budget_s": 2.471716660898419,
"last_prompt_tokens": 105
},
"gpqa_diamond:gpqa_diamond/recDj2Y2BbtV02Wv5#1": {
"requests": 1,
"prompt_tokens": 136,
"completion_tokens": 480,
"allowance_tokens": 32632,
"length_stops": 0,
"errors": 0,
"seconds": 3.162,
"budget_s": 2.9031192753536548,
"last_prompt_tokens": 136
},
"gpqa_diamond:gpqa_diamond/recDytVnNYZe2HuUU#1": {
"requests": 1,
"prompt_tokens": 159,
"completion_tokens": 752,
"allowance_tokens": 32609,
"length_stops": 0,
"errors": 0,
"seconds": 4.959,
"budget_s": 4.542198343276274,
"last_prompt_tokens": 159
},
"gpqa_diamond:gpqa_diamond/recE2ihVfqEK4R9d0#1": {
"requests": 1,
"prompt_tokens": 164,
"completion_tokens": 2835,
"allowance_tokens": 32604,
"length_stops": 0,
"errors": 0,
"seconds": 18.764,
"budget_s": 17.06941080232721,
"last_prompt_tokens": 164
},
"gpqa_diamond:gpqa_diamond/recEmTBhx2hgw6tPQ#1": {
"requests": 1,
"prompt_tokens": 303,
"completion_tokens": 602,
"allowance_tokens": 32465,
"length_stops": 0,
"errors": 0,
"seconds": 4.008,
"budget_s": 3.662481963919229,
"last_prompt_tokens": 303
},
"gpqa_diamond:gpqa_diamond/recFaL6j8UMhutXrc#1": {
"requests": 1,
"prompt_tokens": 451,
"completion_tokens": 324,
"allowance_tokens": 32317,
"length_stops": 0,
"errors": 0,
"seconds": 2.205,
"budget_s": 2.0099637869704456,
"last_prompt_tokens": 451
},
"gpqa_diamond:gpqa_diamond/recG1ljJQcVy6CbSl#1": {
"requests": 1,
"prompt_tokens": 318,
"completion_tokens": 505,
"allowance_tokens": 32450,
"length_stops": 0,
"errors": 0,
"seconds": 3.373,
"budget_s": 3.080799354875859,
"last_prompt_tokens": 318
},
"gpqa_diamond:gpqa_diamond/recGFKDUd8qgz81Zl#1": {
"requests": 1,
"prompt_tokens": 158,
"completion_tokens": 374,
"allowance_tokens": 32610,
"length_stops": 0,
"errors": 0,
"seconds": 2.476,
"budget_s": 2.268985748573174,
"last_prompt_tokens": 158
},
"gpqa_diamond:gpqa_diamond/recGFNRVl1qBZGwyU#1": {
"requests": 1,
"prompt_tokens": 164,
"completion_tokens": 698,
"allowance_tokens": 32604,
"length_stops": 0,
"errors": 0,
"seconds": 4.608,
"budget_s": 4.2182702067778335,
"last_prompt_tokens": 164
},
"gpqa_diamond:gpqa_diamond/recGee5m84dg5FZkc#1": {
"requests": 1,
"prompt_tokens": 237,
"completion_tokens": 639,
"allowance_tokens": 32531,
"length_stops": 0,
"errors": 0,
"seconds": 4.24,
"budget_s": 3.8748903502478655,
"last_prompt_tokens": 237
},
"gpqa_diamond:gpqa_diamond/recHVUd2DkoKfedJZ#1": {
"requests": 1,
"prompt_tokens": 298,
"completion_tokens": 519,
"allowance_tokens": 32470,
"length_stops": 0,
"errors": 0,
"seconds": 3.469,
"budget_s": 3.162067611097513,
"last_prompt_tokens": 298
},
"gpqa_diamond:gpqa_diamond/recI1ls9OXdxatHQn#1": {
"requests": 1,
"prompt_tokens": 253,
"completion_tokens": 1514,
"allowance_tokens": 32515,
"length_stops": 0,
"errors": 0,
"seconds": 10.031,
"budget_s": 9.142884070510496,
"last_prompt_tokens": 253
},
"gpqa_diamond:gpqa_diamond/recIDiDKKrN61Auyr#1": {
"requests": 1,
"prompt_tokens": 179,
"completion_tokens": 259,
"allowance_tokens": 32589,
"length_stops": 0,
"errors": 0,
"seconds": 1.729,
"budget_s": 1.5803709968261865,
"last_prompt_tokens": 179
},
"gpqa_diamond:gpqa_diamond/recIIUynGGpsGEYuo#1": {
"requests": 1,
"prompt_tokens": 163,
"completion_tokens": 573,
"allowance_tokens": 32605,
"length_stops": 0,
"errors": 0,
"seconds": 3.781,
"budget_s": 3.4664122067040406,
"last_prompt_tokens": 163
},
"gpqa_diamond:gpqa_diamond/recIMcQfaQNa6vzsT#1": {
"requests": 1,
"prompt_tokens": 239,
"completion_tokens": 736,
"allowance_tokens": 32529,
"length_stops": 0,
"errors": 0,
"seconds": 4.879,
"budget_s": 4.458858420388707,
"last_prompt_tokens": 239
},
"gpqa_diamond:gpqa_diamond/recINGR1z01Fh1Z3A#1": {
"requests": 1,
"prompt_tokens": 254,
"completion_tokens": 769,
"allowance_tokens": 32514,
"length_stops": 0,
"errors": 0,
"seconds": 5.097,
"budget_s": 4.659858451616789,
"last_prompt_tokens": 254
},
"gpqa_diamond:gpqa_diamond/recINSFEYLCyyd08m#1": {
"requests": 1,
"prompt_tokens": 235,
"completion_tokens": 408,
"allowance_tokens": 32533,
"length_stops": 0,
"errors": 0,
"seconds": 2.719,
"budget_s": 2.484678179782149,
"last_prompt_tokens": 235
},
"gpqa_diamond:gpqa_diamond/recIOlKBsOeEcgkA1#1": {
"requests": 1,
"prompt_tokens": 286,
"completion_tokens": 375,
"allowance_tokens": 32482,
"length_stops": 0,
"errors": 0,
"seconds": 2.515,
"budget_s": 2.293488562716847,
"last_prompt_tokens": 286
},
"gpqa_diamond:gpqa_diamond/recIj8lR4tuDgrHou#1": {
"requests": 1,
"prompt_tokens": 186,
"completion_tokens": 511,
"allowance_tokens": 32582,
"length_stops": 0,
"errors": 0,
"seconds": 3.385,
"budget_s": 3.0970344579584346,
"last_prompt_tokens": 186
},
"gpqa_diamond:gpqa_diamond/recIlghoOw5VbY4Zh#1": {
"requests": 1,
"prompt_tokens": 103,
"completion_tokens": 350,
"allowance_tokens": 32665,
"length_stops": 0,
"errors": 0,
"seconds": 2.309,
"budget_s": 2.116784417279999,
"last_prompt_tokens": 103
},
"gpqa_diamond:gpqa_diamond/recJJcOHhFeSaRpl4#1": {
"requests": 1,
"prompt_tokens": 215,
"completion_tokens": 587,
"allowance_tokens": 32553,
"length_stops": 0,
"errors": 0,
"seconds": 3.894,
"budget_s": 3.5586168437679726,
"last_prompt_tokens": 215
},
"gpqa_diamond:gpqa_diamond/recJZ3QEfRKjYw9a7#1": {
"requests": 1,
"prompt_tokens": 149,
"completion_tokens": 712,
"allowance_tokens": 32619,
"length_stops": 0,
"errors": 0,
"seconds": 4.698,
"budget_s": 4.300063884279057,
"last_prompt_tokens": 149
},
"gpqa_diamond:gpqa_diamond/recJpyGtGIsxulevT#1": {
"requests": 1,
"prompt_tokens": 157,
"completion_tokens": 712,
"allowance_tokens": 32611,
"length_stops": 0,
"errors": 0,
"seconds": 4.702,
"budget_s": 4.301342392881235,
"last_prompt_tokens": 157
},
"gpqa_diamond:gpqa_diamond/recK9F5aqdaybl8bb#1": {
"requests": 1,
"prompt_tokens": 413,
"completion_tokens": 307,
"allowance_tokens": 32355,
"length_stops": 0,
"errors": 0,
"seconds": 2.089,
"budget_s": 1.9021360535576826,
"last_prompt_tokens": 413
},
"gpqa_diamond:gpqa_diamond/recKcjTwONILZqHAX#1": {
"requests": 1,
"prompt_tokens": 207,
"completion_tokens": 195,
"allowance_tokens": 32561,
"length_stops": 0,
"errors": 0,
"seconds": 1.325,
"budget_s": 1.1992686074090242,
"last_prompt_tokens": 207
},
"gpqa_diamond:gpqa_diamond/recKm6LNWykGapmCr#1": {
"requests": 1,
"prompt_tokens": 174,
"completion_tokens": 672,
"allowance_tokens": 32594,
"length_stops": 0,
"errors": 0,
"seconds": 4.445,
"budget_s": 4.063495617039234,
"last_prompt_tokens": 174
},
"gpqa_diamond:gpqa_diamond/recKv4hFyNpViUXnL#1": {
"requests": 1,
"prompt_tokens": 255,
"completion_tokens": 1435,
"allowance_tokens": 32513,
"length_stops": 0,
"errors": 0,
"seconds": 9.525,
"budget_s": 8.667871434421652,
"last_prompt_tokens": 255
},
"gpqa_diamond:gpqa_diamond/recL9MFV5zmdlle5T#1": {
"requests": 1,
"prompt_tokens": 233,
"completion_tokens": 579,
"allowance_tokens": 32535,
"length_stops": 0,
"errors": 0,
"seconds": 3.849,
"budget_s": 3.513257001344606,
"last_prompt_tokens": 233
},
"gpqa_diamond:gpqa_diamond/recLAgwx9vbgB5EHk#1": {
"requests": 1,
"prompt_tokens": 176,
"completion_tokens": 491,
"allowance_tokens": 32592,
"length_stops": 0,
"errors": 0,
"seconds": 3.255,
"budget_s": 2.9752442891698,
"last_prompt_tokens": 176
},
"gpqa_diamond:gpqa_diamond/recLB0EkQ54bYVhnd#1": {
"requests": 1,
"prompt_tokens": 213,
"completion_tokens": 2492,
"allowance_tokens": 32555,
"length_stops": 0,
"errors": 0,
"seconds": 16.549,
"budget_s": 15.018548689584184,
"last_prompt_tokens": 213
},
"gpqa_diamond:gpqa_diamond/recLSzwBUS3olCjhl#1": {
"requests": 1,
"prompt_tokens": 199,
"completion_tokens": 525,
"allowance_tokens": 32569,
"length_stops": 0,
"errors": 0,
"seconds": 3.492,
"budget_s": 3.183206226745566,
"last_prompt_tokens": 199
},
"gpqa_diamond:gpqa_diamond/recLrRirj7iXoB0rU#1": {
"requests": 1,
"prompt_tokens": 268,
"completion_tokens": 676,
"allowance_tokens": 32500,
"length_stops": 0,
"errors": 0,
"seconds": 4.501,
"budget_s": 4.1024229955337566,
"last_prompt_tokens": 268
},
"gpqa_diamond:gpqa_diamond/recMicVBcqy1xM1jq#1": {
"requests": 1,
"prompt_tokens": 216,
"completion_tokens": 1731,
"allowance_tokens": 32552,
"length_stops": 0,
"errors": 0,
"seconds": 11.498,
"budget_s": 10.441087721047918,
"last_prompt_tokens": 216
},
"gpqa_diamond:gpqa_diamond/recN4DY9Q5V03glmQ#1": {
"requests": 1,
"prompt_tokens": 196,
"completion_tokens": 1315,
"allowance_tokens": 32572,
"length_stops": 0,
"errors": 0,
"seconds": 8.73,
"budget_s": 7.934679358740875,
"last_prompt_tokens": 196
},
"gpqa_diamond:gpqa_diamond/recNAiPUKOulQbKVP#1": {
"requests": 1,
"prompt_tokens": 93,
"completion_tokens": 277,
"allowance_tokens": 32675,
"length_stops": 0,
"errors": 0,
"seconds": 1.834,
"budget_s": 1.6765927042772282,
"last_prompt_tokens": 93
},
"gpqa_diamond:gpqa_diamond/recNFJjE5PPTqVJGv#1": {
"requests": 1,
"prompt_tokens": 140,
"completion_tokens": 1666,
"allowance_tokens": 32628,
"length_stops": 0,
"errors": 0,
"seconds": 11.038,
"budget_s": 10.0345894571621,
"last_prompt_tokens": 140
},
"gpqa_diamond:gpqa_diamond/recNu3MXkvWUzHZr9#1": {
"requests": 1,
"prompt_tokens": 211,
"completion_tokens": 1055,
"allowance_tokens": 32557,
"length_stops": 0,
"errors": 0,
"seconds": 6.997,
"budget_s": 6.373386618518677,
"last_prompt_tokens": 211
},
"gpqa_diamond:gpqa_diamond/recNuT2oSnO86bxOx#1": {
"requests": 1,
"prompt_tokens": 220,
"completion_tokens": 521,
"allowance_tokens": 32548,
"length_stops": 0,
"errors": 0,
"seconds": 3.471,
"budget_s": 3.1623187165570146,
"last_prompt_tokens": 220
},
"gpqa_diamond:gpqa_diamond/recO3hvCWRGiG0odN#1": {
"requests": 1,
"prompt_tokens": 243,
"completion_tokens": 859,
"allowance_tokens": 32525,
"length_stops": 0,
"errors": 0,
"seconds": 5.708,
"budget_s": 5.1996233730087065,
"last_prompt_tokens": 243
},
"gpqa_diamond:gpqa_diamond/recOYsaYs6RmtlTDy#1": {
"requests": 1,
"prompt_tokens": 251,
"completion_tokens": 454,
"allowance_tokens": 32517,
"length_stops": 0,
"errors": 0,
"seconds": 3.034,
"budget_s": 2.763823674996234,
"last_prompt_tokens": 251
},
"gpqa_diamond:gpqa_diamond/recOkA2zsy4GSMD9R#1": {
"requests": 1,
"prompt_tokens": 245,
"completion_tokens": 1480,
"allowance_tokens": 32523,
"length_stops": 0,
"errors": 0,
"seconds": 9.832,
"budget_s": 8.936722452221073,
"last_prompt_tokens": 245
},
"gpqa_diamond:gpqa_diamond/recOvqPKUtyy9ISA1#1": {
"requests": 1,
"prompt_tokens": 292,
"completion_tokens": 341,
"allowance_tokens": 32476,
"length_stops": 0,
"errors": 0,
"seconds": 2.3,
"budget_s": 2.0896928515130146,
"last_prompt_tokens": 292
},
"gpqa_diamond:gpqa_diamond/recPIzpnuYpB4yvmp#1": {
"requests": 1,
"prompt_tokens": 283,
"completion_tokens": 969,
"allowance_tokens": 32485,
"length_stops": 0,
"errors": 0,
"seconds": 6.446,
"budget_s": 5.868385066143107,
"last_prompt_tokens": 283
},
"gpqa_diamond:gpqa_diamond/recPL4ZPpVYxgMu57#1": {
"requests": 1,
"prompt_tokens": 124,
"completion_tokens": 302,
"allowance_tokens": 32644,
"length_stops": 0,
"errors": 0,
"seconds": 2.006,
"budget_s": 1.8312263906336244,
"last_prompt_tokens": 124
},
"gpqa_diamond:gpqa_diamond/recPSTGXK3P39yNYT#1": {
"requests": 1,
"prompt_tokens": 241,
"completion_tokens": 338,
"allowance_tokens": 32527,
"length_stops": 0,
"errors": 0,
"seconds": 2.273,
"budget_s": 2.06435319554504,
"last_prompt_tokens": 241
},
"gpqa_diamond:gpqa_diamond/recPzW1WqRnPs57D6#1": {
"requests": 1,
"prompt_tokens": 171,
"completion_tokens": 386,
"allowance_tokens": 32597,
"length_stops": 0,
"errors": 0,
"seconds": 2.566,
"budget_s": 2.34303026524374,
"last_prompt_tokens": 171
},
"gpqa_diamond:gpqa_diamond/recQ8oxSY2aCwFRS8#1": {
"requests": 1,
"prompt_tokens": 205,
"completion_tokens": 2180,
"allowance_tokens": 32563,
"length_stops": 0,
"errors": 0,
"seconds": 14.481,
"budget_s": 13.139780726683584,
"last_prompt_tokens": 205
},
"gpqa_diamond:gpqa_diamond/recQiWOxhkXRz9NVE#1": {
"requests": 1,
"prompt_tokens": 175,
"completion_tokens": 681,
"allowance_tokens": 32593,
"length_stops": 0,
"errors": 0,
"seconds": 4.506,
"budget_s": 4.117780860824428,
"last_prompt_tokens": 175
},
"gpqa_diamond:gpqa_diamond/recRTiyaRymerDJ4y#1": {
"requests": 1,
"prompt_tokens": 175,
"completion_tokens": 551,
"allowance_tokens": 32593,
"length_stops": 0,
"errors": 0,
"seconds": 3.648,
"budget_s": 3.335942862574392,
"last_prompt_tokens": 175
},
"gpqa_diamond:gpqa_diamond/recRZjaElf8ft7xTq#1": {
"requests": 1,
"prompt_tokens": 226,
"completion_tokens": 388,
"allowance_tokens": 32542,
"length_stops": 0,
"errors": 0,
"seconds": 2.591,
"budget_s": 2.3630351860949603,
"last_prompt_tokens": 226
},
"gpqa_diamond:gpqa_diamond/recReg13iV2HwJTaA#1": {
"requests": 1,
"prompt_tokens": 222,
"completion_tokens": 513,
"allowance_tokens": 32546,
"length_stops": 0,
"errors": 0,
"seconds": 3.412,
"budget_s": 3.1144906737779157,
"last_prompt_tokens": 222
},
"gpqa_diamond:gpqa_diamond/recRgabRzMaEoBRcM#1": {
"requests": 1,
"prompt_tokens": 194,
"completion_tokens": 625,
"allowance_tokens": 32574,
"length_stops": 0,
"errors": 0,
"seconds": 4.15,
"budget_s": 3.7839503722415757,
"last_prompt_tokens": 194
},
"gpqa_diamond:gpqa_diamond/recRhwQwHijW0gCLo#1": {
"requests": 1,
"prompt_tokens": 317,
"completion_tokens": 969,
"allowance_tokens": 32451,
"length_stops": 0,
"errors": 0,
"seconds": 6.447,
"budget_s": 5.874220943833628,
"last_prompt_tokens": 317
},
"gpqa_diamond:gpqa_diamond/recS48OsU6kVadBPW#1": {
"requests": 1,
"prompt_tokens": 197,
"completion_tokens": 1179,
"allowance_tokens": 32571,
"length_stops": 0,
"errors": 0,
"seconds": 7.799,
"budget_s": 7.116807266433465,
"last_prompt_tokens": 197
},
"gpqa_diamond:gpqa_diamond/recSBcGLPatKb3Ygu#1": {
"requests": 1,
"prompt_tokens": 179,
"completion_tokens": 1467,
"allowance_tokens": 32589,
"length_stops": 0,
"errors": 0,
"seconds": 9.704,
"budget_s": 8.845670883680967,
"last_prompt_tokens": 179
},
"gpqa_diamond:gpqa_diamond/recTs7qzfJs6kfLUK#1": {
"requests": 1,
"prompt_tokens": 577,
"completion_tokens": 362,
"allowance_tokens": 32191,
"length_stops": 0,
"errors": 0,
"seconds": 2.484,
"budget_s": 2.257115931105059,
"last_prompt_tokens": 577
},
"gpqa_diamond:gpqa_diamond/recUBgVlkKzcRPDdK#1": {
"requests": 1,
"prompt_tokens": 217,
"completion_tokens": 512,
"allowance_tokens": 32551,
"length_stops": 0,
"errors": 0,
"seconds": 3.405,
"budget_s": 3.107720848487306,
"last_prompt_tokens": 217
},
"gpqa_diamond:gpqa_diamond/recUOePh79cp4T2Bg#1": {
"requests": 1,
"prompt_tokens": 556,
"completion_tokens": 715,
"allowance_tokens": 32212,
"length_stops": 0,
"errors": 0,
"seconds": 4.816,
"budget_s": 4.383242953519855,
"last_prompt_tokens": 556
},
"gpqa_diamond:gpqa_diamond/recUc29lMDBEvurYo#1": {
"requests": 1,
"prompt_tokens": 183,
"completion_tokens": 1812,
"allowance_tokens": 32585,
"length_stops": 0,
"errors": 0,
"seconds": 11.986,
"budget_s": 10.916613980490164,
"last_prompt_tokens": 183
},
"gpqa_diamond:gpqa_diamond/recV1nqYQvpII94oC#1": {
"requests": 1,
"prompt_tokens": 514,
"completion_tokens": 858,
"allowance_tokens": 32254,
"length_stops": 0,
"errors": 0,
"seconds": 5.739,
"budget_s": 5.234047469383708,
"last_prompt_tokens": 514
},
"gpqa_diamond:gpqa_diamond/recVE8cUNHpHZIAvL#1": {
"requests": 1,
"prompt_tokens": 289,
"completion_tokens": 590,
"allowance_tokens": 32479,
"length_stops": 0,
"errors": 0,
"seconds": 3.929,
"budget_s": 3.5834985601133993,
"last_prompt_tokens": 289
},
"gpqa_diamond:gpqa_diamond/recVEXvhYxAhELVqB#1": {
"requests": 1,
"prompt_tokens": 387,
"completion_tokens": 610,
"allowance_tokens": 32381,
"length_stops": 0,
"errors": 0,
"seconds": 4.076,
"budget_s": 3.7192259746367142,
"last_prompt_tokens": 387
},
"gpqa_diamond:gpqa_diamond/recVvpD8miVjmmyfe#1": {
"requests": 1,
"prompt_tokens": 125,
"completion_tokens": 3347,
"allowance_tokens": 32643,
"length_stops": 0,
"errors": 0,
"seconds": 22.189,
"budget_s": 20.13314111486423,
"last_prompt_tokens": 125
},
"gpqa_diamond:gpqa_diamond/recWPMm91mGDLXONm#1": {
"requests": 1,
"prompt_tokens": 227,
"completion_tokens": 382,
"allowance_tokens": 32541,
"length_stops": 0,
"errors": 0,
"seconds": 2.543,
"budget_s": 2.322879796617797,
"last_prompt_tokens": 227
},
"gpqa_diamond:gpqa_diamond/recWXwn9v4IG9ZrM6#1": {
"requests": 1,
"prompt_tokens": 278,
"completion_tokens": 424,
"allowance_tokens": 32490,
"length_stops": 0,
"errors": 0,
"seconds": 2.829,
"budget_s": 2.5831822260574984,
"last_prompt_tokens": 278
},
"gpqa_diamond:gpqa_diamond/recWxGU8Q4YReJ1tb#1": {
"requests": 1,
"prompt_tokens": 815,
"completion_tokens": 337,
"allowance_tokens": 31953,
"length_stops": 0,
"errors": 0,
"seconds": 2.353,
"budget_s": 2.1363475491873043,
"last_prompt_tokens": 815
},
"gpqa_diamond:gpqa_diamond/recWzUUYBN0NuXO2k#1": {
"requests": 1,
"prompt_tokens": 194,
"completion_tokens": 1641,
"allowance_tokens": 32574,
"length_stops": 0,
"errors": 0,
"seconds": 10.868,
"budget_s": 9.891249882686258,
"last_prompt_tokens": 194
},
"gpqa_diamond:gpqa_diamond/recXsYa3i2UhGF5fe#1": {
"requests": 1,
"prompt_tokens": 114,
"completion_tokens": 1047,
"allowance_tokens": 32654,
"length_stops": 0,
"errors": 0,
"seconds": 6.894,
"budget_s": 6.3044550959856895,
"last_prompt_tokens": 114
},
"gpqa_diamond:gpqa_diamond/recXsuOHRBLcyenF2#1": {
"requests": 1,
"prompt_tokens": 129,
"completion_tokens": 276,
"allowance_tokens": 32639,
"length_stops": 0,
"errors": 0,
"seconds": 1.827,
"budget_s": 1.671926752395919,
"last_prompt_tokens": 129
},
"gpqa_diamond:gpqa_diamond/recXvQ6gWAmyakrpD#1": {
"requests": 1,
"prompt_tokens": 292,
"completion_tokens": 937,
"allowance_tokens": 32476,
"length_stops": 0,
"errors": 0,
"seconds": 6.227,
"budget_s": 5.673741461052299,
"last_prompt_tokens": 292
},
"gpqa_diamond:gpqa_diamond/recYA3LPsCvF1fTMI#1": {
"requests": 1,
"prompt_tokens": 322,
"completion_tokens": 575,
"allowance_tokens": 32446,
"length_stops": 0,
"errors": 0,
"seconds": 3.839,
"budget_s": 3.4994236496082722,
"last_prompt_tokens": 322
},
"gpqa_diamond:gpqa_diamond/recYCMJWH7IpcdqwE#1": {
"requests": 1,
"prompt_tokens": 427,
"completion_tokens": 526,
"allowance_tokens": 32341,
"length_stops": 0,
"errors": 0,
"seconds": 3.533,
"budget_s": 3.220426374049595,
"last_prompt_tokens": 427
},
"gpqa_diamond:gpqa_diamond/recYOzCsevNz61Lyn#1": {
"requests": 1,
"prompt_tokens": 304,
"completion_tokens": 538,
"allowance_tokens": 32464,
"length_stops": 0,
"errors": 0,
"seconds": 3.59,
"budget_s": 3.274171768662967,
"last_prompt_tokens": 304
},
"gpqa_diamond:gpqa_diamond/recYt8xx80OTyDsL0#1": {
"requests": 1,
"prompt_tokens": 166,
"completion_tokens": 363,
"allowance_tokens": 32602,
"length_stops": 0,
"errors": 0,
"seconds": 2.41,
"budget_s": 2.2009343184761185,
"last_prompt_tokens": 166
},
"gpqa_diamond:gpqa_diamond/recZ13cwgDQf9jRd9#1": {
"requests": 1,
"prompt_tokens": 253,
"completion_tokens": 337,
"allowance_tokens": 32515,
"length_stops": 0,
"errors": 0,
"seconds": 2.262,
"budget_s": 2.0571204475387885,
"last_prompt_tokens": 253
},
"gpqa_diamond:gpqa_diamond/recZSGUkn56v9kEp1#1": {
"requests": 1,
"prompt_tokens": 198,
"completion_tokens": 391,
"allowance_tokens": 32570,
"length_stops": 0,
"errors": 0,
"seconds": 2.599,
"budget_s": 2.3742187273592523,
"last_prompt_tokens": 198
},
"gpqa_diamond:gpqa_diamond/recZWeueB7lSPR6wN#1": {
"requests": 1,
"prompt_tokens": 210,
"completion_tokens": 512,
"allowance_tokens": 32558,
"length_stops": 0,
"errors": 0,
"seconds": 3.396,
"budget_s": 3.1039923204486084,
"last_prompt_tokens": 210
},
"gpqa_diamond:gpqa_diamond/recZbxrocrxh9YENH#1": {
"requests": 1,
"prompt_tokens": 216,
"completion_tokens": 1368,
"allowance_tokens": 32552,
"length_stops": 0,
"errors": 0,
"seconds": 9.056,
"budget_s": 8.254729403980155,
"last_prompt_tokens": 216
},
"gpqa_diamond:gpqa_diamond/recZdZxsN0AxDx0pB#1": {
"requests": 1,
"prompt_tokens": 393,
"completion_tokens": 663,
"allowance_tokens": 32375,
"length_stops": 0,
"errors": 0,
"seconds": 4.433,
"budget_s": 4.041472323951451,
"last_prompt_tokens": 393
},
"gpqa_diamond:gpqa_diamond/recZt4x514UQNnsgy#1": {
"requests": 1,
"prompt_tokens": 193,
"completion_tokens": 480,
"allowance_tokens": 32575,
"length_stops": 0,
"errors": 0,
"seconds": 3.186,
"budget_s": 2.9093335993766045,
"last_prompt_tokens": 193
},
"gpqa_diamond:gpqa_diamond/reca44yABeO2fx7UB#1": {
"requests": 1,
"prompt_tokens": 258,
"completion_tokens": 446,
"allowance_tokens": 32510,
"length_stops": 0,
"errors": 0,
"seconds": 2.974,
"budget_s": 2.7145523792141484,
"last_prompt_tokens": 258
},
"gpqa_diamond:gpqa_diamond/recaXdGn3FAIkxLGM#1": {
"requests": 1,
"prompt_tokens": 160,
"completion_tokens": 697,
"allowance_tokens": 32608,
"length_stops": 0,
"errors": 0,
"seconds": 4.601,
"budget_s": 4.209583018988272,
"last_prompt_tokens": 160
},
"gpqa_diamond:gpqa_diamond/recafnH3RRnMBXEb1#1": {
"requests": 1,
"prompt_tokens": 254,
"completion_tokens": 512,
"allowance_tokens": 32514,
"length_stops": 0,
"errors": 0,
"seconds": 3.41,
"budget_s": 3.111387549942715,
"last_prompt_tokens": 254
},
"gpqa_diamond:gpqa_diamond/recakFKZioXUuldcd#1": {
"requests": 1,
"prompt_tokens": 172,
"completion_tokens": 440,
"allowance_tokens": 32596,
"length_stops": 0,
"errors": 0,
"seconds": 2.911,
"budget_s": 2.666148891135539,
"last_prompt_tokens": 172
},
"gpqa_diamond:gpqa_diamond/recatj4YXzbOaq3Ar#1": {
"requests": 1,
"prompt_tokens": 166,
"completion_tokens": 845,
"allowance_tokens": 32602,
"length_stops": 0,
"errors": 0,
"seconds": 5.572,
"budget_s": 5.1009514360483434,
"last_prompt_tokens": 166
},
"gpqa_diamond:gpqa_diamond/recb2M22zaD3tL6Qc#1": {
"requests": 1,
"prompt_tokens": 359,
"completion_tokens": 639,
"allowance_tokens": 32409,
"length_stops": 0,
"errors": 0,
"seconds": 4.266,
"budget_s": 3.8924611670262377,
"last_prompt_tokens": 359
},
"gpqa_diamond:gpqa_diamond/recb4cGsC6BJUCU3V#1": {
"requests": 1,
"prompt_tokens": 222,
"completion_tokens": 608,
"allowance_tokens": 32546,
"length_stops": 0,
"errors": 0,
"seconds": 4.037,
"budget_s": 3.6846373260286667,
"last_prompt_tokens": 222
},
"gpqa_diamond:gpqa_diamond/recb80OwMgNnceA9t#1": {
"requests": 1,
"prompt_tokens": 193,
"completion_tokens": 366,
"allowance_tokens": 32575,
"length_stops": 0,
"errors": 0,
"seconds": 2.435,
"budget_s": 2.224646616043373,
"last_prompt_tokens": 193
},
"gpqa_diamond:gpqa_diamond/reccOKzFNmyqeJ6ry#1": {
"requests": 1,
"prompt_tokens": 201,
"completion_tokens": 364,
"allowance_tokens": 32567,
"length_stops": 0,
"errors": 0,
"seconds": 2.423,
"budget_s": 2.2138955279225074,
"last_prompt_tokens": 201
},
"gpqa_diamond:gpqa_diamond/reccVBrYdwsB84fGy#1": {
"requests": 1,
"prompt_tokens": 345,
"completion_tokens": 565,
"allowance_tokens": 32423,
"length_stops": 0,
"errors": 0,
"seconds": 3.775,
"budget_s": 3.4451570399653777,
"last_prompt_tokens": 345
},
"gpqa_diamond:gpqa_diamond/reccVFW11Elovnf8T#1": {
"requests": 1,
"prompt_tokens": 410,
"completion_tokens": 526,
"allowance_tokens": 32358,
"length_stops": 0,
"errors": 0,
"seconds": 3.535,
"budget_s": 3.220271185128109,
"last_prompt_tokens": 410
},
"gpqa_diamond:gpqa_diamond/recclFbsjbaiVVnnV#1": {
"requests": 1,
"prompt_tokens": 345,
"completion_tokens": 609,
"allowance_tokens": 32423,
"length_stops": 0,
"errors": 0,
"seconds": 4.066,
"budget_s": 3.710375746122225,
"last_prompt_tokens": 345
},
"gpqa_diamond:gpqa_diamond/recdya6FuYraBU5Rh#1": {
"requests": 1,
"prompt_tokens": 227,
"completion_tokens": 1005,
"allowance_tokens": 32541,
"length_stops": 0,
"errors": 0,
"seconds": 6.658,
"budget_s": 6.074732656576581,
"last_prompt_tokens": 227
},
"gpqa_diamond:gpqa_diamond/receYQxnUBvxPcvbQ#1": {
"requests": 1,
"prompt_tokens": 249,
"completion_tokens": 1100,
"allowance_tokens": 32519,
"length_stops": 0,
"errors": 0,
"seconds": 7.301,
"budget_s": 6.650335737485782,
"last_prompt_tokens": 249
},
"gpqa_diamond:gpqa_diamond/receo3UUzRvWCqyfA#1": {
"requests": 1,
"prompt_tokens": 265,
"completion_tokens": 365,
"allowance_tokens": 32503,
"length_stops": 0,
"errors": 0,
"seconds": 2.452,
"budget_s": 2.229890395853127,
"last_prompt_tokens": 265
},
"gpqa_diamond:gpqa_diamond/recf6ayQmL1SxKbvW#1": {
"requests": 1,
"prompt_tokens": 263,
"completion_tokens": 417,
"allowance_tokens": 32505,
"length_stops": 0,
"errors": 0,
"seconds": 2.79,
"budget_s": 2.542673989032604,
"last_prompt_tokens": 263
},
"gpqa_diamond:gpqa_diamond/recfTlTMjZBuOducT#1": {
"requests": 1,
"prompt_tokens": 149,
"completion_tokens": 477,
"allowance_tokens": 32619,
"length_stops": 0,
"errors": 0,
"seconds": 3.157,
"budget_s": 2.8868929701337223,
"last_prompt_tokens": 149
},
"gpqa_diamond:gpqa_diamond/recgC4GJ8wUemlUQD#1": {
"requests": 1,
"prompt_tokens": 92,
"completion_tokens": 345,
"allowance_tokens": 32676,
"length_stops": 0,
"errors": 0,
"seconds": 2.283,
"budget_s": 2.085156623295034,
"last_prompt_tokens": 92
},
"gpqa_diamond:gpqa_diamond/recgI6tUQ7RLJRWGx#1": {
"requests": 1,
"prompt_tokens": 195,
"completion_tokens": 666,
"allowance_tokens": 32573,
"length_stops": 0,
"errors": 0,
"seconds": 4.41,
"budget_s": 4.030723356837516,
"last_prompt_tokens": 195
},
"gpqa_diamond:gpqa_diamond/recgM7o1tcc7tP778#1": {
"requests": 1,
"prompt_tokens": 285,
"completion_tokens": 1776,
"allowance_tokens": 32483,
"length_stops": 0,
"errors": 0,
"seconds": 11.8,
"budget_s": 10.726194791061326,
"last_prompt_tokens": 285
},
"gpqa_diamond:gpqa_diamond/recgXxEgllSGEpELP#1": {
"requests": 1,
"prompt_tokens": 536,
"completion_tokens": 470,
"allowance_tokens": 32232,
"length_stops": 0,
"errors": 0,
"seconds": 3.191,
"budget_s": 2.9025265080664377,
"last_prompt_tokens": 536
},
"gpqa_diamond:gpqa_diamond/rechKl68Uc6H7vU0N#1": {
"requests": 1,
"prompt_tokens": 300,
"completion_tokens": 353,
"allowance_tokens": 32468,
"length_stops": 0,
"errors": 0,
"seconds": 2.379,
"budget_s": 2.160782726003353,
"last_prompt_tokens": 300
},
"gpqa_diamond:gpqa_diamond/rechgQucGlrnt8KRV#1": {
"requests": 1,
"prompt_tokens": 256,
"completion_tokens": 1865,
"allowance_tokens": 32512,
"length_stops": 0,
"errors": 0,
"seconds": 12.395,
"budget_s": 11.25357091037875,
"last_prompt_tokens": 256
},
"gpqa_diamond:gpqa_diamond/rechtUKxC6G67H7Pr#1": {
"requests": 1,
"prompt_tokens": 242,
"completion_tokens": 587,
"allowance_tokens": 32526,
"length_stops": 0,
"errors": 0,
"seconds": 3.904,
"budget_s": 3.5607415665740567,
"last_prompt_tokens": 242
},
"gpqa_diamond:gpqa_diamond/recihePFulRgNKsIn#1": {
"requests": 1,
"prompt_tokens": 177,
"completion_tokens": 489,
"allowance_tokens": 32591,
"length_stops": 0,
"errors": 0,
"seconds": 3.242,
"budget_s": 2.961455858473843,
"last_prompt_tokens": 177
},
"gpqa_diamond:gpqa_diamond/reciuppdQde4FGX1z#1": {
"requests": 1,
"prompt_tokens": 187,
"completion_tokens": 1868,
"allowance_tokens": 32581,
"length_stops": 0,
"errors": 0,
"seconds": 12.403,
"budget_s": 11.257332205889561,
"last_prompt_tokens": 187
},
"gpqa_diamond:gpqa_diamond/recixxJmdux0d8LZQ#1": {
"requests": 1,
"prompt_tokens": 291,
"completion_tokens": 367,
"allowance_tokens": 32477,
"length_stops": 0,
"errors": 0,
"seconds": 2.467,
"budget_s": 2.2444002973540615,
"last_prompt_tokens": 291
},
"gpqa_diamond:gpqa_diamond/recjJ54TXc04enRkZ#1": {
"requests": 1,
"prompt_tokens": 252,
"completion_tokens": 546,
"allowance_tokens": 32516,
"length_stops": 0,
"errors": 0,
"seconds": 3.632,
"budget_s": 3.316068369278436,
"last_prompt_tokens": 252
},
"gpqa_diamond:gpqa_diamond/recjgMJaMxz4ESDF2#1": {
"requests": 1,
"prompt_tokens": 213,
"completion_tokens": 328,
"allowance_tokens": 32555,
"length_stops": 0,
"errors": 0,
"seconds": 2.192,
"budget_s": 1.9987967429368085,
"last_prompt_tokens": 213
},
"gpqa_diamond:gpqa_diamond/reck1zKeCssmN82Ga#1": {
"requests": 1,
"prompt_tokens": 155,
"completion_tokens": 429,
"allowance_tokens": 32613,
"length_stops": 0,
"errors": 0,
"seconds": 2.835,
"budget_s": 2.5980068484215506,
"last_prompt_tokens": 155
},
"gpqa_diamond:gpqa_diamond/reck4G4xxv3YnpbtQ#1": {
"requests": 1,
"prompt_tokens": 304,
"completion_tokens": 461,
"allowance_tokens": 32464,
"length_stops": 0,
"errors": 0,
"seconds": 3.081,
"budget_s": 2.81266612953611,
"last_prompt_tokens": 304
},
"gpqa_diamond:gpqa_diamond/reckEnrOPFT9Ru7tW#1": {
"requests": 1,
"prompt_tokens": 253,
"completion_tokens": 417,
"allowance_tokens": 32515,
"length_stops": 0,
"errors": 0,
"seconds": 2.78,
"budget_s": 2.540446323025054,
"last_prompt_tokens": 253
},
"gpqa_diamond:gpqa_diamond/reckYqR92oJkqJXkE#1": {
"requests": 1,
"prompt_tokens": 132,
"completion_tokens": 312,
"allowance_tokens": 32636,
"length_stops": 0,
"errors": 0,
"seconds": 2.064,
"budget_s": 1.8915856996244642,
"last_prompt_tokens": 132
},
"gpqa_diamond:gpqa_diamond/reckl6PkaEMWM50yZ#1": {
"requests": 1,
"prompt_tokens": 275,
"completion_tokens": 599,
"allowance_tokens": 32493,
"length_stops": 0,
"errors": 0,
"seconds": 3.978,
"budget_s": 3.6393261760962745,
"last_prompt_tokens": 275
},
"gpqa_diamond:gpqa_diamond/recl1UtgTVKishAq4#1": {
"requests": 1,
"prompt_tokens": 202,
"completion_tokens": 554,
"allowance_tokens": 32566,
"length_stops": 0,
"errors": 0,
"seconds": 3.669,
"budget_s": 3.3574693186846116,
"last_prompt_tokens": 202
},
"gpqa_diamond:gpqa_diamond/recmI7EiLv72PxmYK#1": {
"requests": 1,
"prompt_tokens": 234,
"completion_tokens": 249,
"allowance_tokens": 32534,
"length_stops": 0,
"errors": 0,
"seconds": 1.675,
"budget_s": 1.5273464120053126,
"last_prompt_tokens": 234
},
"gpqa_diamond:gpqa_diamond/recmkvk6EFAmqyMxR#1": {
"requests": 1,
"prompt_tokens": 164,
"completion_tokens": 812,
"allowance_tokens": 32604,
"length_stops": 0,
"errors": 0,
"seconds": 5.359,
"budget_s": 4.903442794511703,
"last_prompt_tokens": 164
},
"gpqa_diamond:gpqa_diamond/recmwwQJnx7ll7bqL#1": {
"requests": 1,
"prompt_tokens": 245,
"completion_tokens": 809,
"allowance_tokens": 32523,
"length_stops": 0,
"errors": 0,
"seconds": 5.365,
"budget_s": 4.89883507963699,
"last_prompt_tokens": 245
},
"gpqa_diamond:gpqa_diamond/recn3NhOhqAPLda16#1": {
"requests": 1,
"prompt_tokens": 251,
"completion_tokens": 1055,
"allowance_tokens": 32517,
"length_stops": 0,
"errors": 0,
"seconds": 6.995,
"budget_s": 6.3802803933337335,
"last_prompt_tokens": 251
},
"gpqa_diamond:gpqa_diamond/recnGEpF1srQpaqWq#1": {
"requests": 1,
"prompt_tokens": 154,
"completion_tokens": 374,
"allowance_tokens": 32614,
"length_stops": 0,
"errors": 0,
"seconds": 2.478,
"budget_s": 2.26840820447489,
"last_prompt_tokens": 154
},
"gpqa_diamond:gpqa_diamond/recnTTKdBzfuoZ7w7#1": {
"requests": 1,
"prompt_tokens": 2421,
"completion_tokens": 76,
"allowance_tokens": 30347,
"length_stops": 0,
"errors": 0,
"seconds": 0.858,
"budget_s": 0.7731029883668921,
"last_prompt_tokens": 2421
},
"gpqa_diamond:gpqa_diamond/recnawZPumSCLMVgp#1": {
"requests": 1,
"prompt_tokens": 272,
"completion_tokens": 565,
"allowance_tokens": 32496,
"length_stops": 0,
"errors": 0,
"seconds": 3.774,
"budget_s": 3.4349945322378197,
"last_prompt_tokens": 272
},
"gpqa_diamond:gpqa_diamond/recnjViFrqlZNL3fY#1": {
"requests": 1,
"prompt_tokens": 130,
"completion_tokens": 2247,
"allowance_tokens": 32638,
"length_stops": 0,
"errors": 0,
"seconds": 14.876,
"budget_s": 13.525577101115516,
"last_prompt_tokens": 130
},
"gpqa_diamond:gpqa_diamond/reco8WSc71p8zAv4q#1": {
"requests": 1,
"prompt_tokens": 340,
"completion_tokens": 603,
"allowance_tokens": 32428,
"length_stops": 0,
"errors": 0,
"seconds": 4.026,
"budget_s": 3.6742335751820705,
"last_prompt_tokens": 340
},
"gpqa_diamond:gpqa_diamond/recoiTJPGUmzAkief#1": {
"requests": 1,
"prompt_tokens": 327,
"completion_tokens": 685,
"allowance_tokens": 32441,
"length_stops": 0,
"errors": 0,
"seconds": 4.563,
"budget_s": 4.1659488278036045,
"last_prompt_tokens": 327
},
"gpqa_diamond:gpqa_diamond/recooG6bivTUJpDBz#1": {
"requests": 1,
"prompt_tokens": 302,
"completion_tokens": 567,
"allowance_tokens": 32466,
"length_stops": 0,
"errors": 0,
"seconds": 3.782,
"budget_s": 3.4516291033519972,
"last_prompt_tokens": 302
},
"gpqa_diamond:gpqa_diamond/recoy9ZLBsc7HIRBy#1": {
"requests": 1,
"prompt_tokens": 254,
"completion_tokens": 2020,
"allowance_tokens": 32514,
"length_stops": 0,
"errors": 0,
"seconds": 13.399,
"budget_s": 12.188061008322476,
"last_prompt_tokens": 254
},
"gpqa_diamond:gpqa_diamond/recpJmTOXx6T48nDJ#1": {
"requests": 1,
"prompt_tokens": 142,
"completion_tokens": 1312,
"allowance_tokens": 32626,
"length_stops": 0,
"errors": 0,
"seconds": 8.668,
"budget_s": 7.906524418871081,
"last_prompt_tokens": 142
},
"gpqa_diamond:gpqa_diamond/recpki12iG9RUGrz9#1": {
"requests": 1,
"prompt_tokens": 336,
"completion_tokens": 1070,
"allowance_tokens": 32432,
"length_stops": 0,
"errors": 0,
"seconds": 7.117,
"budget_s": 6.485653897537208,
"last_prompt_tokens": 336
},
"gpqa_diamond:gpqa_diamond/recqFDvYKKRmbnoUb#1": {
"requests": 1,
"prompt_tokens": 195,
"completion_tokens": 1430,
"allowance_tokens": 32573,
"length_stops": 0,
"errors": 0,
"seconds": 9.47,
"budget_s": 8.626222940132559,
"last_prompt_tokens": 195
},
"gpqa_diamond:gpqa_diamond/recqGD3fxPCI59vPQ#1": {
"requests": 1,
"prompt_tokens": 233,
"completion_tokens": 1249,
"allowance_tokens": 32535,
"length_stops": 0,
"errors": 0,
"seconds": 8.28,
"budget_s": 7.544504937159195,
"last_prompt_tokens": 233
},
"gpqa_diamond:gpqa_diamond/recqLRu4BcJvlSANc#1": {
"requests": 1,
"prompt_tokens": 176,
"completion_tokens": 325,
"allowance_tokens": 32592,
"length_stops": 0,
"errors": 0,
"seconds": 2.164,
"budget_s": 1.9768897269736645,
"last_prompt_tokens": 176
},
"gpqa_diamond:gpqa_diamond/recr3VHM4zYf6dMFY#1": {
"requests": 1,
"prompt_tokens": 271,
"completion_tokens": 531,
"allowance_tokens": 32497,
"length_stops": 0,
"errors": 0,
"seconds": 3.539,
"budget_s": 3.230211528084481,
"last_prompt_tokens": 271
},
"gpqa_diamond:gpqa_diamond/recrHBEJJoDTV05JR#1": {
"requests": 1,
"prompt_tokens": 230,
"completion_tokens": 1185,
"allowance_tokens": 32538,
"length_stops": 0,
"errors": 0,
"seconds": 7.857,
"budget_s": 7.158886101585742,
"last_prompt_tokens": 230
},
"gpqa_diamond:gpqa_diamond/recrNbtgTNoabJJi6#1": {
"requests": 1,
"prompt_tokens": 104,
"completion_tokens": 431,
"allowance_tokens": 32664,
"length_stops": 0,
"errors": 0,
"seconds": 2.841,
"budget_s": 2.603810453285438,
"last_prompt_tokens": 104
},
"gpqa_diamond:gpqa_diamond/recrs0kUo7Ua1xhKO#1": {
"requests": 1,
"prompt_tokens": 159,
"completion_tokens": 596,
"allowance_tokens": 32609,
"length_stops": 0,
"errors": 0,
"seconds": 3.938,
"budget_s": 3.604106701866855,
"last_prompt_tokens": 159
},
"gpqa_diamond:gpqa_diamond/recs3PLPUEMiqg4P8#1": {
"requests": 1,
"prompt_tokens": 366,
"completion_tokens": 443,
"allowance_tokens": 32402,
"length_stops": 0,
"errors": 0,
"seconds": 2.981,
"budget_s": 2.7146019613162977,
"last_prompt_tokens": 366
},
"gpqa_diamond:gpqa_diamond/rect4iLrSfUwkNTno#1": {
"requests": 1,
"prompt_tokens": 178,
"completion_tokens": 836,
"allowance_tokens": 32590,
"length_stops": 0,
"errors": 0,
"seconds": 5.522,
"budget_s": 5.0504687821473855,
"last_prompt_tokens": 178
},
"gpqa_diamond:gpqa_diamond/rectXfsCM1dj4Kv2c#1": {
"requests": 1,
"prompt_tokens": 111,
"completion_tokens": 403,
"allowance_tokens": 32657,
"length_stops": 0,
"errors": 0,
"seconds": 2.658,
"budget_s": 2.4365251762326503,
"last_prompt_tokens": 111
},
"gpqa_diamond:gpqa_diamond/rectlyG9pCAAuWhoB#1": {
"requests": 1,
"prompt_tokens": 292,
"completion_tokens": 310,
"allowance_tokens": 32476,
"length_stops": 0,
"errors": 0,
"seconds": 2.085,
"budget_s": 1.903088799236492,
"last_prompt_tokens": 292
},
"gpqa_diamond:gpqa_diamond/recuYEkVjdVUwA3c0#1": {
"requests": 1,
"prompt_tokens": 142,
"completion_tokens": 1234,
"allowance_tokens": 32626,
"length_stops": 0,
"errors": 0,
"seconds": 8.15,
"budget_s": 7.437539122387904,
"last_prompt_tokens": 142
},
"gpqa_diamond:gpqa_diamond/recuyeuT5rQ6qDt8F#1": {
"requests": 1,
"prompt_tokens": 225,
"completion_tokens": 1466,
"allowance_tokens": 32543,
"length_stops": 0,
"errors": 0,
"seconds": 9.718,
"budget_s": 8.848593462178512,
"last_prompt_tokens": 225
},
"gpqa_diamond:gpqa_diamond/recv7GsQg3f0fvB1f#1": {
"requests": 1,
"prompt_tokens": 246,
"completion_tokens": 418,
"allowance_tokens": 32522,
"length_stops": 0,
"errors": 0,
"seconds": 2.791,
"budget_s": 2.5464575829061147,
"last_prompt_tokens": 246
},
"gpqa_diamond:gpqa_diamond/recvZlMbpPz0yLDle#1": {
"requests": 1,
"prompt_tokens": 246,
"completion_tokens": 620,
"allowance_tokens": 32522,
"length_stops": 0,
"errors": 0,
"seconds": 4.119,
"budget_s": 3.761968775426743,
"last_prompt_tokens": 246
},
"gpqa_diamond:gpqa_diamond/recvlnu8BuLmXZhSX#1": {
"requests": 1,
"prompt_tokens": 203,
"completion_tokens": 1067,
"allowance_tokens": 32565,
"length_stops": 0,
"errors": 0,
"seconds": 7.06,
"budget_s": 6.444167548216025,
"last_prompt_tokens": 203
},
"gpqa_diamond:gpqa_diamond/recw4rROcnHKNZhtK#1": {
"requests": 1,
"prompt_tokens": 176,
"completion_tokens": 561,
"allowance_tokens": 32592,
"length_stops": 0,
"errors": 0,
"seconds": 3.713,
"budget_s": 3.396237176842869,
"last_prompt_tokens": 176
},
"gpqa_diamond:gpqa_diamond/recwW1A85nfyQpReG#1": {
"requests": 1,
"prompt_tokens": 134,
"completion_tokens": 187,
"allowance_tokens": 32634,
"length_stops": 0,
"errors": 0,
"seconds": 1.258,
"budget_s": 1.1412266318030273,
"last_prompt_tokens": 134
},
"gpqa_diamond:gpqa_diamond/recwlwlcPvXVNPmvl#1": {
"requests": 1,
"prompt_tokens": 224,
"completion_tokens": 1251,
"allowance_tokens": 32544,
"length_stops": 0,
"errors": 0,
"seconds": 8.291,
"budget_s": 7.5548781728518195,
"last_prompt_tokens": 224
},
"gpqa_diamond:gpqa_diamond/recxTy0x7M6T9ikc0#1": {
"requests": 1,
"prompt_tokens": 478,
"completion_tokens": 510,
"allowance_tokens": 32290,
"length_stops": 0,
"errors": 0,
"seconds": 3.44,
"budget_s": 3.1350115201524433,
"last_prompt_tokens": 478
},
"gpqa_diamond:gpqa_diamond/recxsX66ktJ1Lwt4f#1": {
"requests": 1,
"prompt_tokens": 248,
"completion_tokens": 651,
"allowance_tokens": 32520,
"length_stops": 0,
"errors": 0,
"seconds": 4.329,
"budget_s": 3.948821765055081,
"last_prompt_tokens": 248
},
"gpqa_diamond:gpqa_diamond/recyPffUDqC2k7ZZO#1": {
"requests": 1,
"prompt_tokens": 416,
"completion_tokens": 880,
"allowance_tokens": 32352,
"length_stops": 0,
"errors": 0,
"seconds": 5.878,
"budget_s": 5.354978582114312,
"last_prompt_tokens": 416
},
"gpqa_diamond:gpqa_diamond/recyl3usDqb7ruXJx#1": {
"requests": 1,
"prompt_tokens": 249,
"completion_tokens": 462,
"allowance_tokens": 32519,
"length_stops": 0,
"errors": 0,
"seconds": 3.085,
"budget_s": 2.811667694252959,
"last_prompt_tokens": 249
},
"gpqa_diamond:gpqa_diamond/recypVp2NmPlBKVTp#1": {
"requests": 1,
"prompt_tokens": 214,
"completion_tokens": 308,
"allowance_tokens": 32554,
"length_stops": 0,
"errors": 0,
"seconds": 2.069,
"budget_s": 1.8800210973054463,
"last_prompt_tokens": 214
},
"gpqa_diamond:gpqa_diamond/recyrk6XBVwugG5u8#1": {
"requests": 1,
"prompt_tokens": 209,
"completion_tokens": 1282,
"allowance_tokens": 32559,
"length_stops": 0,
"errors": 0,
"seconds": 8.503,
"budget_s": 7.738597526634535,
"last_prompt_tokens": 209
},
"gpqa_diamond:gpqa_diamond/recywRj5a8EEjj2Ib#1": {
"requests": 1,
"prompt_tokens": 221,
"completion_tokens": 1184,
"allowance_tokens": 32547,
"length_stops": 0,
"errors": 0,
"seconds": 7.85,
"budget_s": 7.151236688758862,
"last_prompt_tokens": 221
},
"gpqa_diamond:gpqa_diamond/recz3MfYHPRhsEQkW#1": {
"requests": 1,
"prompt_tokens": 264,
"completion_tokens": 707,
"allowance_tokens": 32504,
"length_stops": 0,
"errors": 0,
"seconds": 4.7,
"budget_s": 4.288354458529043,
"last_prompt_tokens": 264
},
"gpqa_diamond:gpqa_diamond/reczQ4I0VpENdMtIj#1": {
"requests": 1,
"prompt_tokens": 212,
"completion_tokens": 483,
"allowance_tokens": 32556,
"length_stops": 0,
"errors": 0,
"seconds": 3.213,
"budget_s": 2.9325082941504235,
"last_prompt_tokens": 212
},
"gpqa_diamond:gpqa_diamond/reczUoM8JsxU6pYxr#1": {
"requests": 1,
"prompt_tokens": 218,
"completion_tokens": 1346,
"allowance_tokens": 32550,
"length_stops": 0,
"errors": 0,
"seconds": 8.933,
"budget_s": 8.125301057887235,
"last_prompt_tokens": 218
},
"gpqa_diamond:gpqa_diamond/reczjcMtrB1YGS2fO#1": {
"requests": 1,
"prompt_tokens": 201,
"completion_tokens": 954,
"allowance_tokens": 32567,
"length_stops": 0,
"errors": 0,
"seconds": 6.319,
"budget_s": 5.764083616242322,
"last_prompt_tokens": 201
},
"gpqa_diamond:gpqa_diamond/reczkBiPPNrNN49Hp#1": {
"requests": 1,
"prompt_tokens": 198,
"completion_tokens": 1685,
"allowance_tokens": 32570,
"length_stops": 0,
"errors": 0,
"seconds": 11.16,
"budget_s": 10.160673985524705,
"last_prompt_tokens": 198
},
"gpqa_diamond:gpqa_diamond/reczzzihL7btBH7RO#1": {
"requests": 1,
"prompt_tokens": 305,
"completion_tokens": 511,
"allowance_tokens": 32463,
"length_stops": 0,
"errors": 0,
"seconds": 3.421,
"budget_s": 3.114964370824171,
"last_prompt_tokens": 305
}
},
"proxy_count_mismatches": 0,
"parallel": 1,
"limited": false,
"launch_overrides": {},
"sessions": [
{
"started_at": "2026-09-16T21:46:56Z",
"seconds": 1275.7
},
{
"started_at": "2026-09-16T21:55:20Z",
"seconds": 1318.3,
"lab_version": "0.1.0+a29cd5a",
"throttled": false
},
{
"started_at": "2026-09-17T16:14:07Z",
"seconds": 1032.5,
"lab_version": "0.1.0+ba2d048",
"throttled": false
}
],
"progress": {
"aime_2025": {
"done": 30,
"planned": 30
},
"bfcl": {
"done": 400,
"planned": 400
},
"livecodebench": {
"done": 100,
"planned": 100
},
"gpqa_diamond": {
"done": 198,
"planned": 198
}
},
"telemetry_summary": {
"vram_baseline_mb": 0.9375,
"vram_peak_mb": 3303.125,
"ram_baseline_mb": 22150.17578125,
"ram_peak_mb": 22572.359375,
"proc_rss_peak_mb": 1128.0234375,
"temp_max_c": 66,
"power_max_w": 249.936,
"clock_max_mhz": 1620,
"throttled": false,
"power_capped_frac": 0.9807692307692307,
"phases": {
"aime_2025": {
"duration_s": 0.006402631057426333,
"energy_j": 0,
"power_avg_w": null
},
"bfcl": {
"duration_s": 0.004097971017472446,
"energy_j": 0,
"power_avg_w": null
},
"livecodebench": {
"duration_s": 0.0033316239714622498,
"energy_j": 0,
"power_avg_w": null
},
"gpqa_diamond": {
"duration_s": 1029.8578153699636,
"energy_j": 255395.133,
"power_avg_w": 247.99067326420442
}
}
},
"benchmarks": [
"aime_2025",
"bfcl",
"livecodebench",
"gpqa_diamond"
],
"sandbox_images": {
"bfcl": {
"tag": "lab-bfcl:62e1f0090a9a",
"image_id": "sha256:6d68e65b0bf15aed74b4ce9fe84b03b10747c304cbdf50804122f5a7f94d2442",
"host": "alex@web"
},
"livecodebench": {
"tag": "lab-livecodebench:b4d2e5fd3081",
"image_id": "sha256:bf9fe62701b4781138c193ec2ef79dd7adb0caea68bed7e9044be645841dbaf9",
"host": "alex@web"
}
},
"harness_options": {
"bfcl": {
"mode": "prompting"
},
"livecodebench": {}
}
}