From da5105e17870571ad7b1389b7d46f806cfb84bd9 Mon Sep 17 00:00:00 2001 From: Peng Cheng Date: Sat, 7 Nov 2026 16:49:26 +0800 Subject: [PATCH] bench: compare inference runtimes on a realtime caption eval set MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a 60-case eval set that mimics what on-device ASR actually emits (short turns, unfinished drafts, no punctuation, fillers, numbers, names, zh->en, context follow-ups) and a runner that scores any Ollama or OpenAI-compatible server on streaming TTFT / end-to-end latency, generation speed, instruction compliance, term preservation and chrF. Results on an M2 Pro 16 GB with the same Qwen3.5-4B weights: llama.cpp (Metal, Q4_K_M) and Ollama (MLX runner, nvfp4) are within noise of each other (p50 end-to-end 542 vs 534 ms, TTFT 146 vs 160 ms, chrF 55.8 vs 57.6, terms kept 97% vs 92%). The Qwen2.5-1.5B model that PR #27 bundles is 3x faster but drops to 90% compliance, 89% terms kept and chrF 47.7: it echoes inputs untranslated, turns 38% into 3.8% and 五十万 into fifty thousand. Co-Authored-By: Claude Fable 5.1 --- benchmark/engine_compare_report.md | 114 + benchmark/engine_compare_results.json | 10203 ++++++++++++++++++++++++ benchmark/realtime_eval_set.json | 548 ++ benchmark/run_engine_compare.py | 331 + 4 files changed, 11196 insertions(+) create mode 100644 benchmark/engine_compare_report.md create mode 100644 benchmark/engine_compare_results.json create mode 100644 benchmark/realtime_eval_set.json create mode 100644 benchmark/run_engine_compare.py diff --git a/benchmark/engine_compare_report.md b/benchmark/engine_compare_report.md new file mode 100644 index 0000000..3262f5b --- /dev/null +++ b/benchmark/engine_compare_report.md @@ -0,0 +1,114 @@ +# LCT engine comparison: same weights, different runtimes + +Generated 2026-11-07 16:48. Eval set `realtime_eval_set.json` (60 cases × 2 runs), temperature 0.3, num_ctx 4096, max_tokens 256, thinking off, streaming; latency measured client-side from request start. + +Hardware: Apple M2 Pro, 16 GB, macOS 27.2. Ollama 0.40.14 (Homebrew formula, MLX runner; both `qwen3.5:4b-mlx` and `qwen3.5:4b` tags are the same safetensors/nvfp4 4.5B weights on the MLX runner in this version, so the two Ollama columns are a repeat run, not a different backend). llama.cpp build 12110 via `llama-server -c 4096 -ngl 99 -fa on -t 6 --jinja` with Metal; weights `unsloth/Qwen3.5-4B-GGUF` Q4_K_M (2.74 GB on disk, 0.95 GB phys footprint + 2.6 GB GPU) and, for the PR #27 column, the exact `Qwen/Qwen2.5-1.5B-Instruct-GGUF` q4_k_m file that PR #27 bundles. Ollama resident memory at num_ctx 4096: 4.1 GB (7.6 GB at its 32K default). Note PR #27's in-app runtime uses n_ctx 2048, temperature 0 and at most 6 threads; this benchmark runs all engines at the same 4096 / 0.3 settings. + +## Overall + +| Metric | ollama-mlx | ollama-4b-tag | llamacpp-metal | llamacpp-qwen2.5-1.5b(PR27) | +|---|---:|---:|---:|---:| +| Cases scored | 120 | 120 | 120 | 120 | +| TTFT avg | 209 ms | 225 ms | 175 ms | 42 ms | +| TTFT p50 | 160 ms | 162 ms | 146 ms | 40 ms | +| TTFT p95 | 489 ms | 537 ms | 353 ms | 81 ms | +| End-to-end avg | 568 ms | 593 ms | 569 ms | 193 ms | +| End-to-end p50 | 534 ms | 565 ms | 542 ms | 172 ms | +| End-to-end p95 | 1086 ms | 1260 ms | 1062 ms | 401 ms | +| Generation speed avg | 41.4 tok/s | 41.0 tok/s | 42.1 tok/s | 107.3 tok/s | +| Instruction compliance | 98% | 98% | 98% | 90% | +| Numbers / names kept | 92% | 92% | 97% | 89% | +| Over-long outputs (drafts completed, commentary) | 0 | 0 | 0 | 0 | +| chrF vs reference | 57.6 | 55.3 | 55.8 | 47.7 | + +## By category (end-to-end p50 / chrF) + +| Category | ollama-mlx | ollama-4b-tag | llamacpp-metal | llamacpp-qwen2.5-1.5b(PR27) | +|---|---:|---:|---:|---:| +| A_short_turns — 短对话轮次，会议里最常见 | 288 ms / 60 | 280 ms / 49 | 316 ms / 46 | 97 ms / 37 | +| B_asr_drafts — ASR 草稿：句子说到一半就被切出来，模型不能脑补结尾 | 261 ms / 53 | 260 ms / 51 | 308 ms / 52 | 78 ms / 54 | +| C_asr_noisy — 无标点、全小写、口头禅、重复词，模拟真实识别输出 | 404 ms / 40 | 420 ms / 38 | 448 ms / 33 | 152 ms / 25 | +| D_numbers_names — 数字、日期、货币、人名公司名必须原样保留 | 692 ms / 73 | 633 ms / 73 | 710 ms / 68 | 220 ms / 53 | +| E_tech_terms — 技术/产品术语，保留英文或用标准译法 | 584 ms / 67 | 626 ms / 66 | 574 ms / 66 | 245 ms / 59 | +| F_long_sentences — 讲座/会议里的长句，30-45 词 | 981 ms / 53 | 1210 ms / 52 | 1054 ms / 53 | 410 ms / 45 | +| G_zh_to_en — 中文会议发言译成英文，含口语和数字 | 689 ms / 65 | 730 ms / 64 | 610 ms / 69 | 232 ms / 60 | +| H_context_followup — 依赖上文的代词和省略，带 2 条上下文历史 | 521 ms / 46 | 556 ms / 46 | 503 ms / 60 | 154 ms / 50 | + +## Problems + +- **ollama-mlx**: preamble:好的 ×2; missing terms (run 0): [('D05', ['60', '40']), ('E07', ['Developer ID']), ('F06', ['SSO']), ('H03', ['Apple Silicon']), ('H04', ['Intel'])] +- **ollama-4b-tag**: preamble:好的 ×2; missing terms (run 0): [('E07', ['Developer ID']), ('F06', ['SSO']), ('H03', ['Apple Silicon']), ('H04', ['Intel'])] +- **llamacpp-metal**: preamble:好的 ×2; missing terms (run 0): [('D08', ['NPS']), ('E07', ['Developer ID'])] +- **llamacpp-qwen2.5-1.5b(PR27)**: wrong-script ×5, preamble:好的 ×4, echo ×3, preamble:The translation ×2, preamble:注： ×1; missing terms (run 0): [('D03', ['p99']), ('D06', ['Nakamura/中村']), ('E07', ['Developer ID']), ('G01', ['Wednesday']), ('G03', ['38', '45']), ('G05', ['Zhang', '500/500,000/half a million'])] + +## Side-by-side samples (run 0) + +**B01** `so what we found was that the` +ref: 所以我们发现的是 +- ollama-mlx (252 ms): 所以我们发现的是…… +- ollama-4b-tag (277 ms): 所以，我们发现的是…… +- llamacpp-metal (336 ms): 所以，我们发现的是 +- llamacpp-qwen2.5-1.5b(PR27) (82 ms): 我们发现的情况是： + +**B06** `we need to decide by` +ref: 我们需要在……之前决定 +- ollama-mlx (311 ms): 我们需要做出决定。 +- ollama-4b-tag (251 ms): 我们需要做出决定。 +- llamacpp-metal (307 ms): 我们需要在...做出决定。 +- llamacpp-qwen2.5-1.5b(PR27) (56 ms): 我们需要决定。 + +**C01** `um so yeah basically what we're gonna do is we're gonna ship it next week` +ref: 所以基本上我们要做的就是下周发布。 +- ollama-mlx (447 ms): 嗯，所以基本上我们要做的事情是，我们下周发货。 +- ollama-4b-tag (441 ms): 嗯，所以基本上我们要做的就是把货物下周发货。 +- llamacpp-metal (392 ms): 所以基本上，我们下周就发货。 +- llamacpp-qwen2.5-1.5b(PR27) (172 ms): 我们下周会发货。 + +**D01** `Revenue grew 12% year over year to $4.3 billion.` +ref: 收入同比增长12%，达到43亿美元。 +- ollama-mlx (582 ms): 收入同比增长 12%，达到 43 亿美元。 +- ollama-4b-tag (465 ms): 收入同比增长 12%，达到 43 亿美元。 +- llamacpp-metal (492 ms): 收入同比增长 12%，达到 43 亿美元。 +- llamacpp-qwen2.5-1.5b(PR27) (141 ms): 营业收入同比增长12%至43亿元。 + +**D03** `We processed 2.7 million requests with a p99 latency of 180 milliseconds.` +ref: 我们处理了270万个请求，p99延迟为180毫秒。 +- ollama-mlx (767 ms): 我们处理了 270 万次请求，p99 延迟为 180 毫秒。 +- ollama-4b-tag (805 ms): 我们处理了 270 万请求，p99 延迟为 180 毫秒。 +- llamacpp-metal (876 ms): 我们处理了 270 万次请求，p99 延迟为 180 毫秒。 +- llamacpp-qwen2.5-1.5b(PR27) (261 ms): 我们处理了270万请求，99百分位的延迟为180毫秒。 + +**D05** `The budget is 850 thousand euros, split 60-40 between the two teams.` +ref: 预算是85万欧元，两个团队按60比40分配。 +- ollama-mlx (697 ms): 预算为 85 万欧元，两个团队按 6:4 比例分配。 +- ollama-4b-tag (732 ms): 预算为 85 万欧元，两个团队按 60:40 分配。 +- llamacpp-metal (873 ms): 预算为 85 万欧元，在两个团队之间按 60 比 40 分配。 +- llamacpp-qwen2.5-1.5b(PR27) (234 ms): 预算为85万欧元，两队各分60%和40%。 + +**G03** `上个季度我们的留存率从百分之三十八涨到了百分之四十五` +ref: Last quarter our retention rate went from 38% to 45%. +- ollama-mlx (603 ms): Last quarter, our retention rate rose from 38 percent to 45 percent. +- ollama-4b-tag (605 ms): Last quarter, our retention rate rose from 38 percent to 45 percent. +- llamacpp-metal (589 ms): Last quarter, our retention rate rose from 38 percent to 45 percent. +- llamacpp-qwen2.5-1.5b(PR27) (235 ms): Last quarter, our retention rate increased from 3.8% to 4.5%. + +**G05** `张总说预算最多给到五十万，超了要重新审批` +ref: Mr. Zhang said the budget is capped at 500 thousand; anything above that needs re-approval. +- ollama-mlx (778 ms): Mr. Zhang says the budget cap is 500,000; anything exceeding that requires re-approval. +- ollama-4b-tag (730 ms): Mr. Zhang said the budget cap is 500,000; any excess requires re-approval. +- llamacpp-metal (788 ms): Mr. Zhang said the budget is capped at 500,000; anything exceeding that requires re-approval. +- llamacpp-qwen2.5-1.5b(PR27) (277 ms): The CEO said the budget can be given up to fifty thousand, and exceeding it will require a re-approval. + +**H02** `It was about twice as fast on first token.` +ref: 它的首字速度大约快了一倍。 +- ollama-mlx (479 ms): 首词速度大约快了一倍。 +- ollama-4b-tag (456 ms): 首词生成速度大约快了一倍。 +- llamacpp-metal (454 ms): 首字生成速度大约快了两倍。 +- llamacpp-qwen2.5-1.5b(PR27) (133 ms): 第一次 token 上它快了两倍。 + +**F03** `If the benchmark shows that the embedded runtime is within twenty percent of the Ollama path on first-token latency, I'm comfortable making it the default and keeping Ollama as an advanced option.` +ref: 如果基准测试显示内置运行时的首字延迟与Ollama路径相差在百分之二十以内，我可以接受把它设为默认，并把Ollama保留为高级选项。 +- ollama-mlx (1225 ms): 如果基准测试显示，在首令牌延迟方面，嵌入式运行时处于 Ollama 路径的百分之二十以内，我就可以将其设为默认选项，并将 Ollama 保留为高级选项。 +- ollama-4b-tag (1259 ms): 如果基准测试显示，在首字延迟方面，嵌入式运行时处于 Ollama 路径的百分之二十以内，我就可以将其设为默认选项，并将 Ollama 保留为高级选项。 +- llamacpp-metal (1250 ms): 如果基准测试显示，在首字延迟方面，嵌入式运行时距离 Ollama 路径在二十个百分点以内，那么我乐意将其设为默认选项，并将 Ollama 保留为高级选项。 +- llamacpp-qwen2.5-1.5b(PR27) (474 ms): 如果基准显示嵌入式运行时在首次令牌延迟上与Ollama路径的误差不超过20%，我有信心将其设置为默认选项，同时保留Ollama作为高级选项。 diff --git a/benchmark/engine_compare_results.json b/benchmark/engine_compare_results.json new file mode 100644 index 0000000..501114f --- /dev/null +++ b/benchmark/engine_compare_results.json @@ -0,0 +1,10203 @@ +{ + "generated": "2026-11-07T16:48:09.232951", + "config": { + "engine": [], + "report_only": true, + "data": "realtime_eval_set.json", + "categories": null, + "runs": 2, + "output": "engine_compare_results.json", + "report": "engine_compare_report.md", + "append": false, + "note": "Hardware: Apple M2 Pro, 16 GB, macOS 27.2. Ollama 0.40.14 (Homebrew formula, MLX runner; both `qwen3.5:4b-mlx` and `qwen3.5:4b` tags are the same safetensors/nvfp4 4.5B weights on the MLX runner in this version, so the two Ollama columns are a repeat run, not a different backend). llama.cpp build 12110 via `llama-server -c 4096 -ngl 99 -fa on -t 6 --jinja` with Metal; weights `unsloth/Qwen3.5-4B-GGUF` Q4_K_M (2.74 GB on disk, 0.95 GB phys footprint + 2.6 GB GPU) and, for the PR #27 column, the exact `Qwen/Qwen2.5-1.5B-Instruct-GGUF` q4_k_m file that PR #27 bundles. Ollama resident memory at num_ctx 4096: 4.1 GB (7.6 GB at its 32K default). Note PR #27's in-app runtime uses n_ctx 2048, temperature 0 and at most 6 threads; this benchmark runs all engines at the same 4096 / 0.3 settings." + }, + "results": [ + { + "engine": "ollama-mlx", + "category": "A_short_turns", + "id": "A01", + "run": 0, + "input": "Let's get started.", + "reference": "我们开始吧。", + "ttft_ms": 764, + "e2e_ms": 855, + "gen_tokens": 4, + "tps": 44.0, + "prompt_tokens": 131, + "prompt_ms": 696, + "output": "我们开始吧。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 1.0 + }, + { + "engine": "ollama-mlx", + "category": "A_short_turns", + "id": "A02", + "run": 0, + "input": "Can everyone see my screen?", + "reference": "大家能看到我的屏幕吗？", + "ttft_ms": 546, + "e2e_ms": 685, + "gen_tokens": 6, + "tps": 43.3, + "prompt_tokens": 132, + "prompt_ms": 495, + "output": "大家能看到我的屏幕吗？", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 1.0 + }, + { + "engine": "ollama-mlx", + "category": "A_short_turns", + "id": "A03", + "run": 0, + "input": "Sorry, you cut out for a second.", + "reference": "抱歉，你刚才断了一下。", + "ttft_ms": 161, + "e2e_ms": 304, + "gen_tokens": 6, + "tps": 42.0, + "prompt_tokens": 135, + "prompt_ms": 103, + "output": "抱歉，您中断了一下。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.516 + }, + { + "engine": "ollama-mlx", + "category": "A_short_turns", + "id": "A04", + "run": 0, + "input": "I'll take that offline.", + "reference": "这个我们线下再聊。", + "ttft_ms": 155, + "e2e_ms": 276, + "gen_tokens": 5, + "tps": 41.2, + "prompt_tokens": 132, + "prompt_ms": 98, + "output": "我把它离线了。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.087 + }, + { + "engine": "ollama-mlx", + "category": "A_short_turns", + "id": "A05", + "run": 0, + "input": "Any questions before we move on?", + "reference": "在继续之前有什么问题吗？", + "ttft_ms": 163, + "e2e_ms": 352, + "gen_tokens": 8, + "tps": 42.4, + "prompt_tokens": 133, + "prompt_ms": 99, + "output": "在我们继续之前，有任何问题吗？", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.474 + }, + { + "engine": "ollama-mlx", + "category": "A_short_turns", + "id": "A06", + "run": 0, + "input": "That makes sense, thanks for clarifying.", + "reference": "明白了，谢谢你的说明。", + "ttft_ms": 159, + "e2e_ms": 346, + "gen_tokens": 8, + "tps": 42.7, + "prompt_tokens": 135, + "prompt_ms": 100, + "output": "这说得通，谢谢您的澄清。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.21 + }, + { + "engine": "ollama-mlx", + "category": "A_short_turns", + "id": "A07", + "run": 0, + "input": "We're running about five minutes behind.", + "reference": "我们大概晚了五分钟。", + "ttft_ms": 155, + "e2e_ms": 275, + "gen_tokens": 5, + "tps": 41.6, + "prompt_tokens": 134, + "prompt_ms": 99, + "output": "我们落后大约五分钟。", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 1, + "terms_missing": [], + "chrf": 0.384 + }, + { + "engine": "ollama-mlx", + "category": "A_short_turns", + "id": "A08", + "run": 0, + "input": "Let me share the link in the chat.", + "reference": "我把链接发到聊天里。", + "ttft_ms": 157, + "e2e_ms": 328, + "gen_tokens": 7, + "tps": 41.0, + "prompt_tokens": 135, + "prompt_ms": 100, + "output": "让我把链接发到聊天里。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.977 + }, + { + "engine": "ollama-mlx", + "category": "A_short_turns", + "id": "A01", + "run": 1, + "input": "Let's get started.", + "reference": "我们开始吧。", + "ttft_ms": 88, + "e2e_ms": 181, + "gen_tokens": 4, + "tps": 43.1, + "prompt_tokens": 131, + "prompt_ms": 33, + "output": "让我们开始吧。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.955 + }, + { + "engine": "ollama-mlx", + "category": "A_short_turns", + "id": "A02", + "run": 1, + "input": "Can everyone see my screen?", + "reference": "大家能看到我的屏幕吗？", + "ttft_ms": 92, + "e2e_ms": 236, + "gen_tokens": 6, + "tps": 41.7, + "prompt_tokens": 132, + "prompt_ms": 36, + "output": "大家能看到我的屏幕吗？", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 1.0 + }, + { + "engine": "ollama-mlx", + "category": "A_short_turns", + "id": "A03", + "run": 1, + "input": "Sorry, you cut out for a second.", + "reference": "抱歉，你刚才断了一下。", + "ttft_ms": 85, + "e2e_ms": 224, + "gen_tokens": 6, + "tps": 43.1, + "prompt_tokens": 135, + "prompt_ms": 31, + "output": "抱歉，您中断了一下。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.516 + }, + { + "engine": "ollama-mlx", + "category": "A_short_turns", + "id": "A04", + "run": 1, + "input": "I'll take that offline.", + "reference": "这个我们线下再聊。", + "ttft_ms": 87, + "e2e_ms": 205, + "gen_tokens": 5, + "tps": 42.4, + "prompt_tokens": 132, + "prompt_ms": 32, + "output": "我把它放下来了。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.085 + }, + { + "engine": "ollama-mlx", + "category": "A_short_turns", + "id": "A05", + "run": 1, + "input": "Any questions before we move on?", + "reference": "在继续之前有什么问题吗？", + "ttft_ms": 127, + "e2e_ms": 300, + "gen_tokens": 7, + "tps": 40.5, + "prompt_tokens": 133, + "prompt_ms": 71, + "output": "在我们继续之前，有什么问题吗？", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.727 + }, + { + "engine": "ollama-mlx", + "category": "A_short_turns", + "id": "A06", + "run": 1, + "input": "That makes sense, thanks for clarifying.", + "reference": "明白了，谢谢你的说明。", + "ttft_ms": 144, + "e2e_ms": 314, + "gen_tokens": 7, + "tps": 41.3, + "prompt_tokens": 135, + "prompt_ms": 75, + "output": "有道理，谢谢你的澄清。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.382 + }, + { + "engine": "ollama-mlx", + "category": "A_short_turns", + "id": "A07", + "run": 1, + "input": "We're running about five minutes behind.", + "reference": "我们大概晚了五分钟。", + "ttft_ms": 88, + "e2e_ms": 204, + "gen_tokens": 5, + "tps": 43.2, + "prompt_tokens": 134, + "prompt_ms": 30, + "output": "我们落后大约五分钟。", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 1, + "terms_missing": [], + "chrf": 0.384 + }, + { + "engine": "ollama-mlx", + "category": "A_short_turns", + "id": "A08", + "run": 1, + "input": "Let me share the link in the chat.", + "reference": "我把链接发到聊天里。", + "ttft_ms": 89, + "e2e_ms": 258, + "gen_tokens": 7, + "tps": 41.3, + "prompt_tokens": 135, + "prompt_ms": 31, + "output": "让我把链接发到聊天里。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.977 + }, + { + "engine": "ollama-mlx", + "category": "B_asr_drafts", + "id": "B01", + "run": 0, + "input": "so what we found was that the", + "reference": "所以我们发现的是", + "ttft_ms": 154, + "e2e_ms": 252, + "gen_tokens": 4, + "tps": 40.8, + "prompt_tokens": 133, + "prompt_ms": 98, + "output": "所以我们发现的是……", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.941 + }, + { + "engine": "ollama-mlx", + "category": "B_asr_drafts", + "id": "B02", + "run": 0, + "input": "if you look at the second chart you can see", + "reference": "如果你看第二张图，你可以看到", + "ttft_ms": 154, + "e2e_ms": 346, + "gen_tokens": 8, + "tps": 41.5, + "prompt_tokens": 136, + "prompt_ms": 101, + "output": "如果您看第二个图表，您可以看到。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.385 + }, + { + "engine": "ollama-mlx", + "category": "B_asr_drafts", + "id": "B03", + "run": 0, + "input": "the main reason we chose this approach is", + "reference": "我们选择这个方案的主要原因是", + "ttft_ms": 151, + "e2e_ms": 296, + "gen_tokens": 6, + "tps": 41.5, + "prompt_tokens": 134, + "prompt_ms": 98, + "output": "我们选择这一方法的主要原因是", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.647 + }, + { + "engine": "ollama-mlx", + "category": "B_asr_drafts", + "id": "B04", + "run": 0, + "input": "and then the customer said that they", + "reference": "然后客户说他们", + "ttft_ms": 160, + "e2e_ms": 279, + "gen_tokens": 5, + "tps": 41.9, + "prompt_tokens": 133, + "prompt_ms": 103, + "output": "然后客户说他们……", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.93 + }, + { + "engine": "ollama-mlx", + "category": "B_asr_drafts", + "id": "B05", + "run": 0, + "input": "I think the biggest risk here", + "reference": "我觉得这里最大的风险", + "ttft_ms": 153, + "e2e_ms": 298, + "gen_tokens": 6, + "tps": 41.4, + "prompt_tokens": 132, + "prompt_ms": 100, + "output": "我认为这里最大的风险是……", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.622 + }, + { + "engine": "ollama-mlx", + "category": "B_asr_drafts", + "id": "B06", + "run": 0, + "input": "we need to decide by", + "reference": "我们需要在……之前决定", + "ttft_ms": 221, + "e2e_ms": 311, + "gen_tokens": 4, + "tps": 44.5, + "prompt_tokens": 131, + "prompt_ms": 168, + "output": "我们需要做出决定。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.337 + }, + { + "engine": "ollama-mlx", + "category": "B_asr_drafts", + "id": "B07", + "run": 0, + "input": "which brings me to", + "reference": "这就引出了", + "ttft_ms": 151, + "e2e_ms": 270, + "gen_tokens": 5, + "tps": 41.7, + "prompt_tokens": 130, + "prompt_ms": 95, + "output": "这带我到了……", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.093 + }, + { + "engine": "ollama-mlx", + "category": "B_asr_drafts", + "id": "B08", + "run": 0, + "input": "the data shows a clear", + "reference": "数据显示出明显的", + "ttft_ms": 156, + "e2e_ms": 279, + "gen_tokens": 5, + "tps": 40.6, + "prompt_tokens": 131, + "prompt_ms": 100, + "output": "数据显示，情况明显。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.438 + }, + { + "engine": "ollama-mlx", + "category": "B_asr_drafts", + "id": "B01", + "run": 1, + "input": "so what we found was that the", + "reference": "所以我们发现的是", + "ttft_ms": 91, + "e2e_ms": 218, + "gen_tokens": 5, + "tps": 39.3, + "prompt_tokens": 133, + "prompt_ms": 36, + "output": "所以，我们发现的是……", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.716 + }, + { + "engine": "ollama-mlx", + "category": "B_asr_drafts", + "id": "B02", + "run": 1, + "input": "if you look at the second chart you can see", + "reference": "如果你看第二张图，你可以看到", + "ttft_ms": 96, + "e2e_ms": 287, + "gen_tokens": 8, + "tps": 41.8, + "prompt_tokens": 136, + "prompt_ms": 44, + "output": "如果您查看第二个图表，您可以看到。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.38 + }, + { + "engine": "ollama-mlx", + "category": "B_asr_drafts", + "id": "B03", + "run": 1, + "input": "the main reason we chose this approach is", + "reference": "我们选择这个方案的主要原因是", + "ttft_ms": 112, + "e2e_ms": 213, + "gen_tokens": 4, + "tps": 39.9, + "prompt_tokens": 134, + "prompt_ms": 48, + "output": "我们选择这种方法的主要原因", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.575 + }, + { + "engine": "ollama-mlx", + "category": "B_asr_drafts", + "id": "B04", + "run": 1, + "input": "and then the customer said that they", + "reference": "然后客户说他们", + "ttft_ms": 116, + "e2e_ms": 232, + "gen_tokens": 5, + "tps": 43.1, + "prompt_tokens": 133, + "prompt_ms": 30, + "output": "然后客户说他们……", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.93 + }, + { + "engine": "ollama-mlx", + "category": "B_asr_drafts", + "id": "B05", + "run": 1, + "input": "I think the biggest risk here", + "reference": "我觉得这里最大的风险", + "ttft_ms": 92, + "e2e_ms": 236, + "gen_tokens": 6, + "tps": 41.6, + "prompt_tokens": 132, + "prompt_ms": 32, + "output": "我认为这里最大的风险是……", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.622 + }, + { + "engine": "ollama-mlx", + "category": "B_asr_drafts", + "id": "B06", + "run": 1, + "input": "we need to decide by", + "reference": "我们需要在……之前决定", + "ttft_ms": 87, + "e2e_ms": 182, + "gen_tokens": 4, + "tps": 42.3, + "prompt_tokens": 131, + "prompt_ms": 31, + "output": "我们需要做出决定。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.337 + }, + { + "engine": "ollama-mlx", + "category": "B_asr_drafts", + "id": "B07", + "run": 1, + "input": "which brings me to", + "reference": "这就引出了", + "ttft_ms": 85, + "e2e_ms": 180, + "gen_tokens": 4, + "tps": 42.1, + "prompt_tokens": 130, + "prompt_ms": 29, + "output": "这使我联想到……", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.045 + }, + { + "engine": "ollama-mlx", + "category": "B_asr_drafts", + "id": "B08", + "run": 1, + "input": "the data shows a clear", + "reference": "数据显示出明显的", + "ttft_ms": 86, + "e2e_ms": 206, + "gen_tokens": 5, + "tps": 41.9, + "prompt_tokens": 131, + "prompt_ms": 31, + "output": "数据显示，情况明显。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.438 + }, + { + "engine": "ollama-mlx", + "category": "C_asr_noisy", + "id": "C01", + "run": 0, + "input": "um so yeah basically what we're gonna do is we're gonna ship it next week", + "reference": "所以基本上我们要做的就是下周发布。", + "ttft_ms": 157, + "e2e_ms": 447, + "gen_tokens": 12, + "tps": 41.4, + "prompt_tokens": 143, + "prompt_ms": 102, + "output": "嗯，所以基本上我们要做的事情是，我们下周发货。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.62 + }, + { + "engine": "ollama-mlx", + "category": "C_asr_noisy", + "id": "C02", + "run": 0, + "input": "i mean it's it's not perfect but it works you know", + "reference": "我是说它不完美，但能用，你懂的。", + "ttft_ms": 154, + "e2e_ms": 492, + "gen_tokens": 14, + "tps": 41.5, + "prompt_tokens": 139, + "prompt_ms": 101, + "output": "我的意思是，它不完美，但它能起作用，你懂的。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.61 + }, + { + "engine": "ollama-mlx", + "category": "C_asr_noisy", + "id": "C03", + "run": 0, + "input": "okay okay let me let me just pull that up real quick", + "reference": "好好，让我快速调出来看一下。", + "ttft_ms": 161, + "e2e_ms": 372, + "gen_tokens": 9, + "tps": 42.7, + "prompt_tokens": 138, + "prompt_ms": 105, + "output": "好的，好的，让我快速查一下。", + "compliant": false, + "problems": [ + "preamble:好的" + ], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.423 + }, + { + "engine": "ollama-mlx", + "category": "C_asr_noisy", + "id": "C04", + "run": 0, + "input": "right so the the thing is we don't actually have the budget for that", + "reference": "对，问题是我们其实没有这方面的预算。", + "ttft_ms": 154, + "e2e_ms": 343, + "gen_tokens": 8, + "tps": 42.5, + "prompt_tokens": 141, + "prompt_ms": 100, + "output": "所以，我们实际上没有那个预算。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.207 + }, + { + "engine": "ollama-mlx", + "category": "C_asr_noisy", + "id": "C05", + "run": 0, + "input": "like honestly I don't think anyone has looked at this since like march", + "reference": "说实话我觉得从三月以来就没人看过这个了。", + "ttft_ms": 159, + "e2e_ms": 400, + "gen_tokens": 10, + "tps": 41.4, + "prompt_tokens": 140, + "prompt_ms": 101, + "output": "说实话，我觉得自三月以来没人再看过了。", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 1, + "terms_missing": [], + "chrf": 0.406 + }, + { + "engine": "ollama-mlx", + "category": "C_asr_noisy", + "id": "C06", + "run": 0, + "input": "and uh we also need to loop in the legal team before we sign anything", + "reference": "另外在签任何东西之前我们还需要让法务团队参与进来。", + "ttft_ms": 180, + "e2e_ms": 495, + "gen_tokens": 13, + "tps": 41.2, + "prompt_tokens": 141, + "prompt_ms": 116, + "output": "而且我们还需要在签署任何文件之前让法务团队介入。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.399 + }, + { + "engine": "ollama-mlx", + "category": "C_asr_noisy", + "id": "C07", + "run": 0, + "input": "yeah no that's fair that's totally fair", + "reference": "对，没错，这很合理。", + "ttft_ms": 192, + "e2e_ms": 416, + "gen_tokens": 9, + "tps": 40.1, + "prompt_tokens": 135, + "prompt_ms": 112, + "output": "是啊，那很公平，完全没问题。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.116 + }, + { + "engine": "ollama-mlx", + "category": "C_asr_noisy", + "id": "C08", + "run": 0, + "input": "so hold on are we saying the deadline moved or not", + "reference": "等等，我们是说截止日期改了还是没改？", + "ttft_ms": 182, + "e2e_ms": 409, + "gen_tokens": 9, + "tps": 39.7, + "prompt_tokens": 137, + "prompt_ms": 130, + "output": "所以我们要说的是截止日期变了还是没变？", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.375 + }, + { + "engine": "ollama-mlx", + "category": "C_asr_noisy", + "id": "C01", + "run": 1, + "input": "um so yeah basically what we're gonna do is we're gonna ship it next week", + "reference": "所以基本上我们要做的就是下周发布。", + "ttft_ms": 272, + "e2e_ms": 535, + "gen_tokens": 11, + "tps": 41.9, + "prompt_tokens": 143, + "prompt_ms": 213, + "output": "嗯，所以基本上我们要做的就是把货物下周发货。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.735 + }, + { + "engine": "ollama-mlx", + "category": "C_asr_noisy", + "id": "C02", + "run": 1, + "input": "i mean it's it's not perfect but it works you know", + "reference": "我是说它不完美，但能用，你懂的。", + "ttft_ms": 93, + "e2e_ms": 438, + "gen_tokens": 14, + "tps": 40.6, + "prompt_tokens": 139, + "prompt_ms": 37, + "output": "我的意思是，它不完美，但它是能用的，你知道。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.43 + }, + { + "engine": "ollama-mlx", + "category": "C_asr_noisy", + "id": "C03", + "run": 1, + "input": "okay okay let me let me just pull that up real quick", + "reference": "好好，让我快速调出来看一下。", + "ttft_ms": 106, + "e2e_ms": 331, + "gen_tokens": 9, + "tps": 40.1, + "prompt_tokens": 138, + "prompt_ms": 39, + "output": "好的，好的，让我快速查一下。", + "compliant": false, + "problems": [ + "preamble:好的" + ], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.423 + }, + { + "engine": "ollama-mlx", + "category": "C_asr_noisy", + "id": "C04", + "run": 1, + "input": "right so the the thing is we don't actually have the budget for that", + "reference": "对，问题是我们其实没有这方面的预算。", + "ttft_ms": 104, + "e2e_ms": 313, + "gen_tokens": 8, + "tps": 38.3, + "prompt_tokens": 141, + "prompt_ms": 42, + "output": "所以，我们实际上没有那个预算。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.207 + }, + { + "engine": "ollama-mlx", + "category": "C_asr_noisy", + "id": "C05", + "run": 1, + "input": "like honestly I don't think anyone has looked at this since like march", + "reference": "说实话我觉得从三月以来就没人看过这个了。", + "ttft_ms": 96, + "e2e_ms": 336, + "gen_tokens": 10, + "tps": 41.5, + "prompt_tokens": 140, + "prompt_ms": 40, + "output": "说实话，我觉得自三月以来没人再看过了。", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 1, + "terms_missing": [], + "chrf": 0.406 + }, + { + "engine": "ollama-mlx", + "category": "C_asr_noisy", + "id": "C06", + "run": 1, + "input": "and uh we also need to loop in the legal team before we sign anything", + "reference": "另外在签任何东西之前我们还需要让法务团队参与进来。", + "ttft_ms": 89, + "e2e_ms": 454, + "gen_tokens": 15, + "tps": 41.0, + "prompt_tokens": 141, + "prompt_ms": 33, + "output": "而且，在我们签署任何文件之前，我们还需要让法务团队介入。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.44 + }, + { + "engine": "ollama-mlx", + "category": "C_asr_noisy", + "id": "C07", + "run": 1, + "input": "yeah no that's fair that's totally fair", + "reference": "对，没错，这很合理。", + "ttft_ms": 91, + "e2e_ms": 306, + "gen_tokens": 9, + "tps": 41.8, + "prompt_tokens": 135, + "prompt_ms": 37, + "output": "是啊，这很公平，完全没问题。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.218 + }, + { + "engine": "ollama-mlx", + "category": "C_asr_noisy", + "id": "C08", + "run": 1, + "input": "so hold on are we saying the deadline moved or not", + "reference": "等等，我们是说截止日期改了还是没改？", + "ttft_ms": 173, + "e2e_ms": 396, + "gen_tokens": 9, + "tps": 40.4, + "prompt_tokens": 137, + "prompt_ms": 119, + "output": "所以我们要说截止日期变了还是没变？", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.417 + }, + { + "engine": "ollama-mlx", + "category": "D_numbers_names", + "id": "D01", + "run": 0, + "input": "Revenue grew 12% year over year to $4.3 billion.", + "reference": "收入同比增长12%，达到43亿美元。", + "ttft_ms": 269, + "e2e_ms": 582, + "gen_tokens": 13, + "tps": 41.5, + "prompt_tokens": 142, + "prompt_ms": 212, + "output": "收入同比增长 12%，达到 43 亿美元。", + "compliant": true, + "problems": [], + "terms_total": 2, + "terms_kept": 2, + "terms_missing": [], + "chrf": 1.0 + }, + { + "engine": "ollama-mlx", + "category": "D_numbers_names", + "id": "D02", + "run": 0, + "input": "Sarah from Stripe will join us on March 15th at 3 p.m.", + "reference": "Stripe的Sarah将在3月15日下午3点加入我们。", + "ttft_ms": 160, + "e2e_ms": 689, + "gen_tokens": 22, + "tps": 41.6, + "prompt_tokens": 144, + "prompt_ms": 100, + "output": "来自 Stripe 的 Sarah 将于 3 月 15 日下午 3 点加入我们。", + "compliant": true, + "problems": [], + "terms_total": 3, + "terms_kept": 3, + "terms_missing": [], + "chrf": 0.89 + }, + { + "engine": "ollama-mlx", + "category": "D_numbers_names", + "id": "D03", + "run": 0, + "input": "We processed 2.7 million requests with a p99 latency of 180 milliseconds.", + "reference": "我们处理了270万个请求，p99延迟为180毫秒。", + "ttft_ms": 188, + "e2e_ms": 767, + "gen_tokens": 24, + "tps": 41.5, + "prompt_tokens": 147, + "prompt_ms": 104, + "output": "我们处理了 270 万次请求，p99 延迟为 180 毫秒。", + "compliant": true, + "problems": [], + "terms_total": 3, + "terms_kept": 3, + "terms_missing": [], + "chrf": 0.891 + }, + { + "engine": "ollama-mlx", + "category": "D_numbers_names", + "id": "D04", + "run": 0, + "input": "Microsoft and OpenAI announced the partnership in Q2 2024.", + "reference": "Microsoft和OpenAI在2024年第二季度宣布了合作。", + "ttft_ms": 159, + "e2e_ms": 590, + "gen_tokens": 18, + "tps": 41.7, + "prompt_tokens": 142, + "prompt_ms": 100, + "output": "微软和 OpenAI 于 2024 年第二季度宣布了这一合作关系。", + "compliant": true, + "problems": [], + "terms_total": 3, + "terms_kept": 3, + "terms_missing": [], + "chrf": 0.565 + }, + { + "engine": "ollama-mlx", + "category": "D_numbers_names", + "id": "D05", + "run": 0, + "input": "The budget is 850 thousand euros, split 60-40 between the two teams.", + "reference": "预算是85万欧元，两个团队按60比40分配。", + "ttft_ms": 242, + "e2e_ms": 697, + "gen_tokens": 19, + "tps": 41.8, + "prompt_tokens": 148, + "prompt_ms": 181, + "output": "预算为 85 万欧元，两个团队按 6:4 比例分配。", + "compliant": true, + "problems": [], + "terms_total": 3, + "terms_kept": 1, + "terms_missing": [ + "60", + "40" + ], + "chrf": 0.638 + }, + { + "engine": "ollama-mlx", + "category": "D_numbers_names", + "id": "D06", + "run": 0, + "input": "Dr. Nakamura's team at Stanford published the results on arXiv last Tuesday.", + "reference": "Nakamura博士在斯坦福的团队上周二在arXiv上发表了结果。", + "ttft_ms": 164, + "e2e_ms": 696, + "gen_tokens": 22, + "tps": 41.3, + "prompt_tokens": 144, + "prompt_ms": 101, + "output": "Nakamura 博士在斯坦福大学的团队于上周二将结果发布在 arXiv 上。", + "compliant": true, + "problems": [], + "terms_total": 2, + "terms_kept": 2, + "terms_missing": [], + "chrf": 0.688 + }, + { + "engine": "ollama-mlx", + "category": "D_numbers_names", + "id": "D07", + "run": 0, + "input": "Version 3.2.1 fixes the bug reported in ticket LCT-418.", + "reference": "3.2.1版本修复了工单LCT-418中报告的问题。", + "ttft_ms": 165, + "e2e_ms": 712, + "gen_tokens": 23, + "tps": 42.0, + "prompt_tokens": 146, + "prompt_ms": 104, + "output": "版本 3.2.1 修复了工单 LCT-418 中报告的错误。", + "compliant": true, + "problems": [], + "terms_total": 2, + "terms_kept": 2, + "terms_missing": [], + "chrf": 0.771 + }, + { + "engine": "ollama-mlx", + "category": "D_numbers_names", + "id": "D08", + "run": 0, + "input": "Our NPS went from 31 to 47 after the redesign.", + "reference": "改版后我们的NPS从31升到了47。", + "ttft_ms": 155, + "e2e_ms": 597, + "gen_tokens": 18, + "tps": 40.7, + "prompt_tokens": 142, + "prompt_ms": 99, + "output": "我们的 NPS 在重新设计后从 31 升至 47。", + "compliant": true, + "problems": [], + "terms_total": 3, + "terms_kept": 3, + "terms_missing": [], + "chrf": 0.506 + }, + { + "engine": "ollama-mlx", + "category": "D_numbers_names", + "id": "D01", + "run": 1, + "input": "Revenue grew 12% year over year to $4.3 billion.", + "reference": "收入同比增长12%，达到43亿美元。", + "ttft_ms": 156, + "e2e_ms": 465, + "gen_tokens": 13, + "tps": 42.2, + "prompt_tokens": 142, + "prompt_ms": 97, + "output": "收入同比增长 12%，达到 43 亿美元。", + "compliant": true, + "problems": [], + "terms_total": 2, + "terms_kept": 2, + "terms_missing": [], + "chrf": 1.0 + }, + { + "engine": "ollama-mlx", + "category": "D_numbers_names", + "id": "D02", + "run": 1, + "input": "Sarah from Stripe will join us on March 15th at 3 p.m.", + "reference": "Stripe的Sarah将在3月15日下午3点加入我们。", + "ttft_ms": 160, + "e2e_ms": 681, + "gen_tokens": 22, + "tps": 42.1, + "prompt_tokens": 144, + "prompt_ms": 97, + "output": "来自 Stripe 的 Sarah 将于 3 月 15 日下午 3 点加入我们。", + "compliant": true, + "problems": [], + "terms_total": 3, + "terms_kept": 3, + "terms_missing": [], + "chrf": 0.89 + }, + { + "engine": "ollama-mlx", + "category": "D_numbers_names", + "id": "D03", + "run": 1, + "input": "We processed 2.7 million requests with a p99 latency of 180 milliseconds.", + "reference": "我们处理了270万个请求，p99延迟为180毫秒。", + "ttft_ms": 227, + "e2e_ms": 804, + "gen_tokens": 24, + "tps": 41.6, + "prompt_tokens": 147, + "prompt_ms": 165, + "output": "我们处理了 270 万请求，p99 延迟为 180 毫秒。", + "compliant": true, + "problems": [], + "terms_total": 3, + "terms_kept": 3, + "terms_missing": [], + "chrf": 0.899 + }, + { + "engine": "ollama-mlx", + "category": "D_numbers_names", + "id": "D04", + "run": 1, + "input": "Microsoft and OpenAI announced the partnership in Q2 2024.", + "reference": "Microsoft和OpenAI在2024年第二季度宣布了合作。", + "ttft_ms": 242, + "e2e_ms": 664, + "gen_tokens": 18, + "tps": 42.8, + "prompt_tokens": 142, + "prompt_ms": 187, + "output": "微软和 OpenAI 于 2024 年第二季度宣布了合作。", + "compliant": true, + "problems": [], + "terms_total": 3, + "terms_kept": 3, + "terms_missing": [], + "chrf": 0.651 + }, + { + "engine": "ollama-mlx", + "category": "D_numbers_names", + "id": "D05", + "run": 1, + "input": "The budget is 850 thousand euros, split 60-40 between the two teams.", + "reference": "预算是85万欧元，两个团队按60比40分配。", + "ttft_ms": 174, + "e2e_ms": 651, + "gen_tokens": 20, + "tps": 41.9, + "prompt_tokens": 148, + "prompt_ms": 119, + "output": "预算为 85 万欧元，两个团队按 60:40 分配。", + "compliant": true, + "problems": [], + "terms_total": 3, + "terms_kept": 3, + "terms_missing": [], + "chrf": 0.763 + }, + { + "engine": "ollama-mlx", + "category": "D_numbers_names", + "id": "D06", + "run": 1, + "input": "Dr. Nakamura's team at Stanford published the results on arXiv last Tuesday.", + "reference": "Nakamura博士在斯坦福的团队上周二在arXiv上发表了结果。", + "ttft_ms": 354, + "e2e_ms": 770, + "gen_tokens": 17, + "tps": 40.9, + "prompt_tokens": 144, + "prompt_ms": 296, + "output": "斯坦福大学中村博士的团队于上周二在 arXiv 发布了研究成果。", + "compliant": true, + "problems": [], + "terms_total": 2, + "terms_kept": 2, + "terms_missing": [], + "chrf": 0.399 + }, + { + "engine": "ollama-mlx", + "category": "D_numbers_names", + "id": "D07", + "run": 1, + "input": "Version 3.2.1 fixes the bug reported in ticket LCT-418.", + "reference": "3.2.1版本修复了工单LCT-418中报告的问题。", + "ttft_ms": 381, + "e2e_ms": 941, + "gen_tokens": 23, + "tps": 41.1, + "prompt_tokens": 146, + "prompt_ms": 246, + "output": "版本 3.2.1 修复了 LCT-418 工单中报告的错误。", + "compliant": true, + "problems": [], + "terms_total": 2, + "terms_kept": 2, + "terms_missing": [], + "chrf": 0.591 + }, + { + "engine": "ollama-mlx", + "category": "D_numbers_names", + "id": "D08", + "run": 1, + "input": "Our NPS went from 31 to 47 after the redesign.", + "reference": "改版后我们的NPS从31升到了47。", + "ttft_ms": 488, + "e2e_ms": 921, + "gen_tokens": 18, + "tps": 41.6, + "prompt_tokens": 142, + "prompt_ms": 352, + "output": "我们的 NPS 在重新设计后从 31 升至 47。", + "compliant": true, + "problems": [], + "terms_total": 3, + "terms_kept": 3, + "terms_missing": [], + "chrf": 0.506 + }, + { + "engine": "ollama-mlx", + "category": "E_tech_terms", + "id": "E01", + "run": 0, + "input": "We migrated the service from Kubernetes to a single Rust binary on bare metal.", + "reference": "我们把这个服务从Kubernetes迁移到了裸金属上的单个Rust二进制。", + "ttft_ms": 243, + "e2e_ms": 625, + "gen_tokens": 16, + "tps": 41.9, + "prompt_tokens": 141, + "prompt_ms": 175, + "output": "我们将服务从 Kubernetes 迁移到了裸金属上的单一 Rust 二进制文件。", + "compliant": true, + "problems": [], + "terms_total": 2, + "terms_kept": 2, + "terms_missing": [], + "chrf": 0.787 + }, + { + "engine": "ollama-mlx", + "category": "E_tech_terms", + "id": "E02", + "run": 0, + "input": "The model runs fully on-device with Metal acceleration and a 4-bit quantized Qwen checkpoint.", + "reference": "模型完全在设备端运行，使用Metal加速和4位量化的Qwen检查点。", + "ttft_ms": 278, + "e2e_ms": 840, + "gen_tokens": 23, + "tps": 40.9, + "prompt_tokens": 146, + "prompt_ms": 219, + "output": "该模型完全在设备上运行，采用 Metal 加速和 4 位量化 Qwen 检查点。", + "compliant": true, + "problems": [], + "terms_total": 3, + "terms_kept": 3, + "terms_missing": [], + "chrf": 0.765 + }, + { + "engine": "ollama-mlx", + "category": "E_tech_terms", + "id": "E03", + "run": 0, + "input": "Enable WASAPI loopback so we capture system audio without a virtual device.", + "reference": "启用WASAPI环回，这样不需要虚拟设备也能采集系统音频。", + "ttft_ms": 158, + "e2e_ms": 564, + "gen_tokens": 17, + "tps": 41.9, + "prompt_tokens": 141, + "prompt_ms": 102, + "output": "启用 WASAPI 回环，以便在不使用虚拟设备的情况下捕获系统音频。", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 1, + "terms_missing": [], + "chrf": 0.475 + }, + { + "engine": "ollama-mlx", + "category": "E_tech_terms", + "id": "E04", + "run": 0, + "input": "The API returns a 429 when you exceed the rate limit, so add exponential backoff.", + "reference": "超过速率限制时API会返回429，所以要加指数退避。", + "ttft_ms": 170, + "e2e_ms": 646, + "gen_tokens": 20, + "tps": 42.1, + "prompt_tokens": 147, + "prompt_ms": 109, + "output": "API 在超过速率限制时返回 429，因此请添加指数退避。", + "compliant": true, + "problems": [], + "terms_total": 2, + "terms_kept": 2, + "terms_missing": [], + "chrf": 0.641 + }, + { + "engine": "ollama-mlx", + "category": "E_tech_terms", + "id": "E05", + "run": 0, + "input": "We're seeing GPU memory pressure because num_ctx defaults to 32K.", + "reference": "我们遇到了GPU内存压力，因为num_ctx默认是32K。", + "ttft_ms": 473, + "e2e_ms": 880, + "gen_tokens": 17, + "tps": 41.7, + "prompt_tokens": 142, + "prompt_ms": 391, + "output": "我们看到 GPU 内存压力，因为 num_ctx 默认为 32K。", + "compliant": true, + "problems": [], + "terms_total": 3, + "terms_kept": 3, + "terms_missing": [], + "chrf": 0.766 + }, + { + "engine": "ollama-mlx", + "category": "E_tech_terms", + "id": "E06", + "run": 0, + "input": "The PR adds a TranslationEngine protocol with an Ollama adapter and an embedded llama.cpp runtime.", + "reference": "这个PR增加了TranslationEngine协议，带有一个Ollama适配器和一个内置的llama.cpp运行时。", + "ttft_ms": 161, + "e2e_ms": 684, + "gen_tokens": 22, + "tps": 42.1, + "prompt_tokens": 146, + "prompt_ms": 106, + "output": "PR 添加了 TranslationEngine 协议，包含 Ollama 适配器及嵌入的 llama.cpp 运行时。", + "compliant": true, + "problems": [], + "terms_total": 4, + "terms_kept": 4, + "terms_missing": [], + "chrf": 0.732 + }, + { + "engine": "ollama-mlx", + "category": "E_tech_terms", + "id": "E07", + "run": 0, + "input": "Notarization requires a Developer ID certificate and the hardened runtime.", + "reference": "公证需要Developer ID证书和强化运行时。", + "ttft_ms": 155, + "e2e_ms": 393, + "gen_tokens": 10, + "tps": 42.1, + "prompt_tokens": 139, + "prompt_ms": 102, + "output": "公证需要开发者 ID 证书和加固运行时。", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 0, + "terms_missing": [ + "Developer ID" + ], + "chrf": 0.392 + }, + { + "engine": "ollama-mlx", + "category": "E_tech_terms", + "id": "E08", + "run": 0, + "input": "Our CI runs on windows-latest, which no longer ships NSIS.", + "reference": "我们的CI跑在windows-latest上，它不再自带NSIS。", + "ttft_ms": 161, + "e2e_ms": 534, + "gen_tokens": 16, + "tps": 42.8, + "prompt_tokens": 141, + "prompt_ms": 105, + "output": "我们的 CI 运行在 windows-latest 上，不再分发 NSIS。", + "compliant": true, + "problems": [], + "terms_total": 3, + "terms_kept": 3, + "terms_missing": [], + "chrf": 0.734 + }, + { + "engine": "ollama-mlx", + "category": "E_tech_terms", + "id": "E01", + "run": 1, + "input": "We migrated the service from Kubernetes to a single Rust binary on bare metal.", + "reference": "我们把这个服务从Kubernetes迁移到了裸金属上的单个Rust二进制。", + "ttft_ms": 268, + "e2e_ms": 643, + "gen_tokens": 16, + "tps": 42.7, + "prompt_tokens": 141, + "prompt_ms": 213, + "output": "我们将服务从 Kubernetes 迁移到了裸金属上的单个 Rust 二进制文件。", + "compliant": true, + "problems": [], + "terms_total": 2, + "terms_kept": 2, + "terms_missing": [], + "chrf": 0.858 + }, + { + "engine": "ollama-mlx", + "category": "E_tech_terms", + "id": "E02", + "run": 1, + "input": "The model runs fully on-device with Metal acceleration and a 4-bit quantized Qwen checkpoint.", + "reference": "模型完全在设备端运行，使用Metal加速和4位量化的Qwen检查点。", + "ttft_ms": 782, + "e2e_ms": 1356, + "gen_tokens": 24, + "tps": 41.8, + "prompt_tokens": 146, + "prompt_ms": 710, + "output": "该模型完全在设备端运行，采用 Metal 加速和 4 位量化 Qwen 检查点。", + "compliant": true, + "problems": [], + "terms_total": 3, + "terms_kept": 3, + "terms_missing": [], + "chrf": 0.844 + }, + { + "engine": "ollama-mlx", + "category": "E_tech_terms", + "id": "E03", + "run": 1, + "input": "Enable WASAPI loopback so we capture system audio without a virtual device.", + "reference": "启用WASAPI环回，这样不需要虚拟设备也能采集系统音频。", + "ttft_ms": 87, + "e2e_ms": 495, + "gen_tokens": 17, + "tps": 41.8, + "prompt_tokens": 141, + "prompt_ms": 32, + "output": "启用 WASAPI 回环，以便在不使用虚拟设备的情况下捕获系统音频。", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 1, + "terms_missing": [], + "chrf": 0.475 + }, + { + "engine": "ollama-mlx", + "category": "E_tech_terms", + "id": "E04", + "run": 1, + "input": "The API returns a 429 when you exceed the rate limit, so add exponential backoff.", + "reference": "超过速率限制时API会返回429，所以要加指数退避。", + "ttft_ms": 91, + "e2e_ms": 582, + "gen_tokens": 20, + "tps": 40.7, + "prompt_tokens": 147, + "prompt_ms": 31, + "output": "API 在超过速率限制时返回 429，因此请添加指数退避。", + "compliant": true, + "problems": [], + "terms_total": 2, + "terms_kept": 2, + "terms_missing": [], + "chrf": 0.641 + }, + { + "engine": "ollama-mlx", + "category": "E_tech_terms", + "id": "E05", + "run": 1, + "input": "We're seeing GPU memory pressure because num_ctx defaults to 32K.", + "reference": "我们遇到了GPU内存压力，因为num_ctx默认是32K。", + "ttft_ms": 95, + "e2e_ms": 510, + "gen_tokens": 17, + "tps": 40.9, + "prompt_tokens": 142, + "prompt_ms": 33, + "output": "我们看到 GPU 内存压力，因为 num_ctx 默认为 32K。", + "compliant": true, + "problems": [], + "terms_total": 3, + "terms_kept": 3, + "terms_missing": [], + "chrf": 0.766 + }, + { + "engine": "ollama-mlx", + "category": "E_tech_terms", + "id": "E06", + "run": 1, + "input": "The PR adds a TranslationEngine protocol with an Ollama adapter and an embedded llama.cpp runtime.", + "reference": "这个PR增加了TranslationEngine协议，带有一个Ollama适配器和一个内置的llama.cpp运行时。", + "ttft_ms": 87, + "e2e_ms": 586, + "gen_tokens": 21, + "tps": 42.1, + "prompt_tokens": 146, + "prompt_ms": 31, + "output": "PR 添加了带有 Ollama 适配器和嵌入式 llama.cpp 运行时的 TranslationEngine 协议。", + "compliant": true, + "problems": [], + "terms_total": 4, + "terms_kept": 4, + "terms_missing": [], + "chrf": 0.695 + }, + { + "engine": "ollama-mlx", + "category": "E_tech_terms", + "id": "E07", + "run": 1, + "input": "Notarization requires a Developer ID certificate and the hardened runtime.", + "reference": "公证需要Developer ID证书和强化运行时。", + "ttft_ms": 96, + "e2e_ms": 343, + "gen_tokens": 10, + "tps": 40.5, + "prompt_tokens": 139, + "prompt_ms": 36, + "output": "公证需要开发者 ID 证书和加固运行时。", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 0, + "terms_missing": [ + "Developer ID" + ], + "chrf": 0.392 + }, + { + "engine": "ollama-mlx", + "category": "E_tech_terms", + "id": "E08", + "run": 1, + "input": "Our CI runs on windows-latest, which no longer ships NSIS.", + "reference": "我们的CI跑在windows-latest上，它不再自带NSIS。", + "ttft_ms": 87, + "e2e_ms": 474, + "gen_tokens": 16, + "tps": 41.3, + "prompt_tokens": 141, + "prompt_ms": 30, + "output": "我们的 CI 运行在 windows-latest 上，不再分发 NSIS。", + "compliant": true, + "problems": [], + "terms_total": 3, + "terms_kept": 3, + "terms_missing": [], + "chrf": 0.734 + }, + { + "engine": "ollama-mlx", + "category": "F_long_sentences", + "id": "F01", + "run": 0, + "input": "What I want to emphasize today is that the transition to on-device inference is not primarily about cost, it's about latency and privacy, and those two things are what actually change user behavior.", + "reference": "我今天想强调的是，向设备端推理的转变主要不是为了成本，而是为了延迟和隐私，而这两点才是真正改变用户行为的因素。", + "ttft_ms": 241, + "e2e_ms": 959, + "gen_tokens": 30, + "tps": 41.8, + "prompt_tokens": 164, + "prompt_ms": 182, + "output": "我今天要强调的是，向设备端推理的转型并非主要关乎成本，而是关于延迟和隐私；而这两者才是真正改变用户行为的关键。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.595 + }, + { + "engine": "ollama-mlx", + "category": "F_long_sentences", + "id": "F02", + "run": 0, + "input": "Before we commit to the Q3 roadmap, I'd like each team lead to send me a one-page summary of their top three risks and what support they need from leadership to mitigate them.", + "reference": "在我们确定第三季度路线图之前，我希望每位团队负责人给我发一页纸的摘要，说明他们最主要的三个风险，以及需要管理层提供什么支持来缓解这些风险。", + "ttft_ms": 247, + "e2e_ms": 1087, + "gen_tokens": 35, + "tps": 41.7, + "prompt_tokens": 164, + "prompt_ms": 189, + "output": "在我们承诺第三季度路线图之前，我希望每位团队负责人向我发送一份一页纸的摘要，列出他们面临的三大风险，以及他们需要领导层提供的支持以减轻这些风险。", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 1, + "terms_missing": [], + "chrf": 0.552 + }, + { + "engine": "ollama-mlx", + "category": "F_long_sentences", + "id": "F03", + "run": 0, + "input": "If the benchmark shows that the embedded runtime is within twenty percent of the Ollama path on first-token latency, I'm comfortable making it the default and keeping Ollama as an advanced option.", + "reference": "如果基准测试显示内置运行时的首字延迟与Ollama路径相差在百分之二十以内，我可以接受把它设为默认，并把Ollama保留为高级选项。", + "ttft_ms": 235, + "e2e_ms": 1225, + "gen_tokens": 41, + "tps": 41.4, + "prompt_tokens": 166, + "prompt_ms": 182, + "output": "如果基准测试显示，在首令牌延迟方面，嵌入式运行时处于 Ollama 路径的百分之二十以内，我就可以将其设为默认选项，并将 Ollama 保留为高级选项。", + "compliant": true, + "problems": [], + "terms_total": 2, + "terms_kept": 2, + "terms_missing": [], + "chrf": 0.606 + }, + { + "engine": "ollama-mlx", + "category": "F_long_sentences", + "id": "F04", + "run": 0, + "input": "The reason the microphone channel picks up the speaker audio is simply acoustic feedback, so with headphones the duplicate lines disappear, but we should still say that in onboarding.", + "reference": "麦克风通道会拾取扬声器声音的原因就是声学回授，所以戴上耳机重复的字幕就会消失，但我们仍然应该在引导里说明这一点。", + "ttft_ms": 254, + "e2e_ms": 1039, + "gen_tokens": 33, + "tps": 42.1, + "prompt_tokens": 159, + "prompt_ms": 198, + "output": "麦克风通道拾取到说话人音频的原因仅仅是声学反馈，因此佩戴耳机后重复的杂音就会消失，但我们仍应在入职培训中说明这一点。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.411 + }, + { + "engine": "ollama-mlx", + "category": "F_long_sentences", + "id": "F05", + "run": 0, + "input": "Historically every attempt to bundle a second inference server inside a desktop app has failed on maintenance cost, so the question is whether a statically linked library changes that calculus.", + "reference": "历史上每一次在桌面应用里捆绑第二个推理服务的尝试都败在了维护成本上，所以问题在于静态链接的库是否改变了这个算计。", + "ttft_ms": 244, + "e2e_ms": 1003, + "gen_tokens": 31, + "tps": 40.8, + "prompt_tokens": 159, + "prompt_ms": 184, + "output": "历史上，每次试图将第二个推理服务器打包到桌面应用中都会因维护成本而失败，因此问题是：静态链接的库是否改变了这一计算。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.47 + }, + { + "engine": "ollama-mlx", + "category": "F_long_sentences", + "id": "F06", + "run": 0, + "input": "Let's park the pricing discussion, because until we know whether enterprise customers actually need SSO and audit logs, any number we put on the slide is going to be a guess.", + "reference": "我们先把定价讨论放一放，因为在知道企业客户是否真的需要SSO和审计日志之前，我们放在幻灯片上的任何数字都只是猜测。", + "ttft_ms": 747, + "e2e_ms": 1579, + "gen_tokens": 35, + "tps": 42.0, + "prompt_tokens": 162, + "prompt_ms": 676, + "output": "让我们把定价讨论放一放，因为在我们不知道企业客户是否真正需要单点登录和审计日志之前，我们在幻灯片上列出的任何数字都只能是猜测。", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 0, + "terms_missing": [ + "SSO" + ], + "chrf": 0.694 + }, + { + "engine": "ollama-mlx", + "category": "F_long_sentences", + "id": "F01", + "run": 1, + "input": "What I want to emphasize today is that the transition to on-device inference is not primarily about cost, it's about latency and privacy, and those two things are what actually change user behavior.", + "reference": "我今天想强调的是，向设备端推理的转变主要不是为了成本，而是为了延迟和隐私，而这两点才是真正改变用户行为的因素。", + "ttft_ms": 96, + "e2e_ms": 811, + "gen_tokens": 30, + "tps": 41.9, + "prompt_tokens": 164, + "prompt_ms": 36, + "output": "今天我想要强调的是，向端侧推理的转型主要并非关乎成本，而是关乎延迟与隐私；这两者才是真正改变用户行为的关键。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.461 + }, + { + "engine": "ollama-mlx", + "category": "F_long_sentences", + "id": "F02", + "run": 1, + "input": "Before we commit to the Q3 roadmap, I'd like each team lead to send me a one-page summary of their top three risks and what support they need from leadership to mitigate them.", + "reference": "在我们确定第三季度路线图之前，我希望每位团队负责人给我发一页纸的摘要，说明他们最主要的三个风险，以及需要管理层提供什么支持来缓解这些风险。", + "ttft_ms": 91, + "e2e_ms": 891, + "gen_tokens": 34, + "tps": 42.5, + "prompt_tokens": 164, + "prompt_ms": 34, + "output": "在我们承诺第三季度路线图之前，我希望每位团队负责人向我发送一份一页纸的摘要，列出他们面临的三大主要风险以及需要从管理层获得的支持以缓解这些风险。", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 1, + "terms_missing": [], + "chrf": 0.577 + }, + { + "engine": "ollama-mlx", + "category": "F_long_sentences", + "id": "F03", + "run": 1, + "input": "If the benchmark shows that the embedded runtime is within twenty percent of the Ollama path on first-token latency, I'm comfortable making it the default and keeping Ollama as an advanced option.", + "reference": "如果基准测试显示内置运行时的首字延迟与Ollama路径相差在百分之二十以内，我可以接受把它设为默认，并把Ollama保留为高级选项。", + "ttft_ms": 113, + "e2e_ms": 1147, + "gen_tokens": 42, + "tps": 40.6, + "prompt_tokens": 166, + "prompt_ms": 49, + "output": "如果基准测试显示，在首 token 延迟方面，嵌入式运行时处于 Ollama 路径的百分之二十以内，我将其作为默认选项并保留 Ollama 作为高级选项感到放心。", + "compliant": true, + "problems": [], + "terms_total": 2, + "terms_kept": 2, + "terms_missing": [], + "chrf": 0.518 + }, + { + "engine": "ollama-mlx", + "category": "F_long_sentences", + "id": "F04", + "run": 1, + "input": "The reason the microphone channel picks up the speaker audio is simply acoustic feedback, so with headphones the duplicate lines disappear, but we should still say that in onboarding.", + "reference": "麦克风通道会拾取扬声器声音的原因就是声学回授，所以戴上耳机重复的字幕就会消失，但我们仍然应该在引导里说明这一点。", + "ttft_ms": 94, + "e2e_ms": 896, + "gen_tokens": 33, + "tps": 41.1, + "prompt_tokens": 159, + "prompt_ms": 34, + "output": "麦克风通道拾取到说话人音频的原因仅仅是声学反馈，因此佩戴耳机后重复的杂音就会消失，但我们仍应在入职培训中说明这一点。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.411 + }, + { + "engine": "ollama-mlx", + "category": "F_long_sentences", + "id": "F05", + "run": 1, + "input": "Historically every attempt to bundle a second inference server inside a desktop app has failed on maintenance cost, so the question is whether a statically linked library changes that calculus.", + "reference": "历史上每一次在桌面应用里捆绑第二个推理服务的尝试都败在了维护成本上，所以问题在于静态链接的库是否改变了这个算计。", + "ttft_ms": 87, + "e2e_ms": 895, + "gen_tokens": 33, + "tps": 40.8, + "prompt_tokens": 159, + "prompt_ms": 31, + "output": "历史上，每次试图将第二个推理服务器打包进桌面应用时，由于维护成本问题都失败了，因此问题是：静态链接的库是否会改变这一计算。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.419 + }, + { + "engine": "ollama-mlx", + "category": "F_long_sentences", + "id": "F06", + "run": 1, + "input": "Let's park the pricing discussion, because until we know whether enterprise customers actually need SSO and audit logs, any number we put on the slide is going to be a guess.", + "reference": "我们先把定价讨论放一放，因为在知道企业客户是否真的需要SSO和审计日志之前，我们放在幻灯片上的任何数字都只是猜测。", + "ttft_ms": 113, + "e2e_ms": 946, + "gen_tokens": 34, + "tps": 40.8, + "prompt_tokens": 162, + "prompt_ms": 35, + "output": "让我们把定价讨论放一放，因为在我们还不知道企业客户是否真的需要单点登录和审计日志之前，幻灯片上列出的任何数字都只能是猜测。", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 0, + "terms_missing": [ + "SSO" + ], + "chrf": 0.699 + }, + { + "engine": "ollama-mlx", + "category": "G_zh_to_en", + "id": "G01", + "run": 0, + "input": "这个功能我们下周三之前能上线吗", + "reference": "Can we ship this feature before next Wednesday?", + "ttft_ms": 506, + "e2e_ms": 726, + "gen_tokens": 9, + "tps": 40.9, + "prompt_tokens": 132, + "prompt_ms": 434, + "output": "Can this feature be launched before next Wednesday?", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 1, + "terms_missing": [], + "chrf": 0.793 + }, + { + "engine": "ollama-mlx", + "category": "G_zh_to_en", + "id": "G02", + "run": 0, + "input": "那个就是说我们现在有两个方案一个是自己做一个是买第三方的", + "reference": "So we have two options now: build it ourselves, or buy a third-party solution.", + "ttft_ms": 407, + "e2e_ms": 1086, + "gen_tokens": 28, + "tps": 41.2, + "prompt_tokens": 135, + "prompt_ms": 348, + "output": "That's to say we currently have two options: one is to build it ourselves, and the other is to purchase a third-party solution.", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 1, + "terms_missing": [], + "chrf": 0.705 + }, + { + "engine": "ollama-mlx", + "category": "G_zh_to_en", + "id": "G03", + "run": 0, + "input": "上个季度我们的留存率从百分之三十八涨到了百分之四十五", + "reference": "Last quarter our retention rate went from 38% to 45%.", + "ttft_ms": 160, + "e2e_ms": 603, + "gen_tokens": 18, + "tps": 40.7, + "prompt_tokens": 137, + "prompt_ms": 102, + "output": "Last quarter, our retention rate rose from 38 percent to 45 percent.", + "compliant": true, + "problems": [], + "terms_total": 2, + "terms_kept": 2, + "terms_missing": [], + "chrf": 0.74 + }, + { + "engine": "ollama-mlx", + "category": "G_zh_to_en", + "id": "G04", + "run": 0, + "input": "我觉得这个风险有点大，要不我们先小范围灰度一下", + "reference": "I think the risk is a bit high. How about a small-scale gradual rollout first?", + "ttft_ms": 371, + "e2e_ms": 811, + "gen_tokens": 18, + "tps": 40.9, + "prompt_tokens": 136, + "prompt_ms": 188, + "output": "I think this risk is quite high; let's do a small-scale gray release first.", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.545 + }, + { + "engine": "ollama-mlx", + "category": "G_zh_to_en", + "id": "G05", + "run": 0, + "input": "张总说预算最多给到五十万，超了要重新审批", + "reference": "Mr. Zhang said the budget is capped at 500 thousand; anything above that needs re-approval.", + "ttft_ms": 169, + "e2e_ms": 778, + "gen_tokens": 25, + "tps": 41.0, + "prompt_tokens": 138, + "prompt_ms": 105, + "output": "Mr. Zhang says the budget cap is 500,000; anything exceeding that requires re-approval.", + "compliant": true, + "problems": [], + "terms_total": 2, + "terms_kept": 2, + "terms_missing": [], + "chrf": 0.587 + }, + { + "engine": "ollama-mlx", + "category": "G_zh_to_en", + "id": "G06", + "run": 0, + "input": "识别延迟大概一点五秒，翻译再加一秒左右，用户基本感觉不到", + "reference": "Recognition latency is about 1.5 seconds, translation adds roughly another second, so users barely notice.", + "ttft_ms": 172, + "e2e_ms": 737, + "gen_tokens": 23, + "tps": 40.7, + "prompt_tokens": 139, + "prompt_ms": 104, + "output": "Recognition delay is about 1.5 seconds; translation adds another second or so. Users generally don't notice.", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 1, + "terms_missing": [], + "chrf": 0.718 + }, + { + "engine": "ollama-mlx", + "category": "G_zh_to_en", + "id": "G07", + "run": 0, + "input": "等一下我这边网不太好你们先聊", + "reference": "Hold on, my connection isn't great. You go ahead.", + "ttft_ms": 165, + "e2e_ms": 588, + "gen_tokens": 18, + "tps": 42.6, + "prompt_tokens": 131, + "prompt_ms": 103, + "output": "Wait, my internet connection isn't great here; you guys go ahead and chat first.", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.646 + }, + { + "engine": "ollama-mlx", + "category": "G_zh_to_en", + "id": "G08", + "run": 0, + "input": "这个PR先不要合，Codex还有两条评论没处理", + "reference": "Don't merge this PR yet; there are still two Codex comments to address.", + "ttft_ms": 201, + "e2e_ms": 625, + "gen_tokens": 17, + "tps": 40.1, + "prompt_tokens": 136, + "prompt_ms": 106, + "output": "This PR shouldn't be merged yet; Codex still has two comments to address.", + "compliant": true, + "problems": [], + "terms_total": 3, + "terms_kept": 3, + "terms_missing": [], + "chrf": 0.679 + }, + { + "engine": "ollama-mlx", + "category": "G_zh_to_en", + "id": "G01", + "run": 1, + "input": "这个功能我们下周三之前能上线吗", + "reference": "Can we ship this feature before next Wednesday?", + "ttft_ms": 417, + "e2e_ms": 638, + "gen_tokens": 9, + "tps": 40.7, + "prompt_tokens": 132, + "prompt_ms": 343, + "output": "Can this feature be launched before next Wednesday?", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 1, + "terms_missing": [], + "chrf": 0.793 + }, + { + "engine": "ollama-mlx", + "category": "G_zh_to_en", + "id": "G02", + "run": 1, + "input": "那个就是说我们现在有两个方案一个是自己做一个是买第三方的", + "reference": "So we have two options now: build it ourselves, or buy a third-party solution.", + "ttft_ms": 109, + "e2e_ms": 827, + "gen_tokens": 29, + "tps": 40.4, + "prompt_tokens": 135, + "prompt_ms": 45, + "output": "That is to say, we currently have two options: one is to build our own, and the other is to purchase from a third party.", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 1, + "terms_missing": [], + "chrf": 0.484 + }, + { + "engine": "ollama-mlx", + "category": "G_zh_to_en", + "id": "G03", + "run": 1, + "input": "上个季度我们的留存率从百分之三十八涨到了百分之四十五", + "reference": "Last quarter our retention rate went from 38% to 45%.", + "ttft_ms": 183, + "e2e_ms": 618, + "gen_tokens": 18, + "tps": 41.4, + "prompt_tokens": 137, + "prompt_ms": 122, + "output": "Last quarter, our retention rate increased from 38 percent to 45 percent.", + "compliant": true, + "problems": [], + "terms_total": 2, + "terms_kept": 2, + "terms_missing": [], + "chrf": 0.708 + }, + { + "engine": "ollama-mlx", + "category": "G_zh_to_en", + "id": "G04", + "run": 1, + "input": "我觉得这个风险有点大，要不我们先小范围灰度一下", + "reference": "I think the risk is a bit high. How about a small-scale gradual rollout first?", + "ttft_ms": 184, + "e2e_ms": 666, + "gen_tokens": 20, + "tps": 41.6, + "prompt_tokens": 136, + "prompt_ms": 125, + "output": "I think this risk is quite high; perhaps we should start with a small-scale gray release first.", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.562 + }, + { + "engine": "ollama-mlx", + "category": "G_zh_to_en", + "id": "G05", + "run": 1, + "input": "张总说预算最多给到五十万，超了要重新审批", + "reference": "Mr. Zhang said the budget is capped at 500 thousand; anything above that needs re-approval.", + "ttft_ms": 111, + "e2e_ms": 712, + "gen_tokens": 24, + "tps": 39.9, + "prompt_tokens": 138, + "prompt_ms": 47, + "output": "Mr. Zhang says the budget cap is 500,000; anything over requires re-approval.", + "compliant": true, + "problems": [], + "terms_total": 2, + "terms_kept": 2, + "terms_missing": [], + "chrf": 0.564 + }, + { + "engine": "ollama-mlx", + "category": "G_zh_to_en", + "id": "G06", + "run": 1, + "input": "识别延迟大概一点五秒，翻译再加一秒左右，用户基本感觉不到", + "reference": "Recognition latency is about 1.5 seconds, translation adds roughly another second, so users barely notice.", + "ttft_ms": 158, + "e2e_ms": 766, + "gen_tokens": 24, + "tps": 39.4, + "prompt_tokens": 139, + "prompt_ms": 87, + "output": "Recognition delay is approximately 1.5 seconds; translation adds another second or so, and users generally don't notice.", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 1, + "terms_missing": [], + "chrf": 0.668 + }, + { + "engine": "ollama-mlx", + "category": "G_zh_to_en", + "id": "G07", + "run": 1, + "input": "等一下我这边网不太好你们先聊", + "reference": "Hold on, my connection isn't great. You go ahead.", + "ttft_ms": 100, + "e2e_ms": 597, + "gen_tokens": 20, + "tps": 40.2, + "prompt_tokens": 131, + "prompt_ms": 39, + "output": "Wait, my internet connection isn't good on my end; you guys go ahead and chat first.", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.606 + }, + { + "engine": "ollama-mlx", + "category": "G_zh_to_en", + "id": "G08", + "run": 1, + "input": "这个PR先不要合，Codex还有两条评论没处理", + "reference": "Don't merge this PR yet; there are still two Codex comments to address.", + "ttft_ms": 240, + "e2e_ms": 611, + "gen_tokens": 16, + "tps": 43.0, + "prompt_tokens": 136, + "prompt_ms": 171, + "output": "This PR should not be merged yet; Codex still has two comments pending.", + "compliant": true, + "problems": [], + "terms_total": 3, + "terms_kept": 3, + "terms_missing": [], + "chrf": 0.527 + }, + { + "engine": "ollama-mlx", + "category": "H_context_followup", + "id": "H01", + "run": 0, + "input": "We benchmarked the MLX build last week.", + "reference": "我们上周对MLX版本做了基准测试。", + "ttft_ms": 1185, + "e2e_ms": 1434, + "gen_tokens": 10, + "tps": 40.2, + "prompt_tokens": 136, + "prompt_ms": 1115, + "output": "上周我们基准测试了 MLX 版本。", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 1, + "terms_missing": [], + "chrf": 0.511 + }, + { + "engine": "ollama-mlx", + "category": "H_context_followup", + "id": "H02", + "run": 0, + "input": "It was about twice as fast on first token.", + "reference": "它的首字速度大约快了一倍。", + "ttft_ms": 288, + "e2e_ms": 479, + "gen_tokens": 8, + "tps": 41.9, + "prompt_tokens": 172, + "prompt_ms": 191, + "output": "首词速度大约快了一倍。", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 1, + "terms_missing": [], + "chrf": 0.692 + }, + { + "engine": "ollama-mlx", + "category": "H_context_followup", + "id": "H03", + "run": 0, + "input": "But it only works on Apple Silicon.", + "reference": "但它只能在Apple Silicon上运行。", + "ttft_ms": 266, + "e2e_ms": 439, + "gen_tokens": 7, + "tps": 40.4, + "prompt_tokens": 204, + "prompt_ms": 189, + "output": "但它仅在苹果芯片上运行。", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 0, + "terms_missing": [ + "Apple Silicon" + ], + "chrf": 0.191 + }, + { + "engine": "ollama-mlx", + "category": "H_context_followup", + "id": "H04", + "run": 0, + "input": "So for Intel Macs we'd still need the other one.", + "reference": "所以对于Intel的Mac我们仍然需要另一个。", + "ttft_ms": 452, + "e2e_ms": 746, + "gen_tokens": 12, + "tps": 40.8, + "prompt_tokens": 204, + "prompt_ms": 385, + "output": "因此，对于英特尔 Mac 我们仍然需要另一个版本。", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 0, + "terms_missing": [ + "Intel" + ], + "chrf": 0.531 + }, + { + "engine": "ollama-mlx", + "category": "H_context_followup", + "id": "H05", + "run": 0, + "input": "She asked whether that doubles the maintenance work.", + "reference": "她问这是否会让维护工作翻倍。", + "ttft_ms": 329, + "e2e_ms": 542, + "gen_tokens": 9, + "tps": 42.4, + "prompt_tokens": 207, + "prompt_ms": 268, + "output": "她问这是否会使维护工作量翻倍。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.651 + }, + { + "engine": "ollama-mlx", + "category": "H_context_followup", + "id": "H06", + "run": 0, + "input": "Honestly, it probably does.", + "reference": "说实话，可能会。", + "ttft_ms": 331, + "e2e_ms": 500, + "gen_tokens": 7, + "tps": 41.4, + "prompt_tokens": 207, + "prompt_ms": 271, + "output": "老实说，很可能确实如此。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.203 + }, + { + "engine": "ollama-mlx", + "category": "H_context_followup", + "id": "H01", + "run": 1, + "input": "We benchmarked the MLX build last week.", + "reference": "我们上周对MLX版本做了基准测试。", + "ttft_ms": 262, + "e2e_ms": 531, + "gen_tokens": 11, + "tps": 40.9, + "prompt_tokens": 136, + "prompt_ms": 186, + "output": "我们上周对 MLX 构建进行了基准测试。", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 1, + "terms_missing": [], + "chrf": 0.694 + }, + { + "engine": "ollama-mlx", + "category": "H_context_followup", + "id": "H02", + "run": 1, + "input": "It was about twice as fast on first token.", + "reference": "它的首字速度大约快了一倍。", + "ttft_ms": 289, + "e2e_ms": 522, + "gen_tokens": 9, + "tps": 38.7, + "prompt_tokens": 173, + "prompt_ms": 189, + "output": "首词生成速度大约快了两倍。", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 1, + "terms_missing": [], + "chrf": 0.464 + }, + { + "engine": "ollama-mlx", + "category": "H_context_followup", + "id": "H03", + "run": 1, + "input": "But it only works on Apple Silicon.", + "reference": "但它只能在Apple Silicon上运行。", + "ttft_ms": 347, + "e2e_ms": 520, + "gen_tokens": 7, + "tps": 40.5, + "prompt_tokens": 206, + "prompt_ms": 204, + "output": "但它仅在苹果芯片上运行。", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 0, + "terms_missing": [ + "Apple Silicon" + ], + "chrf": 0.191 + }, + { + "engine": "ollama-mlx", + "category": "H_context_followup", + "id": "H04", + "run": 1, + "input": "So for Intel Macs we'd still need the other one.", + "reference": "所以对于Intel的Mac我们仍然需要另一个。", + "ttft_ms": 340, + "e2e_ms": 630, + "gen_tokens": 12, + "tps": 41.3, + "prompt_tokens": 205, + "prompt_ms": 275, + "output": "因此，对于英特尔 Mac 我们仍然需要另一个版本。", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 0, + "terms_missing": [ + "Intel" + ], + "chrf": 0.531 + }, + { + "engine": "ollama-mlx", + "category": "H_context_followup", + "id": "H05", + "run": 1, + "input": "She asked whether that doubles the maintenance work.", + "reference": "她问这是否会让维护工作翻倍。", + "ttft_ms": 142, + "e2e_ms": 364, + "gen_tokens": 9, + "tps": 40.5, + "prompt_tokens": 207, + "prompt_ms": 80, + "output": "她问这是否会使维护工作量翻倍。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.651 + }, + { + "engine": "ollama-mlx", + "category": "H_context_followup", + "id": "H06", + "run": 1, + "input": "Honestly, it probably does.", + "reference": "说实话，可能会。", + "ttft_ms": 93, + "e2e_ms": 266, + "gen_tokens": 7, + "tps": 40.5, + "prompt_tokens": 207, + "prompt_ms": 35, + "output": "老实说，很可能确实如此。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.203 + }, + { + "engine": "ollama-4b-tag", + "category": "A_short_turns", + "id": "A01", + "run": 0, + "input": "Let's get started.", + "reference": "我们开始吧。", + "ttft_ms": 499, + "e2e_ms": 604, + "gen_tokens": 4, + "tps": 38.3, + "prompt_tokens": 131, + "prompt_ms": 434, + "output": "我们开始吧。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 1.0 + }, + { + "engine": "ollama-4b-tag", + "category": "A_short_turns", + "id": "A02", + "run": 0, + "input": "Can everyone see my screen?", + "reference": "大家能看到我的屏幕吗？", + "ttft_ms": 480, + "e2e_ms": 625, + "gen_tokens": 6, + "tps": 41.5, + "prompt_tokens": 132, + "prompt_ms": 431, + "output": "大家能看到我的屏幕吗？", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 1.0 + }, + { + "engine": "ollama-4b-tag", + "category": "A_short_turns", + "id": "A03", + "run": 0, + "input": "Sorry, you cut out for a second.", + "reference": "抱歉，你刚才断了一下。", + "ttft_ms": 163, + "e2e_ms": 311, + "gen_tokens": 6, + "tps": 40.5, + "prompt_tokens": 135, + "prompt_ms": 104, + "output": "抱歉，您中断了。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.254 + }, + { + "engine": "ollama-4b-tag", + "category": "A_short_turns", + "id": "A04", + "run": 0, + "input": "I'll take that offline.", + "reference": "这个我们线下再聊。", + "ttft_ms": 153, + "e2e_ms": 282, + "gen_tokens": 5, + "tps": 38.7, + "prompt_tokens": 132, + "prompt_ms": 97, + "output": "我把它离线了。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.087 + }, + { + "engine": "ollama-4b-tag", + "category": "A_short_turns", + "id": "A05", + "run": 0, + "input": "Any questions before we move on?", + "reference": "在继续之前有什么问题吗？", + "ttft_ms": 143, + "e2e_ms": 338, + "gen_tokens": 8, + "tps": 41.0, + "prompt_tokens": 133, + "prompt_ms": 99, + "output": "在我们继续之前，有任何问题吗？", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.474 + }, + { + "engine": "ollama-4b-tag", + "category": "A_short_turns", + "id": "A06", + "run": 0, + "input": "That makes sense, thanks for clarifying.", + "reference": "明白了，谢谢你的说明。", + "ttft_ms": 153, + "e2e_ms": 344, + "gen_tokens": 8, + "tps": 41.9, + "prompt_tokens": 135, + "prompt_ms": 100, + "output": "这说得通，谢谢你的澄清。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.397 + }, + { + "engine": "ollama-4b-tag", + "category": "A_short_turns", + "id": "A07", + "run": 0, + "input": "We're running about five minutes behind.", + "reference": "我们大概晚了五分钟。", + "ttft_ms": 157, + "e2e_ms": 275, + "gen_tokens": 5, + "tps": 42.2, + "prompt_tokens": 134, + "prompt_ms": 100, + "output": "我们落后大约五分钟。", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 1, + "terms_missing": [], + "chrf": 0.384 + }, + { + "engine": "ollama-4b-tag", + "category": "A_short_turns", + "id": "A08", + "run": 0, + "input": "Let me share the link in the chat.", + "reference": "我把链接发到聊天里。", + "ttft_ms": 153, + "e2e_ms": 346, + "gen_tokens": 8, + "tps": 41.4, + "prompt_tokens": 135, + "prompt_ms": 98, + "output": "让我在聊天框里分享链接。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.221 + }, + { + "engine": "ollama-4b-tag", + "category": "A_short_turns", + "id": "A01", + "run": 1, + "input": "Let's get started.", + "reference": "我们开始吧。", + "ttft_ms": 85, + "e2e_ms": 184, + "gen_tokens": 4, + "tps": 40.5, + "prompt_tokens": 131, + "prompt_ms": 30, + "output": "让我们开始吧。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.955 + }, + { + "engine": "ollama-4b-tag", + "category": "A_short_turns", + "id": "A02", + "run": 1, + "input": "Can everyone see my screen?", + "reference": "大家能看到我的屏幕吗？", + "ttft_ms": 92, + "e2e_ms": 263, + "gen_tokens": 7, + "tps": 41.0, + "prompt_tokens": 132, + "prompt_ms": 32, + "output": "大家能看见我的屏幕吗？", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.719 + }, + { + "engine": "ollama-4b-tag", + "category": "A_short_turns", + "id": "A03", + "run": 1, + "input": "Sorry, you cut out for a second.", + "reference": "抱歉，你刚才断了一下。", + "ttft_ms": 90, + "e2e_ms": 241, + "gen_tokens": 6, + "tps": 39.9, + "prompt_tokens": 135, + "prompt_ms": 34, + "output": "抱歉，您中断了。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.254 + }, + { + "engine": "ollama-4b-tag", + "category": "A_short_turns", + "id": "A04", + "run": 1, + "input": "I'll take that offline.", + "reference": "这个我们线下再聊。", + "ttft_ms": 94, + "e2e_ms": 212, + "gen_tokens": 5, + "tps": 42.3, + "prompt_tokens": 132, + "prompt_ms": 39, + "output": "我把它离线了。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.087 + }, + { + "engine": "ollama-4b-tag", + "category": "A_short_turns", + "id": "A05", + "run": 1, + "input": "Any questions before we move on?", + "reference": "在继续之前有什么问题吗？", + "ttft_ms": 101, + "e2e_ms": 300, + "gen_tokens": 8, + "tps": 40.1, + "prompt_tokens": 133, + "prompt_ms": 30, + "output": "在我们继续之前，有任何问题吗？", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.474 + }, + { + "engine": "ollama-4b-tag", + "category": "A_short_turns", + "id": "A06", + "run": 1, + "input": "That makes sense, thanks for clarifying.", + "reference": "明白了，谢谢你的说明。", + "ttft_ms": 86, + "e2e_ms": 255, + "gen_tokens": 7, + "tps": 41.4, + "prompt_tokens": 135, + "prompt_ms": 30, + "output": "有道理，谢谢你的澄清。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.382 + }, + { + "engine": "ollama-4b-tag", + "category": "A_short_turns", + "id": "A07", + "run": 1, + "input": "We're running about five minutes behind.", + "reference": "我们大概晚了五分钟。", + "ttft_ms": 91, + "e2e_ms": 217, + "gen_tokens": 5, + "tps": 39.7, + "prompt_tokens": 134, + "prompt_ms": 35, + "output": "我们落后大约五分钟。", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 1, + "terms_missing": [], + "chrf": 0.384 + }, + { + "engine": "ollama-4b-tag", + "category": "A_short_turns", + "id": "A08", + "run": 1, + "input": "Let me share the link in the chat.", + "reference": "我把链接发到聊天里。", + "ttft_ms": 86, + "e2e_ms": 279, + "gen_tokens": 8, + "tps": 41.4, + "prompt_tokens": 135, + "prompt_ms": 31, + "output": "让我把链接发到聊天框里。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.801 + }, + { + "engine": "ollama-4b-tag", + "category": "B_asr_drafts", + "id": "B01", + "run": 0, + "input": "so what we found was that the", + "reference": "所以我们发现的是", + "ttft_ms": 156, + "e2e_ms": 277, + "gen_tokens": 5, + "tps": 41.2, + "prompt_tokens": 133, + "prompt_ms": 101, + "output": "所以，我们发现的是……", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.716 + }, + { + "engine": "ollama-4b-tag", + "category": "B_asr_drafts", + "id": "B02", + "run": 0, + "input": "if you look at the second chart you can see", + "reference": "如果你看第二张图，你可以看到", + "ttft_ms": 155, + "e2e_ms": 371, + "gen_tokens": 9, + "tps": 41.5, + "prompt_tokens": 136, + "prompt_ms": 101, + "output": "如果您看第二张图表，您可以看到。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.524 + }, + { + "engine": "ollama-4b-tag", + "category": "B_asr_drafts", + "id": "B03", + "run": 0, + "input": "the main reason we chose this approach is", + "reference": "我们选择这个方案的主要原因是", + "ttft_ms": 150, + "e2e_ms": 289, + "gen_tokens": 6, + "tps": 43.3, + "prompt_tokens": 134, + "prompt_ms": 98, + "output": "我们选择这一方法的主要原因在于", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.557 + }, + { + "engine": "ollama-4b-tag", + "category": "B_asr_drafts", + "id": "B04", + "run": 0, + "input": "and then the customer said that they", + "reference": "然后客户说他们", + "ttft_ms": 150, + "e2e_ms": 267, + "gen_tokens": 5, + "tps": 42.8, + "prompt_tokens": 133, + "prompt_ms": 98, + "output": "然后客户说他们……", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.93 + }, + { + "engine": "ollama-4b-tag", + "category": "B_asr_drafts", + "id": "B05", + "run": 0, + "input": "I think the biggest risk here", + "reference": "我觉得这里最大的风险", + "ttft_ms": 188, + "e2e_ms": 294, + "gen_tokens": 6, + "tps": 56.7, + "prompt_tokens": 132, + "prompt_ms": 101, + "output": "我认为这里最大的风险是……", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.622 + }, + { + "engine": "ollama-4b-tag", + "category": "B_asr_drafts", + "id": "B06", + "run": 0, + "input": "we need to decide by", + "reference": "我们需要在……之前决定", + "ttft_ms": 157, + "e2e_ms": 251, + "gen_tokens": 4, + "tps": 42.6, + "prompt_tokens": 131, + "prompt_ms": 99, + "output": "我们需要做出决定。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.337 + }, + { + "engine": "ollama-4b-tag", + "category": "B_asr_drafts", + "id": "B07", + "run": 0, + "input": "which brings me to", + "reference": "这就引出了", + "ttft_ms": 176, + "e2e_ms": 273, + "gen_tokens": 4, + "tps": 41.2, + "prompt_tokens": 130, + "prompt_ms": 102, + "output": "这使我得以……", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.046 + }, + { + "engine": "ollama-4b-tag", + "category": "B_asr_drafts", + "id": "B08", + "run": 0, + "input": "the data shows a clear", + "reference": "数据显示出明显的", + "ttft_ms": 205, + "e2e_ms": 324, + "gen_tokens": 5, + "tps": 42.2, + "prompt_tokens": 131, + "prompt_ms": 134, + "output": "数据显示，情况明显。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.438 + }, + { + "engine": "ollama-4b-tag", + "category": "B_asr_drafts", + "id": "B01", + "run": 1, + "input": "so what we found was that the", + "reference": "所以我们发现的是", + "ttft_ms": 133, + "e2e_ms": 253, + "gen_tokens": 5, + "tps": 41.7, + "prompt_tokens": 133, + "prompt_ms": 59, + "output": "所以，我们发现的是……", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.716 + }, + { + "engine": "ollama-4b-tag", + "category": "B_asr_drafts", + "id": "B02", + "run": 1, + "input": "if you look at the second chart you can see", + "reference": "如果你看第二张图，你可以看到", + "ttft_ms": 106, + "e2e_ms": 305, + "gen_tokens": 8, + "tps": 40.1, + "prompt_tokens": 136, + "prompt_ms": 36, + "output": "如果您查看第二个图表，您可以看到...", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.369 + }, + { + "engine": "ollama-4b-tag", + "category": "B_asr_drafts", + "id": "B03", + "run": 1, + "input": "the main reason we chose this approach is", + "reference": "我们选择这个方案的主要原因是", + "ttft_ms": 94, + "e2e_ms": 241, + "gen_tokens": 6, + "tps": 40.7, + "prompt_tokens": 134, + "prompt_ms": 36, + "output": "我们选择这一方法的主要原因在于", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.557 + }, + { + "engine": "ollama-4b-tag", + "category": "B_asr_drafts", + "id": "B04", + "run": 1, + "input": "and then the customer said that they", + "reference": "然后客户说他们", + "ttft_ms": 93, + "e2e_ms": 214, + "gen_tokens": 5, + "tps": 41.4, + "prompt_tokens": 133, + "prompt_ms": 31, + "output": "然后客户说他们……", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.93 + }, + { + "engine": "ollama-4b-tag", + "category": "B_asr_drafts", + "id": "B05", + "run": 1, + "input": "I think the biggest risk here", + "reference": "我觉得这里最大的风险", + "ttft_ms": 89, + "e2e_ms": 241, + "gen_tokens": 6, + "tps": 39.6, + "prompt_tokens": 132, + "prompt_ms": 31, + "output": "我认为这里最大的风险是……", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.622 + }, + { + "engine": "ollama-4b-tag", + "category": "B_asr_drafts", + "id": "B06", + "run": 1, + "input": "we need to decide by", + "reference": "我们需要在……之前决定", + "ttft_ms": 90, + "e2e_ms": 188, + "gen_tokens": 4, + "tps": 40.7, + "prompt_tokens": 131, + "prompt_ms": 32, + "output": "我们需要做出决定。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.337 + }, + { + "engine": "ollama-4b-tag", + "category": "B_asr_drafts", + "id": "B07", + "run": 1, + "input": "which brings me to", + "reference": "这就引出了", + "ttft_ms": 88, + "e2e_ms": 182, + "gen_tokens": 4, + "tps": 42.6, + "prompt_tokens": 130, + "prompt_ms": 30, + "output": "这使我得以……", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.046 + }, + { + "engine": "ollama-4b-tag", + "category": "B_asr_drafts", + "id": "B08", + "run": 1, + "input": "the data shows a clear", + "reference": "数据显示出明显的", + "ttft_ms": 90, + "e2e_ms": 140, + "gen_tokens": 2, + "tps": 39.9, + "prompt_tokens": 131, + "prompt_ms": 31, + "output": "数据显示……", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.389 + }, + { + "engine": "ollama-4b-tag", + "category": "C_asr_noisy", + "id": "C01", + "run": 0, + "input": "um so yeah basically what we're gonna do is we're gonna ship it next week", + "reference": "所以基本上我们要做的就是下周发布。", + "ttft_ms": 176, + "e2e_ms": 441, + "gen_tokens": 11, + "tps": 41.5, + "prompt_tokens": 143, + "prompt_ms": 111, + "output": "嗯，所以基本上我们要做的就是把货物下周发货。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.735 + }, + { + "engine": "ollama-4b-tag", + "category": "C_asr_noisy", + "id": "C02", + "run": 0, + "input": "i mean it's it's not perfect but it works you know", + "reference": "我是说它不完美，但能用，你懂的。", + "ttft_ms": 165, + "e2e_ms": 490, + "gen_tokens": 13, + "tps": 40.0, + "prompt_tokens": 139, + "prompt_ms": 101, + "output": "我的意思是，它不是完美的，但它能起作用，你知道。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.29 + }, + { + "engine": "ollama-4b-tag", + "category": "C_asr_noisy", + "id": "C03", + "run": 0, + "input": "okay okay let me let me just pull that up real quick", + "reference": "好好，让我快速调出来看一下。", + "ttft_ms": 157, + "e2e_ms": 413, + "gen_tokens": 9, + "tps": 35.2, + "prompt_tokens": 138, + "prompt_ms": 101, + "output": "好的，好的，让我快速查一下。", + "compliant": false, + "problems": [ + "preamble:好的" + ], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.423 + }, + { + "engine": "ollama-4b-tag", + "category": "C_asr_noisy", + "id": "C04", + "run": 0, + "input": "right so the the thing is we don't actually have the budget for that", + "reference": "对，问题是我们其实没有这方面的预算。", + "ttft_ms": 142, + "e2e_ms": 339, + "gen_tokens": 8, + "tps": 40.7, + "prompt_tokens": 141, + "prompt_ms": 101, + "output": "所以，我们实际上没有那个预算。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.207 + }, + { + "engine": "ollama-4b-tag", + "category": "C_asr_noisy", + "id": "C05", + "run": 0, + "input": "like honestly I don't think anyone has looked at this since like march", + "reference": "说实话我觉得从三月以来就没人看过这个了。", + "ttft_ms": 155, + "e2e_ms": 398, + "gen_tokens": 10, + "tps": 41.2, + "prompt_tokens": 140, + "prompt_ms": 99, + "output": "说实话，我觉得没人从三月以来看过这个。", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 1, + "terms_missing": [], + "chrf": 0.53 + }, + { + "engine": "ollama-4b-tag", + "category": "C_asr_noisy", + "id": "C06", + "run": 0, + "input": "and uh we also need to loop in the legal team before we sign anything", + "reference": "另外在签任何东西之前我们还需要让法务团队参与进来。", + "ttft_ms": 158, + "e2e_ms": 526, + "gen_tokens": 15, + "tps": 40.8, + "prompt_tokens": 141, + "prompt_ms": 102, + "output": "而且，在我们签署任何文件之前，我们还需要让法务团队介入。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.44 + }, + { + "engine": "ollama-4b-tag", + "category": "C_asr_noisy", + "id": "C07", + "run": 0, + "input": "yeah no that's fair that's totally fair", + "reference": "对，没错，这很合理。", + "ttft_ms": 156, + "e2e_ms": 378, + "gen_tokens": 9, + "tps": 40.6, + "prompt_tokens": 135, + "prompt_ms": 99, + "output": "是啊，那很公平，完全公平。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.094 + }, + { + "engine": "ollama-4b-tag", + "category": "C_asr_noisy", + "id": "C08", + "run": 0, + "input": "so hold on are we saying the deadline moved or not", + "reference": "等等，我们是说截止日期改了还是没改？", + "ttft_ms": 201, + "e2e_ms": 427, + "gen_tokens": 9, + "tps": 39.8, + "prompt_tokens": 137, + "prompt_ms": 101, + "output": "所以我们要说的是截止日期变了还是没变？", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.375 + }, + { + "engine": "ollama-4b-tag", + "category": "C_asr_noisy", + "id": "C01", + "run": 1, + "input": "um so yeah basically what we're gonna do is we're gonna ship it next week", + "reference": "所以基本上我们要做的就是下周发布。", + "ttft_ms": 90, + "e2e_ms": 357, + "gen_tokens": 11, + "tps": 41.3, + "prompt_tokens": 143, + "prompt_ms": 36, + "output": "嗯，所以基本上我们要做的就是把货物下周发货。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.735 + }, + { + "engine": "ollama-4b-tag", + "category": "C_asr_noisy", + "id": "C02", + "run": 1, + "input": "i mean it's it's not perfect but it works you know", + "reference": "我是说它不完美，但能用，你懂的。", + "ttft_ms": 162, + "e2e_ms": 480, + "gen_tokens": 13, + "tps": 40.9, + "prompt_tokens": 139, + "prompt_ms": 95, + "output": "我的意思是，它不是完美的，但它能起作用，你知道。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.29 + }, + { + "engine": "ollama-4b-tag", + "category": "C_asr_noisy", + "id": "C03", + "run": 1, + "input": "okay okay let me let me just pull that up real quick", + "reference": "好好，让我快速调出来看一下。", + "ttft_ms": 177, + "e2e_ms": 430, + "gen_tokens": 10, + "tps": 39.5, + "prompt_tokens": 138, + "prompt_ms": 116, + "output": "好的，好的，让我快速把它调出来。", + "compliant": false, + "problems": [ + "preamble:好的" + ], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.427 + }, + { + "engine": "ollama-4b-tag", + "category": "C_asr_noisy", + "id": "C04", + "run": 1, + "input": "right so the the thing is we don't actually have the budget for that", + "reference": "对，问题是我们其实没有这方面的预算。", + "ttft_ms": 555, + "e2e_ms": 749, + "gen_tokens": 8, + "tps": 41.3, + "prompt_tokens": 141, + "prompt_ms": 476, + "output": "所以，我们实际上没有那个预算。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.207 + }, + { + "engine": "ollama-4b-tag", + "category": "C_asr_noisy", + "id": "C05", + "run": 1, + "input": "like honestly I don't think anyone has looked at this since like march", + "reference": "说实话我觉得从三月以来就没人看过这个了。", + "ttft_ms": 90, + "e2e_ms": 336, + "gen_tokens": 10, + "tps": 40.7, + "prompt_tokens": 140, + "prompt_ms": 36, + "output": "说实话，我觉得自三月以来没人看过这个。", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 1, + "terms_missing": [], + "chrf": 0.546 + }, + { + "engine": "ollama-4b-tag", + "category": "C_asr_noisy", + "id": "C06", + "run": 1, + "input": "and uh we also need to loop in the legal team before we sign anything", + "reference": "另外在签任何东西之前我们还需要让法务团队参与进来。", + "ttft_ms": 124, + "e2e_ms": 452, + "gen_tokens": 13, + "tps": 39.7, + "prompt_tokens": 141, + "prompt_ms": 66, + "output": "而且我们还需要在签署任何文件之前让法务团队介入。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.399 + }, + { + "engine": "ollama-4b-tag", + "category": "C_asr_noisy", + "id": "C07", + "run": 1, + "input": "yeah no that's fair that's totally fair", + "reference": "对，没错，这很合理。", + "ttft_ms": 112, + "e2e_ms": 335, + "gen_tokens": 9, + "tps": 40.3, + "prompt_tokens": 135, + "prompt_ms": 54, + "output": "是啊，那很公平，完全没问题。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.116 + }, + { + "engine": "ollama-4b-tag", + "category": "C_asr_noisy", + "id": "C08", + "run": 1, + "input": "so hold on are we saying the deadline moved or not", + "reference": "等等，我们是说截止日期改了还是没改？", + "ttft_ms": 94, + "e2e_ms": 334, + "gen_tokens": 10, + "tps": 41.6, + "prompt_tokens": 137, + "prompt_ms": 33, + "output": "所以我们要说的是截止日期已变动还是没变？", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.311 + }, + { + "engine": "ollama-4b-tag", + "category": "D_numbers_names", + "id": "D01", + "run": 0, + "input": "Revenue grew 12% year over year to $4.3 billion.", + "reference": "收入同比增长12%，达到43亿美元。", + "ttft_ms": 157, + "e2e_ms": 465, + "gen_tokens": 13, + "tps": 42.2, + "prompt_tokens": 142, + "prompt_ms": 100, + "output": "收入同比增长 12%，达到 43 亿美元。", + "compliant": true, + "problems": [], + "terms_total": 2, + "terms_kept": 2, + "terms_missing": [], + "chrf": 1.0 + }, + { + "engine": "ollama-4b-tag", + "category": "D_numbers_names", + "id": "D02", + "run": 0, + "input": "Sarah from Stripe will join us on March 15th at 3 p.m.", + "reference": "Stripe的Sarah将在3月15日下午3点加入我们。", + "ttft_ms": 164, + "e2e_ms": 714, + "gen_tokens": 23, + "tps": 41.8, + "prompt_tokens": 144, + "prompt_ms": 104, + "output": "来自 Stripe 的 Sarah 将于 3 月 15 日下午 3 点加入我们。", + "compliant": true, + "problems": [], + "terms_total": 3, + "terms_kept": 3, + "terms_missing": [], + "chrf": 0.89 + }, + { + "engine": "ollama-4b-tag", + "category": "D_numbers_names", + "id": "D03", + "run": 0, + "input": "We processed 2.7 million requests with a p99 latency of 180 milliseconds.", + "reference": "我们处理了270万个请求，p99延迟为180毫秒。", + "ttft_ms": 232, + "e2e_ms": 805, + "gen_tokens": 24, + "tps": 41.9, + "prompt_tokens": 147, + "prompt_ms": 103, + "output": "我们处理了 270 万请求，p99 延迟为 180 毫秒。", + "compliant": true, + "problems": [], + "terms_total": 3, + "terms_kept": 3, + "terms_missing": [], + "chrf": 0.899 + }, + { + "engine": "ollama-4b-tag", + "category": "D_numbers_names", + "id": "D04", + "run": 0, + "input": "Microsoft and OpenAI announced the partnership in Q2 2024.", + "reference": "Microsoft和OpenAI在2024年第二季度宣布了合作。", + "ttft_ms": 152, + "e2e_ms": 590, + "gen_tokens": 18, + "tps": 41.1, + "prompt_tokens": 142, + "prompt_ms": 99, + "output": "微软和 OpenAI 于 2024 年第二季度宣布了合作关系。", + "compliant": true, + "problems": [], + "terms_total": 3, + "terms_kept": 3, + "terms_missing": [], + "chrf": 0.616 + }, + { + "engine": "ollama-4b-tag", + "category": "D_numbers_names", + "id": "D05", + "run": 0, + "input": "The budget is 850 thousand euros, split 60-40 between the two teams.", + "reference": "预算是85万欧元，两个团队按60比40分配。", + "ttft_ms": 239, + "e2e_ms": 732, + "gen_tokens": 20, + "tps": 40.6, + "prompt_tokens": 148, + "prompt_ms": 180, + "output": "预算为 85 万欧元，两个团队按 60:40 分配。", + "compliant": true, + "problems": [], + "terms_total": 3, + "terms_kept": 3, + "terms_missing": [], + "chrf": 0.763 + }, + { + "engine": "ollama-4b-tag", + "category": "D_numbers_names", + "id": "D06", + "run": 0, + "input": "Dr. Nakamura's team at Stanford published the results on arXiv last Tuesday.", + "reference": "Nakamura博士在斯坦福的团队上周二在arXiv上发表了结果。", + "ttft_ms": 156, + "e2e_ms": 572, + "gen_tokens": 17, + "tps": 40.9, + "prompt_tokens": 144, + "prompt_ms": 104, + "output": "斯坦福大学中村博士的团队于上周二在 arXiv 发布了研究成果。", + "compliant": true, + "problems": [], + "terms_total": 2, + "terms_kept": 2, + "terms_missing": [], + "chrf": 0.399 + }, + { + "engine": "ollama-4b-tag", + "category": "D_numbers_names", + "id": "D07", + "run": 0, + "input": "Version 3.2.1 fixes the bug reported in ticket LCT-418.", + "reference": "3.2.1版本修复了工单LCT-418中报告的问题。", + "ttft_ms": 165, + "e2e_ms": 758, + "gen_tokens": 23, + "tps": 38.7, + "prompt_tokens": 146, + "prompt_ms": 107, + "output": "版本 3.2.1 修复了工单 LCT-418 中报告的错误。", + "compliant": true, + "problems": [], + "terms_total": 2, + "terms_kept": 2, + "terms_missing": [], + "chrf": 0.771 + }, + { + "engine": "ollama-4b-tag", + "category": "D_numbers_names", + "id": "D08", + "run": 0, + "input": "Our NPS went from 31 to 47 after the redesign.", + "reference": "改版后我们的NPS从31升到了47。", + "ttft_ms": 159, + "e2e_ms": 605, + "gen_tokens": 18, + "tps": 40.4, + "prompt_tokens": 142, + "prompt_ms": 101, + "output": "我们的 NPS 在重新设计后从 31 提升至 47。", + "compliant": true, + "problems": [], + "terms_total": 3, + "terms_kept": 3, + "terms_missing": [], + "chrf": 0.454 + }, + { + "engine": "ollama-4b-tag", + "category": "D_numbers_names", + "id": "D01", + "run": 1, + "input": "Revenue grew 12% year over year to $4.3 billion.", + "reference": "收入同比增长12%，达到43亿美元。", + "ttft_ms": 104, + "e2e_ms": 427, + "gen_tokens": 13, + "tps": 40.3, + "prompt_tokens": 142, + "prompt_ms": 44, + "output": "收入同比增长 12%，达到 43 亿美元。", + "compliant": true, + "problems": [], + "terms_total": 2, + "terms_kept": 2, + "terms_missing": [], + "chrf": 1.0 + }, + { + "engine": "ollama-4b-tag", + "category": "D_numbers_names", + "id": "D02", + "run": 1, + "input": "Sarah from Stripe will join us on March 15th at 3 p.m.", + "reference": "Stripe的Sarah将在3月15日下午3点加入我们。", + "ttft_ms": 161, + "e2e_ms": 702, + "gen_tokens": 22, + "tps": 40.7, + "prompt_tokens": 144, + "prompt_ms": 99, + "output": "来自 Stripe 的 Sarah 将于 3 月 15 日下午 3 点加入我们。", + "compliant": true, + "problems": [], + "terms_total": 3, + "terms_kept": 3, + "terms_missing": [], + "chrf": 0.89 + }, + { + "engine": "ollama-4b-tag", + "category": "D_numbers_names", + "id": "D03", + "run": 1, + "input": "We processed 2.7 million requests with a p99 latency of 180 milliseconds.", + "reference": "我们处理了270万个请求，p99延迟为180毫秒。", + "ttft_ms": 402, + "e2e_ms": 987, + "gen_tokens": 24, + "tps": 41.0, + "prompt_tokens": 147, + "prompt_ms": 331, + "output": "我们处理了 270 万请求，p99 延迟为 180 毫秒。", + "compliant": true, + "problems": [], + "terms_total": 3, + "terms_kept": 3, + "terms_missing": [], + "chrf": 0.899 + }, + { + "engine": "ollama-4b-tag", + "category": "D_numbers_names", + "id": "D04", + "run": 1, + "input": "Microsoft and OpenAI announced the partnership in Q2 2024.", + "reference": "Microsoft和OpenAI在2024年第二季度宣布了合作。", + "ttft_ms": 165, + "e2e_ms": 589, + "gen_tokens": 18, + "tps": 42.4, + "prompt_tokens": 142, + "prompt_ms": 89, + "output": "微软和 OpenAI 于 2024 年第二季度宣布了这一合作。", + "compliant": true, + "problems": [], + "terms_total": 3, + "terms_kept": 3, + "terms_missing": [], + "chrf": 0.59 + }, + { + "engine": "ollama-4b-tag", + "category": "D_numbers_names", + "id": "D05", + "run": 1, + "input": "The budget is 850 thousand euros, split 60-40 between the two teams.", + "reference": "预算是85万欧元，两个团队按60比40分配。", + "ttft_ms": 123, + "e2e_ms": 602, + "gen_tokens": 20, + "tps": 41.8, + "prompt_tokens": 148, + "prompt_ms": 63, + "output": "预算为 85 万欧元，两个团队按 60:40 分配。", + "compliant": true, + "problems": [], + "terms_total": 3, + "terms_kept": 3, + "terms_missing": [], + "chrf": 0.763 + }, + { + "engine": "ollama-4b-tag", + "category": "D_numbers_names", + "id": "D06", + "run": 1, + "input": "Dr. Nakamura's team at Stanford published the results on arXiv last Tuesday.", + "reference": "Nakamura博士在斯坦福的团队上周二在arXiv上发表了结果。", + "ttft_ms": 197, + "e2e_ms": 735, + "gen_tokens": 22, + "tps": 40.9, + "prompt_tokens": 144, + "prompt_ms": 129, + "output": "Nakamura 博士在斯坦福大学的团队于上周二将结果发布在 arXiv 上。", + "compliant": true, + "problems": [], + "terms_total": 2, + "terms_kept": 2, + "terms_missing": [], + "chrf": 0.688 + }, + { + "engine": "ollama-4b-tag", + "category": "D_numbers_names", + "id": "D07", + "run": 1, + "input": "Version 3.2.1 fixes the bug reported in ticket LCT-418.", + "reference": "3.2.1版本修复了工单LCT-418中报告的问题。", + "ttft_ms": 107, + "e2e_ms": 661, + "gen_tokens": 23, + "tps": 41.5, + "prompt_tokens": 146, + "prompt_ms": 50, + "output": "版本 3.2.1 修复了工单 LCT-418 中报告的错误。", + "compliant": true, + "problems": [], + "terms_total": 2, + "terms_kept": 2, + "terms_missing": [], + "chrf": 0.771 + }, + { + "engine": "ollama-4b-tag", + "category": "D_numbers_names", + "id": "D08", + "run": 1, + "input": "Our NPS went from 31 to 47 after the redesign.", + "reference": "改版后我们的NPS从31升到了47。", + "ttft_ms": 121, + "e2e_ms": 588, + "gen_tokens": 20, + "tps": 42.8, + "prompt_tokens": 142, + "prompt_ms": 66, + "output": "我们的净推荐值从 31 提升到 47，这是经过重新设计之后。", + "compliant": true, + "problems": [], + "terms_total": 3, + "terms_kept": 2, + "terms_missing": [ + "NPS" + ], + "chrf": 0.259 + }, + { + "engine": "ollama-4b-tag", + "category": "E_tech_terms", + "id": "E01", + "run": 0, + "input": "We migrated the service from Kubernetes to a single Rust binary on bare metal.", + "reference": "我们把这个服务从Kubernetes迁移到了裸金属上的单个Rust二进制。", + "ttft_ms": 534, + "e2e_ms": 930, + "gen_tokens": 16, + "tps": 40.4, + "prompt_tokens": 141, + "prompt_ms": 478, + "output": "我们将服务从 Kubernetes 迁移到了裸金属上的单一 Rust 二进制文件。", + "compliant": true, + "problems": [], + "terms_total": 2, + "terms_kept": 2, + "terms_missing": [], + "chrf": 0.787 + }, + { + "engine": "ollama-4b-tag", + "category": "E_tech_terms", + "id": "E02", + "run": 0, + "input": "The model runs fully on-device with Metal acceleration and a 4-bit quantized Qwen checkpoint.", + "reference": "模型完全在设备端运行，使用Metal加速和4位量化的Qwen检查点。", + "ttft_ms": 242, + "e2e_ms": 801, + "gen_tokens": 23, + "tps": 41.1, + "prompt_tokens": 146, + "prompt_ms": 184, + "output": "该模型完全在设备上运行，采用 Metal 加速和 4 位量化 Qwen 检查点。", + "compliant": true, + "problems": [], + "terms_total": 3, + "terms_kept": 3, + "terms_missing": [], + "chrf": 0.765 + }, + { + "engine": "ollama-4b-tag", + "category": "E_tech_terms", + "id": "E03", + "run": 0, + "input": "Enable WASAPI loopback so we capture system audio without a virtual device.", + "reference": "启用WASAPI环回，这样不需要虚拟设备也能采集系统音频。", + "ttft_ms": 160, + "e2e_ms": 592, + "gen_tokens": 18, + "tps": 41.6, + "prompt_tokens": 141, + "prompt_ms": 104, + "output": "启用 WASAPI 回环模式，以便在不使用虚拟设备的情况下捕获系统音频。", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 1, + "terms_missing": [], + "chrf": 0.468 + }, + { + "engine": "ollama-4b-tag", + "category": "E_tech_terms", + "id": "E04", + "run": 0, + "input": "The API returns a 429 when you exceed the rate limit, so add exponential backoff.", + "reference": "超过速率限制时API会返回429，所以要加指数退避。", + "ttft_ms": 160, + "e2e_ms": 645, + "gen_tokens": 20, + "tps": 41.2, + "prompt_tokens": 147, + "prompt_ms": 104, + "output": "API 在超出速率限制时返回 429，因此请添加指数退避。", + "compliant": true, + "problems": [], + "terms_total": 2, + "terms_kept": 2, + "terms_missing": [], + "chrf": 0.569 + }, + { + "engine": "ollama-4b-tag", + "category": "E_tech_terms", + "id": "E05", + "run": 0, + "input": "We're seeing GPU memory pressure because num_ctx defaults to 32K.", + "reference": "我们遇到了GPU内存压力，因为num_ctx默认是32K。", + "ttft_ms": 176, + "e2e_ms": 607, + "gen_tokens": 17, + "tps": 39.4, + "prompt_tokens": 142, + "prompt_ms": 103, + "output": "我们看到 GPU 内存压力，因为 num_ctx 默认为 32K。", + "compliant": true, + "problems": [], + "terms_total": 3, + "terms_kept": 3, + "terms_missing": [], + "chrf": 0.766 + }, + { + "engine": "ollama-4b-tag", + "category": "E_tech_terms", + "id": "E06", + "run": 0, + "input": "The PR adds a TranslationEngine protocol with an Ollama adapter and an embedded llama.cpp runtime.", + "reference": "这个PR增加了TranslationEngine协议，带有一个Ollama适配器和一个内置的llama.cpp运行时。", + "ttft_ms": 232, + "e2e_ms": 787, + "gen_tokens": 22, + "tps": 39.7, + "prompt_tokens": 146, + "prompt_ms": 120, + "output": "PR 添加了 TranslationEngine 协议，包含 Ollama 适配器以及嵌入式的 llama.cpp 运行时。", + "compliant": true, + "problems": [], + "terms_total": 4, + "terms_kept": 4, + "terms_missing": [], + "chrf": 0.726 + }, + { + "engine": "ollama-4b-tag", + "category": "E_tech_terms", + "id": "E07", + "run": 0, + "input": "Notarization requires a Developer ID certificate and the hardened runtime.", + "reference": "公证需要Developer ID证书和强化运行时。", + "ttft_ms": 181, + "e2e_ms": 435, + "gen_tokens": 10, + "tps": 39.3, + "prompt_tokens": 139, + "prompt_ms": 105, + "output": "公证需要开发者 ID 证书和加固运行时。", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 0, + "terms_missing": [ + "Developer ID" + ], + "chrf": 0.392 + }, + { + "engine": "ollama-4b-tag", + "category": "E_tech_terms", + "id": "E08", + "run": 0, + "input": "Our CI runs on windows-latest, which no longer ships NSIS.", + "reference": "我们的CI跑在windows-latest上，它不再自带NSIS。", + "ttft_ms": 170, + "e2e_ms": 569, + "gen_tokens": 16, + "tps": 40.1, + "prompt_tokens": 141, + "prompt_ms": 106, + "output": "我们的 CI 运行在 windows-latest 上，不再分发 NSIS。", + "compliant": true, + "problems": [], + "terms_total": 3, + "terms_kept": 3, + "terms_missing": [], + "chrf": 0.734 + }, + { + "engine": "ollama-4b-tag", + "category": "E_tech_terms", + "id": "E01", + "run": 1, + "input": "We migrated the service from Kubernetes to a single Rust binary on bare metal.", + "reference": "我们把这个服务从Kubernetes迁移到了裸金属上的单个Rust二进制。", + "ttft_ms": 309, + "e2e_ms": 708, + "gen_tokens": 16, + "tps": 40.1, + "prompt_tokens": 141, + "prompt_ms": 242, + "output": "我们将服务从 Kubernetes 迁移到了裸金属上的单一 Rust 二进制文件。", + "compliant": true, + "problems": [], + "terms_total": 2, + "terms_kept": 2, + "terms_missing": [], + "chrf": 0.787 + }, + { + "engine": "ollama-4b-tag", + "category": "E_tech_terms", + "id": "E02", + "run": 1, + "input": "The model runs fully on-device with Metal acceleration and a 4-bit quantized Qwen checkpoint.", + "reference": "模型完全在设备端运行，使用Metal加速和4位量化的Qwen检查点。", + "ttft_ms": 169, + "e2e_ms": 722, + "gen_tokens": 23, + "tps": 41.6, + "prompt_tokens": 146, + "prompt_ms": 111, + "output": "该模型完全在设备上运行，利用 Metal 加速和 4 位量化 Qwen 检查点。", + "compliant": true, + "problems": [], + "terms_total": 3, + "terms_kept": 3, + "terms_missing": [], + "chrf": 0.765 + }, + { + "engine": "ollama-4b-tag", + "category": "E_tech_terms", + "id": "E03", + "run": 1, + "input": "Enable WASAPI loopback so we capture system audio without a virtual device.", + "reference": "启用WASAPI环回，这样不需要虚拟设备也能采集系统音频。", + "ttft_ms": 122, + "e2e_ms": 544, + "gen_tokens": 17, + "tps": 40.4, + "prompt_tokens": 141, + "prompt_ms": 64, + "output": "启用 WASAPI 回环，以便在不使用虚拟设备的情况下捕获系统音频。", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 1, + "terms_missing": [], + "chrf": 0.475 + }, + { + "engine": "ollama-4b-tag", + "category": "E_tech_terms", + "id": "E04", + "run": 1, + "input": "The API returns a 429 when you exceed the rate limit, so add exponential backoff.", + "reference": "超过速率限制时API会返回429，所以要加指数退避。", + "ttft_ms": 357, + "e2e_ms": 843, + "gen_tokens": 20, + "tps": 41.2, + "prompt_tokens": 147, + "prompt_ms": 283, + "output": "API 在超过速率限制时返回 429，因此请添加指数退避。", + "compliant": true, + "problems": [], + "terms_total": 2, + "terms_kept": 2, + "terms_missing": [], + "chrf": 0.641 + }, + { + "engine": "ollama-4b-tag", + "category": "E_tech_terms", + "id": "E05", + "run": 1, + "input": "We're seeing GPU memory pressure because num_ctx defaults to 32K.", + "reference": "我们遇到了GPU内存压力，因为num_ctx默认是32K。", + "ttft_ms": 161, + "e2e_ms": 580, + "gen_tokens": 17, + "tps": 40.6, + "prompt_tokens": 142, + "prompt_ms": 104, + "output": "我们看到 GPU 内存压力，因为 num_ctx 默认为 32K。", + "compliant": true, + "problems": [], + "terms_total": 3, + "terms_kept": 3, + "terms_missing": [], + "chrf": 0.766 + }, + { + "engine": "ollama-4b-tag", + "category": "E_tech_terms", + "id": "E06", + "run": 1, + "input": "The PR adds a TranslationEngine protocol with an Ollama adapter and an embedded llama.cpp runtime.", + "reference": "这个PR增加了TranslationEngine协议，带有一个Ollama适配器和一个内置的llama.cpp运行时。", + "ttft_ms": 278, + "e2e_ms": 803, + "gen_tokens": 22, + "tps": 41.9, + "prompt_tokens": 146, + "prompt_ms": 219, + "output": "PR 添加了 TranslationEngine 协议，包含 Ollama 适配器以及嵌入的 llama.cpp 运行时。", + "compliant": true, + "problems": [], + "terms_total": 4, + "terms_kept": 4, + "terms_missing": [], + "chrf": 0.729 + }, + { + "engine": "ollama-4b-tag", + "category": "E_tech_terms", + "id": "E07", + "run": 1, + "input": "Notarization requires a Developer ID certificate and the hardened runtime.", + "reference": "公证需要Developer ID证书和强化运行时。", + "ttft_ms": 87, + "e2e_ms": 333, + "gen_tokens": 10, + "tps": 40.6, + "prompt_tokens": 139, + "prompt_ms": 32, + "output": "公证需要开发者 ID 证书和加固运行时。", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 0, + "terms_missing": [ + "Developer ID" + ], + "chrf": 0.392 + }, + { + "engine": "ollama-4b-tag", + "category": "E_tech_terms", + "id": "E08", + "run": 1, + "input": "Our CI runs on windows-latest, which no longer ships NSIS.", + "reference": "我们的CI跑在windows-latest上，它不再自带NSIS。", + "ttft_ms": 100, + "e2e_ms": 488, + "gen_tokens": 16, + "tps": 41.2, + "prompt_tokens": 141, + "prompt_ms": 39, + "output": "我们的 CI 运行在 windows-latest 上，不再分发 NSIS。", + "compliant": true, + "problems": [], + "terms_total": 3, + "terms_kept": 3, + "terms_missing": [], + "chrf": 0.734 + }, + { + "engine": "ollama-4b-tag", + "category": "F_long_sentences", + "id": "F01", + "run": 0, + "input": "What I want to emphasize today is that the transition to on-device inference is not primarily about cost, it's about latency and privacy, and those two things are what actually change user behavior.", + "reference": "我今天想强调的是，向设备端推理的转变主要不是为了成本，而是为了延迟和隐私，而这两点才是真正改变用户行为的因素。", + "ttft_ms": 536, + "e2e_ms": 1299, + "gen_tokens": 31, + "tps": 40.6, + "prompt_tokens": 164, + "prompt_ms": 456, + "output": "今天我想要强调的是，向端侧推理的转型主要并非出于成本考量，而是关乎延迟与隐私；而这二者才是真正改变用户行为的关键。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.432 + }, + { + "engine": "ollama-4b-tag", + "category": "F_long_sentences", + "id": "F02", + "run": 0, + "input": "Before we commit to the Q3 roadmap, I'd like each team lead to send me a one-page summary of their top three risks and what support they need from leadership to mitigate them.", + "reference": "在我们确定第三季度路线图之前，我希望每位团队负责人给我发一页纸的摘要，说明他们最主要的三个风险，以及需要管理层提供什么支持来缓解这些风险。", + "ttft_ms": 240, + "e2e_ms": 1140, + "gen_tokens": 37, + "tps": 41.1, + "prompt_tokens": 164, + "prompt_ms": 182, + "output": "在我们承诺第三季度路线图之前，我希望每位团队负责人向我发送一份单页总结，列出他们面临的三大主要风险，并说明他们需要哪些来自领导层的支持来降低这些风险。", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 1, + "terms_missing": [], + "chrf": 0.481 + }, + { + "engine": "ollama-4b-tag", + "category": "F_long_sentences", + "id": "F03", + "run": 0, + "input": "If the benchmark shows that the embedded runtime is within twenty percent of the Ollama path on first-token latency, I'm comfortable making it the default and keeping Ollama as an advanced option.", + "reference": "如果基准测试显示内置运行时的首字延迟与Ollama路径相差在百分之二十以内，我可以接受把它设为默认，并把Ollama保留为高级选项。", + "ttft_ms": 242, + "e2e_ms": 1259, + "gen_tokens": 41, + "tps": 40.3, + "prompt_tokens": 166, + "prompt_ms": 184, + "output": "如果基准测试显示，在首字延迟方面，嵌入式运行时处于 Ollama 路径的百分之二十以内，我就可以将其设为默认选项，并将 Ollama 保留为高级选项。", + "compliant": true, + "problems": [], + "terms_total": 2, + "terms_kept": 2, + "terms_missing": [], + "chrf": 0.631 + }, + { + "engine": "ollama-4b-tag", + "category": "F_long_sentences", + "id": "F04", + "run": 0, + "input": "The reason the microphone channel picks up the speaker audio is simply acoustic feedback, so with headphones the duplicate lines disappear, but we should still say that in onboarding.", + "reference": "麦克风通道会拾取扬声器声音的原因就是声学回授，所以戴上耳机重复的字幕就会消失，但我们仍然应该在引导里说明这一点。", + "ttft_ms": 554, + "e2e_ms": 1418, + "gen_tokens": 33, + "tps": 38.2, + "prompt_tokens": 159, + "prompt_ms": 377, + "output": "麦克风通道拾取到说话人音频的原因仅仅是声学反馈，因此佩戴耳机后重复的杂音就会消失，但我们仍应在入职培训中说明这一点。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.411 + }, + { + "engine": "ollama-4b-tag", + "category": "F_long_sentences", + "id": "F05", + "run": 0, + "input": "Historically every attempt to bundle a second inference server inside a desktop app has failed on maintenance cost, so the question is whether a statically linked library changes that calculus.", + "reference": "历史上每一次在桌面应用里捆绑第二个推理服务的尝试都败在了维护成本上，所以问题在于静态链接的库是否改变了这个算计。", + "ttft_ms": 254, + "e2e_ms": 1034, + "gen_tokens": 32, + "tps": 41.0, + "prompt_tokens": 159, + "prompt_ms": 182, + "output": "历史上，每次尝试将第二个推理服务器捆绑到桌面应用中，都因维护成本而失败；因此问题是：静态链接的库是否会改变这一计算。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.437 + }, + { + "engine": "ollama-4b-tag", + "category": "F_long_sentences", + "id": "F06", + "run": 0, + "input": "Let's park the pricing discussion, because until we know whether enterprise customers actually need SSO and audit logs, any number we put on the slide is going to be a guess.", + "reference": "我们先把定价讨论放一放，因为在知道企业客户是否真的需要SSO和审计日志之前，我们放在幻灯片上的任何数字都只是猜测。", + "ttft_ms": 665, + "e2e_ms": 1505, + "gen_tokens": 34, + "tps": 40.5, + "prompt_tokens": 162, + "prompt_ms": 584, + "output": "让我们把定价讨论放一放，因为在我们还不知道企业客户是否真正需要单点登录和审计日志之前，幻灯片上列出的任何数字都只能是猜测。", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 0, + "terms_missing": [ + "SSO" + ], + "chrf": 0.659 + }, + { + "engine": "ollama-4b-tag", + "category": "F_long_sentences", + "id": "F01", + "run": 1, + "input": "What I want to emphasize today is that the transition to on-device inference is not primarily about cost, it's about latency and privacy, and those two things are what actually change user behavior.", + "reference": "我今天想强调的是，向设备端推理的转变主要不是为了成本，而是为了延迟和隐私，而这两点才是真正改变用户行为的因素。", + "ttft_ms": 588, + "e2e_ms": 1390, + "gen_tokens": 32, + "tps": 39.9, + "prompt_tokens": 164, + "prompt_ms": 447, + "output": "我今天想强调的是，向端侧推理的转型主要并非出于成本考量，而是关乎延迟与隐私；这两者才是真正改变用户行为的关键因素。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.507 + }, + { + "engine": "ollama-4b-tag", + "category": "F_long_sentences", + "id": "F02", + "run": 1, + "input": "Before we commit to the Q3 roadmap, I'd like each team lead to send me a one-page summary of their top three risks and what support they need from leadership to mitigate them.", + "reference": "在我们确定第三季度路线图之前，我希望每位团队负责人给我发一页纸的摘要，说明他们最主要的三个风险，以及需要管理层提供什么支持来缓解这些风险。", + "ttft_ms": 332, + "e2e_ms": 1153, + "gen_tokens": 35, + "tps": 42.6, + "prompt_tokens": 164, + "prompt_ms": 268, + "output": "在我们承诺第三季度路线图之前，我希望每位团队负责人向我发送一份一页纸的摘要，列出他们面临的三大主要风险，以及他们需要管理层提供何种支持以进行缓解。", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 1, + "terms_missing": [], + "chrf": 0.572 + }, + { + "engine": "ollama-4b-tag", + "category": "F_long_sentences", + "id": "F03", + "run": 1, + "input": "If the benchmark shows that the embedded runtime is within twenty percent of the Ollama path on first-token latency, I'm comfortable making it the default and keeping Ollama as an advanced option.", + "reference": "如果基准测试显示内置运行时的首字延迟与Ollama路径相差在百分之二十以内，我可以接受把它设为默认，并把Ollama保留为高级选项。", + "ttft_ms": 206, + "e2e_ms": 1271, + "gen_tokens": 45, + "tps": 42.2, + "prompt_tokens": 166, + "prompt_ms": 133, + "output": "如果基准测试显示，在首令牌延迟方面，嵌入式运行时距离 Ollama 路径的偏差在百分之二十以内，我就可以放心地将其设为默认选项，并将 Ollama 保留为高级选项。", + "compliant": true, + "problems": [], + "terms_total": 2, + "terms_kept": 2, + "terms_missing": [], + "chrf": 0.622 + }, + { + "engine": "ollama-4b-tag", + "category": "F_long_sentences", + "id": "F04", + "run": 1, + "input": "The reason the microphone channel picks up the speaker audio is simply acoustic feedback, so with headphones the duplicate lines disappear, but we should still say that in onboarding.", + "reference": "麦克风通道会拾取扬声器声音的原因就是声学回授，所以戴上耳机重复的字幕就会消失，但我们仍然应该在引导里说明这一点。", + "ttft_ms": 136, + "e2e_ms": 941, + "gen_tokens": 33, + "tps": 41.0, + "prompt_tokens": 159, + "prompt_ms": 72, + "output": "麦克风通道拾取到说话人音频的原因仅仅是声学反馈，因此佩戴耳机后重复的杂音就会消失，但我们仍应在入职培训中说明这一点。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.411 + }, + { + "engine": "ollama-4b-tag", + "category": "F_long_sentences", + "id": "F05", + "run": 1, + "input": "Historically every attempt to bundle a second inference server inside a desktop app has failed on maintenance cost, so the question is whether a statically linked library changes that calculus.", + "reference": "历史上每一次在桌面应用里捆绑第二个推理服务的尝试都败在了维护成本上，所以问题在于静态链接的库是否改变了这个算计。", + "ttft_ms": 351, + "e2e_ms": 1162, + "gen_tokens": 33, + "tps": 40.7, + "prompt_tokens": 159, + "prompt_ms": 205, + "output": "历史上，每次试图将第二个推理服务器打包进桌面应用时，因维护成本问题均告失败，因此问题是：静态链接库是否能改变这一计算。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.363 + }, + { + "engine": "ollama-4b-tag", + "category": "F_long_sentences", + "id": "F06", + "run": 1, + "input": "Let's park the pricing discussion, because until we know whether enterprise customers actually need SSO and audit logs, any number we put on the slide is going to be a guess.", + "reference": "我们先把定价讨论放一放，因为在知道企业客户是否真的需要SSO和审计日志之前，我们放在幻灯片上的任何数字都只是猜测。", + "ttft_ms": 139, + "e2e_ms": 963, + "gen_tokens": 34, + "tps": 41.2, + "prompt_tokens": 162, + "prompt_ms": 59, + "output": "让我们先把定价讨论放一放，因为在我们不知道企业客户是否真正需要单点登录和审计日志之前，幻灯片上列出的任何数字都只能是猜测。", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 0, + "terms_missing": [ + "SSO" + ], + "chrf": 0.699 + }, + { + "engine": "ollama-4b-tag", + "category": "G_zh_to_en", + "id": "G01", + "run": 0, + "input": "这个功能我们下周三之前能上线吗", + "reference": "Can we ship this feature before next Wednesday?", + "ttft_ms": 509, + "e2e_ms": 731, + "gen_tokens": 9, + "tps": 40.6, + "prompt_tokens": 132, + "prompt_ms": 431, + "output": "Can this feature be launched before next Wednesday?", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 1, + "terms_missing": [], + "chrf": 0.793 + }, + { + "engine": "ollama-4b-tag", + "category": "G_zh_to_en", + "id": "G02", + "run": 0, + "input": "那个就是说我们现在有两个方案一个是自己做一个是买第三方的", + "reference": "So we have two options now: build it ourselves, or buy a third-party solution.", + "ttft_ms": 510, + "e2e_ms": 1187, + "gen_tokens": 28, + "tps": 41.3, + "prompt_tokens": 135, + "prompt_ms": 351, + "output": "That's to say we currently have two options: one is to build it ourselves, and the other is to purchase from a third party.", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 1, + "terms_missing": [], + "chrf": 0.586 + }, + { + "engine": "ollama-4b-tag", + "category": "G_zh_to_en", + "id": "G03", + "run": 0, + "input": "上个季度我们的留存率从百分之三十八涨到了百分之四十五", + "reference": "Last quarter our retention rate went from 38% to 45%.", + "ttft_ms": 162, + "e2e_ms": 605, + "gen_tokens": 18, + "tps": 40.6, + "prompt_tokens": 137, + "prompt_ms": 103, + "output": "Last quarter, our retention rate rose from 38 percent to 45 percent.", + "compliant": true, + "problems": [], + "terms_total": 2, + "terms_kept": 2, + "terms_missing": [], + "chrf": 0.74 + }, + { + "engine": "ollama-4b-tag", + "category": "G_zh_to_en", + "id": "G04", + "run": 0, + "input": "我觉得这个风险有点大，要不我们先小范围灰度一下", + "reference": "I think the risk is a bit high. How about a small-scale gradual rollout first?", + "ttft_ms": 162, + "e2e_ms": 639, + "gen_tokens": 19, + "tps": 39.9, + "prompt_tokens": 136, + "prompt_ms": 102, + "output": "I think this risk is quite high; let's start with a small-scale gray release first.", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.552 + }, + { + "engine": "ollama-4b-tag", + "category": "G_zh_to_en", + "id": "G05", + "run": 0, + "input": "张总说预算最多给到五十万，超了要重新审批", + "reference": "Mr. Zhang said the budget is capped at 500 thousand; anything above that needs re-approval.", + "ttft_ms": 163, + "e2e_ms": 730, + "gen_tokens": 24, + "tps": 42.3, + "prompt_tokens": 138, + "prompt_ms": 108, + "output": "Mr. Zhang said the budget cap is 500,000; any excess requires re-approval.", + "compliant": true, + "problems": [], + "terms_total": 2, + "terms_kept": 2, + "terms_missing": [], + "chrf": 0.535 + }, + { + "engine": "ollama-4b-tag", + "category": "G_zh_to_en", + "id": "G06", + "run": 0, + "input": "识别延迟大概一点五秒，翻译再加一秒左右，用户基本感觉不到", + "reference": "Recognition latency is about 1.5 seconds, translation adds roughly another second, so users barely notice.", + "ttft_ms": 247, + "e2e_ms": 856, + "gen_tokens": 25, + "tps": 41.1, + "prompt_tokens": 139, + "prompt_ms": 112, + "output": "The recognition delay is approximately 1.5 seconds; translation adds another second or so, and users generally don't notice.", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 1, + "terms_missing": [], + "chrf": 0.667 + }, + { + "engine": "ollama-4b-tag", + "category": "G_zh_to_en", + "id": "G07", + "run": 0, + "input": "等一下我这边网不太好你们先聊", + "reference": "Hold on, my connection isn't great. You go ahead.", + "ttft_ms": 170, + "e2e_ms": 606, + "gen_tokens": 18, + "tps": 41.2, + "prompt_tokens": 131, + "prompt_ms": 114, + "output": "Wait, my internet connection isn't good here; you guys go ahead and chat first.", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.603 + }, + { + "engine": "ollama-4b-tag", + "category": "G_zh_to_en", + "id": "G08", + "run": 0, + "input": "这个PR先不要合，Codex还有两条评论没处理", + "reference": "Don't merge this PR yet; there are still two Codex comments to address.", + "ttft_ms": 187, + "e2e_ms": 594, + "gen_tokens": 17, + "tps": 41.8, + "prompt_tokens": 136, + "prompt_ms": 116, + "output": "This PR shouldn't be merged yet; Codex still has two comments to process.", + "compliant": true, + "problems": [], + "terms_total": 3, + "terms_kept": 3, + "terms_missing": [], + "chrf": 0.592 + }, + { + "engine": "ollama-4b-tag", + "category": "G_zh_to_en", + "id": "G01", + "run": 1, + "input": "这个功能我们下周三之前能上线吗", + "reference": "Can we ship this feature before next Wednesday?", + "ttft_ms": 323, + "e2e_ms": 533, + "gen_tokens": 9, + "tps": 42.9, + "prompt_tokens": 132, + "prompt_ms": 258, + "output": "Can this feature be launched before next Wednesday?", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 1, + "terms_missing": [], + "chrf": 0.793 + }, + { + "engine": "ollama-4b-tag", + "category": "G_zh_to_en", + "id": "G02", + "run": 1, + "input": "那个就是说我们现在有两个方案一个是自己做一个是买第三方的", + "reference": "So we have two options now: build it ourselves, or buy a third-party solution.", + "ttft_ms": 110, + "e2e_ms": 806, + "gen_tokens": 28, + "tps": 40.2, + "prompt_tokens": 135, + "prompt_ms": 52, + "output": "That's to say we currently have two options: one is to build it ourselves, and the other is to buy from a third party.", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 1, + "terms_missing": [], + "chrf": 0.609 + }, + { + "engine": "ollama-4b-tag", + "category": "G_zh_to_en", + "id": "G03", + "run": 1, + "input": "上个季度我们的留存率从百分之三十八涨到了百分之四十五", + "reference": "Last quarter our retention rate went from 38% to 45%.", + "ttft_ms": 298, + "e2e_ms": 767, + "gen_tokens": 18, + "tps": 38.3, + "prompt_tokens": 137, + "prompt_ms": 196, + "output": "Last quarter, our retention rate rose from 38 percent to 45 percent.", + "compliant": true, + "problems": [], + "terms_total": 2, + "terms_kept": 2, + "terms_missing": [], + "chrf": 0.74 + }, + { + "engine": "ollama-4b-tag", + "category": "G_zh_to_en", + "id": "G04", + "run": 1, + "input": "我觉得这个风险有点大，要不我们先小范围灰度一下", + "reference": "I think the risk is a bit high. How about a small-scale gradual rollout first?", + "ttft_ms": 285, + "e2e_ms": 1004, + "gen_tokens": 19, + "tps": 26.4, + "prompt_tokens": 136, + "prompt_ms": 136, + "output": "I think this risk is quite high; let's start with a small-scale gray release first.", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.552 + }, + { + "engine": "ollama-4b-tag", + "category": "G_zh_to_en", + "id": "G05", + "run": 1, + "input": "张总说预算最多给到五十万，超了要重新审批", + "reference": "Mr. Zhang said the budget is capped at 500 thousand; anything above that needs re-approval.", + "ttft_ms": 145, + "e2e_ms": 795, + "gen_tokens": 26, + "tps": 40.0, + "prompt_tokens": 138, + "prompt_ms": 50, + "output": "Mr. Zhang said the budget cap is 500,000; any amount exceeding that requires re-approval.", + "compliant": true, + "problems": [], + "terms_total": 2, + "terms_kept": 2, + "terms_missing": [], + "chrf": 0.592 + }, + { + "engine": "ollama-4b-tag", + "category": "G_zh_to_en", + "id": "G06", + "run": 1, + "input": "识别延迟大概一点五秒，翻译再加一秒左右，用户基本感觉不到", + "reference": "Recognition latency is about 1.5 seconds, translation adds roughly another second, so users barely notice.", + "ttft_ms": 332, + "e2e_ms": 916, + "gen_tokens": 24, + "tps": 41.1, + "prompt_tokens": 139, + "prompt_ms": 253, + "output": "Recognition delay is about 1.5 seconds; translation adds another second or so. Users basically don't notice it.", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 1, + "terms_missing": [], + "chrf": 0.718 + }, + { + "engine": "ollama-4b-tag", + "category": "G_zh_to_en", + "id": "G07", + "run": 1, + "input": "等一下我这边网不太好你们先聊", + "reference": "Hold on, my connection isn't great. You go ahead.", + "ttft_ms": 134, + "e2e_ms": 564, + "gen_tokens": 18, + "tps": 41.9, + "prompt_tokens": 131, + "prompt_ms": 69, + "output": "Wait, my internet connection isn't good here; you guys go ahead and chat first.", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.603 + }, + { + "engine": "ollama-4b-tag", + "category": "G_zh_to_en", + "id": "G08", + "run": 1, + "input": "这个PR先不要合，Codex还有两条评论没处理", + "reference": "Don't merge this PR yet; there are still two Codex comments to address.", + "ttft_ms": 120, + "e2e_ms": 542, + "gen_tokens": 17, + "tps": 40.3, + "prompt_tokens": 136, + "prompt_ms": 63, + "output": "This PR should not be merged yet; Codex still has two comments to process.", + "compliant": true, + "problems": [], + "terms_total": 3, + "terms_kept": 3, + "terms_missing": [], + "chrf": 0.575 + }, + { + "engine": "ollama-4b-tag", + "category": "H_context_followup", + "id": "H01", + "run": 0, + "input": "We benchmarked the MLX build last week.", + "reference": "我们上周对MLX版本做了基准测试。", + "ttft_ms": 987, + "e2e_ms": 1246, + "gen_tokens": 11, + "tps": 42.4, + "prompt_tokens": 136, + "prompt_ms": 842, + "output": "我们上周对 MLX 版本进行了基准测试。", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 1, + "terms_missing": [], + "chrf": 0.822 + }, + { + "engine": "ollama-4b-tag", + "category": "H_context_followup", + "id": "H02", + "run": 0, + "input": "It was about twice as fast on first token.", + "reference": "它的首字速度大约快了一倍。", + "ttft_ms": 244, + "e2e_ms": 456, + "gen_tokens": 9, + "tps": 42.5, + "prompt_tokens": 173, + "prompt_ms": 182, + "output": "首词生成速度大约快了一倍。", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 1, + "terms_missing": [], + "chrf": 0.668 + }, + { + "engine": "ollama-4b-tag", + "category": "H_context_followup", + "id": "H03", + "run": 0, + "input": "But it only works on Apple Silicon.", + "reference": "但它只能在Apple Silicon上运行。", + "ttft_ms": 238, + "e2e_ms": 380, + "gen_tokens": 6, + "tps": 42.4, + "prompt_tokens": 206, + "prompt_ms": 181, + "output": "但它仅适用于苹果芯片。", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 0, + "terms_missing": [ + "Apple Silicon" + ], + "chrf": 0.053 + }, + { + "engine": "ollama-4b-tag", + "category": "H_context_followup", + "id": "H04", + "run": 0, + "input": "So for Intel Macs we'd still need the other one.", + "reference": "所以对于Intel的Mac我们仍然需要另一个。", + "ttft_ms": 353, + "e2e_ms": 639, + "gen_tokens": 12, + "tps": 42.0, + "prompt_tokens": 204, + "prompt_ms": 295, + "output": "因此，对于英特尔 Mac 我们仍然需要另一个版本。", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 0, + "terms_missing": [ + "Intel" + ], + "chrf": 0.531 + }, + { + "engine": "ollama-4b-tag", + "category": "H_context_followup", + "id": "H05", + "run": 0, + "input": "She asked whether that doubles the maintenance work.", + "reference": "她问这是否会让维护工作翻倍。", + "ttft_ms": 336, + "e2e_ms": 547, + "gen_tokens": 9, + "tps": 42.8, + "prompt_tokens": 206, + "prompt_ms": 274, + "output": "她问这是否会使维护工作量翻倍。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.651 + }, + { + "engine": "ollama-4b-tag", + "category": "H_context_followup", + "id": "H06", + "run": 0, + "input": "Honestly, it probably does.", + "reference": "说实话，可能会。", + "ttft_ms": 324, + "e2e_ms": 483, + "gen_tokens": 7, + "tps": 44.0, + "prompt_tokens": 207, + "prompt_ms": 269, + "output": "老实说，很可能确实如此。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.203 + }, + { + "engine": "ollama-4b-tag", + "category": "H_context_followup", + "id": "H01", + "run": 1, + "input": "We benchmarked the MLX build last week.", + "reference": "我们上周对MLX版本做了基准测试。", + "ttft_ms": 103, + "e2e_ms": 369, + "gen_tokens": 11, + "tps": 41.5, + "prompt_tokens": 136, + "prompt_ms": 37, + "output": "我们上周对 MLX 版本进行了基准测试。", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 1, + "terms_missing": [], + "chrf": 0.822 + }, + { + "engine": "ollama-4b-tag", + "category": "H_context_followup", + "id": "H02", + "run": 1, + "input": "It was about twice as fast on first token.", + "reference": "它的首字速度大约快了一倍。", + "ttft_ms": 103, + "e2e_ms": 318, + "gen_tokens": 9, + "tps": 41.8, + "prompt_tokens": 173, + "prompt_ms": 34, + "output": "首词生成速度大约快了两倍。", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 1, + "terms_missing": [], + "chrf": 0.464 + }, + { + "engine": "ollama-4b-tag", + "category": "H_context_followup", + "id": "H03", + "run": 1, + "input": "But it only works on Apple Silicon.", + "reference": "但它只能在Apple Silicon上运行。", + "ttft_ms": 1303, + "e2e_ms": 1470, + "gen_tokens": 7, + "tps": 41.9, + "prompt_tokens": 206, + "prompt_ms": 1190, + "output": "但它仅在苹果芯片上运行。", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 0, + "terms_missing": [ + "Apple Silicon" + ], + "chrf": 0.191 + }, + { + "engine": "ollama-4b-tag", + "category": "H_context_followup", + "id": "H04", + "run": 1, + "input": "So for Intel Macs we'd still need the other one.", + "reference": "所以对于Intel的Mac我们仍然需要另一个。", + "ttft_ms": 342, + "e2e_ms": 633, + "gen_tokens": 12, + "tps": 41.3, + "prompt_tokens": 205, + "prompt_ms": 282, + "output": "因此，对于英特尔 Mac 我们仍然需要另一个版本。", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 0, + "terms_missing": [ + "Intel" + ], + "chrf": 0.531 + }, + { + "engine": "ollama-4b-tag", + "category": "H_context_followup", + "id": "H05", + "run": 1, + "input": "She asked whether that doubles the maintenance work.", + "reference": "她问这是否会让维护工作翻倍。", + "ttft_ms": 342, + "e2e_ms": 566, + "gen_tokens": 9, + "tps": 40.3, + "prompt_tokens": 207, + "prompt_ms": 273, + "output": "她问这是否会增加一倍的工作量。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.439 + }, + { + "engine": "ollama-4b-tag", + "category": "H_context_followup", + "id": "H06", + "run": 1, + "input": "Honestly, it probably does.", + "reference": "说实话，可能会。", + "ttft_ms": 490, + "e2e_ms": 659, + "gen_tokens": 7, + "tps": 41.5, + "prompt_tokens": 207, + "prompt_ms": 277, + "output": "老实说，很可能确实如此。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.203 + }, + { + "engine": "llamacpp-metal", + "category": "A_short_turns", + "id": "A01", + "run": 0, + "input": "Let's get started.", + "reference": "我们开始吧。", + "ttft_ms": 143, + "e2e_ms": 225, + "gen_tokens": 4, + "tps": 48.7, + "prompt_tokens": 131, + "prompt_ms": 139, + "output": "让我们开始。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.567 + }, + { + "engine": "llamacpp-metal", + "category": "A_short_turns", + "id": "A02", + "run": 0, + "input": "Can everyone see my screen?", + "reference": "大家能看到我的屏幕吗？", + "ttft_ms": 139, + "e2e_ms": 302, + "gen_tokens": 7, + "tps": 43.1, + "prompt_tokens": 132, + "prompt_ms": 137, + "output": "大家都能看到我的屏幕吗？", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.839 + }, + { + "engine": "llamacpp-metal", + "category": "A_short_turns", + "id": "A03", + "run": 0, + "input": "Sorry, you cut out for a second.", + "reference": "抱歉，你刚才断了一下。", + "ttft_ms": 137, + "e2e_ms": 326, + "gen_tokens": 8, + "tps": 42.2, + "prompt_tokens": 135, + "prompt_ms": 134, + "output": "抱歉，您刚才中断了。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.316 + }, + { + "engine": "llamacpp-metal", + "category": "A_short_turns", + "id": "A04", + "run": 0, + "input": "I'll take that offline.", + "reference": "这个我们线下再聊。", + "ttft_ms": 141, + "e2e_ms": 280, + "gen_tokens": 6, + "tps": 43.1, + "prompt_tokens": 132, + "prompt_ms": 136, + "output": "我稍后处理。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.06 + }, + { + "engine": "llamacpp-metal", + "category": "A_short_turns", + "id": "A05", + "run": 0, + "input": "Any questions before we move on?", + "reference": "在继续之前有什么问题吗？", + "ttft_ms": 136, + "e2e_ms": 350, + "gen_tokens": 9, + "tps": 42.0, + "prompt_tokens": 133, + "prompt_ms": 133, + "output": "在我们继续之前，有任何问题吗？", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.474 + }, + { + "engine": "llamacpp-metal", + "category": "A_short_turns", + "id": "A06", + "run": 0, + "input": "That makes sense, thanks for clarifying.", + "reference": "明白了，谢谢你的说明。", + "ttft_ms": 138, + "e2e_ms": 347, + "gen_tokens": 9, + "tps": 43.2, + "prompt_tokens": 135, + "prompt_ms": 136, + "output": "说得有道理，谢谢您的澄清。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.206 + }, + { + "engine": "llamacpp-metal", + "category": "A_short_turns", + "id": "A07", + "run": 0, + "input": "We're running about five minutes behind.", + "reference": "我们大概晚了五分钟。", + "ttft_ms": 138, + "e2e_ms": 274, + "gen_tokens": 6, + "tps": 44.0, + "prompt_tokens": 134, + "prompt_ms": 135, + "output": "我们落后大约五分钟。", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 1, + "terms_missing": [], + "chrf": 0.384 + }, + { + "engine": "llamacpp-metal", + "category": "A_short_turns", + "id": "A08", + "run": 0, + "input": "Let me share the link in the chat.", + "reference": "我把链接发到聊天里。", + "ttft_ms": 141, + "e2e_ms": 354, + "gen_tokens": 9, + "tps": 42.4, + "prompt_tokens": 135, + "prompt_ms": 137, + "output": "让我在聊天框里分享链接。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.221 + }, + { + "engine": "llamacpp-metal", + "category": "A_short_turns", + "id": "A01", + "run": 1, + "input": "Let's get started.", + "reference": "我们开始吧。", + "ttft_ms": 137, + "e2e_ms": 242, + "gen_tokens": 5, + "tps": 47.6, + "prompt_tokens": 131, + "prompt_ms": 135, + "output": "让我们开始吧。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.955 + }, + { + "engine": "llamacpp-metal", + "category": "A_short_turns", + "id": "A02", + "run": 1, + "input": "Can everyone see my screen?", + "reference": "大家能看到我的屏幕吗？", + "ttft_ms": 144, + "e2e_ms": 304, + "gen_tokens": 7, + "tps": 43.9, + "prompt_tokens": 132, + "prompt_ms": 142, + "output": "大家都能看到我的屏幕吗？", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.839 + }, + { + "engine": "llamacpp-metal", + "category": "A_short_turns", + "id": "A03", + "run": 1, + "input": "Sorry, you cut out for a second.", + "reference": "抱歉，你刚才断了一下。", + "ttft_ms": 138, + "e2e_ms": 328, + "gen_tokens": 8, + "tps": 42.0, + "prompt_tokens": 135, + "prompt_ms": 135, + "output": "抱歉，您刚才中断了。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.316 + }, + { + "engine": "llamacpp-metal", + "category": "A_short_turns", + "id": "A04", + "run": 1, + "input": "I'll take that offline.", + "reference": "这个我们线下再聊。", + "ttft_ms": 139, + "e2e_ms": 307, + "gen_tokens": 7, + "tps": 41.8, + "prompt_tokens": 132, + "prompt_ms": 137, + "output": "我把它放到后台处理。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.054 + }, + { + "engine": "llamacpp-metal", + "category": "A_short_turns", + "id": "A05", + "run": 1, + "input": "Any questions before we move on?", + "reference": "在继续之前有什么问题吗？", + "ttft_ms": 149, + "e2e_ms": 340, + "gen_tokens": 8, + "tps": 41.8, + "prompt_tokens": 133, + "prompt_ms": 145, + "output": "在我们继续之前，有什么问题吗？", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.727 + }, + { + "engine": "llamacpp-metal", + "category": "A_short_turns", + "id": "A06", + "run": 1, + "input": "That makes sense, thanks for clarifying.", + "reference": "明白了，谢谢你的说明。", + "ttft_ms": 143, + "e2e_ms": 420, + "gen_tokens": 11, + "tps": 39.8, + "prompt_tokens": 135, + "prompt_ms": 140, + "output": "这样讲就合理了，谢谢您的澄清。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.276 + }, + { + "engine": "llamacpp-metal", + "category": "A_short_turns", + "id": "A07", + "run": 1, + "input": "We're running about five minutes behind.", + "reference": "我们大概晚了五分钟。", + "ttft_ms": 145, + "e2e_ms": 278, + "gen_tokens": 6, + "tps": 45.1, + "prompt_tokens": 134, + "prompt_ms": 141, + "output": "我们落后大约五分钟。", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 1, + "terms_missing": [], + "chrf": 0.384 + }, + { + "engine": "llamacpp-metal", + "category": "A_short_turns", + "id": "A08", + "run": 1, + "input": "Let me share the link in the chat.", + "reference": "我把链接发到聊天里。", + "ttft_ms": 137, + "e2e_ms": 360, + "gen_tokens": 9, + "tps": 40.3, + "prompt_tokens": 135, + "prompt_ms": 134, + "output": "让我把链接发到聊天框里。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.801 + }, + { + "engine": "llamacpp-metal", + "category": "B_asr_drafts", + "id": "B01", + "run": 0, + "input": "so what we found was that the", + "reference": "所以我们发现的是", + "ttft_ms": 144, + "e2e_ms": 336, + "gen_tokens": 8, + "tps": 41.6, + "prompt_tokens": 133, + "prompt_ms": 140, + "output": "所以，我们发现的是", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.758 + }, + { + "engine": "llamacpp-metal", + "category": "B_asr_drafts", + "id": "B02", + "run": 0, + "input": "if you look at the second chart you can see", + "reference": "如果你看第二张图，你可以看到", + "ttft_ms": 344, + "e2e_ms": 652, + "gen_tokens": 12, + "tps": 39.0, + "prompt_tokens": 136, + "prompt_ms": 339, + "output": "如果您看第二张图表，您可以看到", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.532 + }, + { + "engine": "llamacpp-metal", + "category": "B_asr_drafts", + "id": "B03", + "run": 0, + "input": "the main reason we chose this approach is", + "reference": "我们选择这个方案的主要原因是", + "ttft_ms": 164, + "e2e_ms": 328, + "gen_tokens": 7, + "tps": 42.8, + "prompt_tokens": 134, + "prompt_ms": 157, + "output": "我们选择这一方法的主要原因是", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.647 + }, + { + "engine": "llamacpp-metal", + "category": "B_asr_drafts", + "id": "B04", + "run": 0, + "input": "and then the customer said that they", + "reference": "然后客户说他们", + "ttft_ms": 144, + "e2e_ms": 312, + "gen_tokens": 7, + "tps": 41.7, + "prompt_tokens": 133, + "prompt_ms": 140, + "output": "然后，客户说他们……", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.663 + }, + { + "engine": "llamacpp-metal", + "category": "B_asr_drafts", + "id": "B05", + "run": 0, + "input": "I think the biggest risk here", + "reference": "我觉得这里最大的风险", + "ttft_ms": 140, + "e2e_ms": 304, + "gen_tokens": 7, + "tps": 42.6, + "prompt_tokens": 132, + "prompt_ms": 137, + "output": "我认为这里最大的风险是。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.636 + }, + { + "engine": "llamacpp-metal", + "category": "B_asr_drafts", + "id": "B06", + "run": 0, + "input": "we need to decide by", + "reference": "我们需要在……之前决定", + "ttft_ms": 145, + "e2e_ms": 307, + "gen_tokens": 7, + "tps": 43.3, + "prompt_tokens": 131, + "prompt_ms": 142, + "output": "我们需要在...做出决定。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.413 + }, + { + "engine": "llamacpp-metal", + "category": "B_asr_drafts", + "id": "B07", + "run": 0, + "input": "which brings me to", + "reference": "这就引出了", + "ttft_ms": 143, + "e2e_ms": 300, + "gen_tokens": 7, + "tps": 44.6, + "prompt_tokens": 130, + "prompt_ms": 139, + "output": "这把我带到了", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.096 + }, + { + "engine": "llamacpp-metal", + "category": "B_asr_drafts", + "id": "B08", + "run": 0, + "input": "the data shows a clear", + "reference": "数据显示出明显的", + "ttft_ms": 146, + "e2e_ms": 314, + "gen_tokens": 7, + "tps": 41.6, + "prompt_tokens": 131, + "prompt_ms": 143, + "output": "数据显示，情况十分明显。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.416 + }, + { + "engine": "llamacpp-metal", + "category": "B_asr_drafts", + "id": "B01", + "run": 1, + "input": "so what we found was that the", + "reference": "所以我们发现的是", + "ttft_ms": 141, + "e2e_ms": 280, + "gen_tokens": 6, + "tps": 43.2, + "prompt_tokens": 133, + "prompt_ms": 136, + "output": "所以，我们发现的是……", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.716 + }, + { + "engine": "llamacpp-metal", + "category": "B_asr_drafts", + "id": "B02", + "run": 1, + "input": "if you look at the second chart you can see", + "reference": "如果你看第二张图，你可以看到", + "ttft_ms": 146, + "e2e_ms": 398, + "gen_tokens": 10, + "tps": 39.7, + "prompt_tokens": 136, + "prompt_ms": 143, + "output": "如果您看第二张图表，您可以看到。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.524 + }, + { + "engine": "llamacpp-metal", + "category": "B_asr_drafts", + "id": "B03", + "run": 1, + "input": "the main reason we chose this approach is", + "reference": "我们选择这个方案的主要原因是", + "ttft_ms": 147, + "e2e_ms": 303, + "gen_tokens": 6, + "tps": 38.4, + "prompt_tokens": 134, + "prompt_ms": 140, + "output": "我们选择这一方法的主要原因", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.575 + }, + { + "engine": "llamacpp-metal", + "category": "B_asr_drafts", + "id": "B04", + "run": 1, + "input": "and then the customer said that they", + "reference": "然后客户说他们", + "ttft_ms": 145, + "e2e_ms": 278, + "gen_tokens": 6, + "tps": 45.2, + "prompt_tokens": 133, + "prompt_ms": 141, + "output": "然后客户说他们……", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.93 + }, + { + "engine": "llamacpp-metal", + "category": "B_asr_drafts", + "id": "B05", + "run": 1, + "input": "I think the biggest risk here", + "reference": "我觉得这里最大的风险", + "ttft_ms": 146, + "e2e_ms": 308, + "gen_tokens": 7, + "tps": 43.4, + "prompt_tokens": 132, + "prompt_ms": 143, + "output": "我认为这里最大的风险是……", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.622 + }, + { + "engine": "llamacpp-metal", + "category": "B_asr_drafts", + "id": "B06", + "run": 1, + "input": "we need to decide by", + "reference": "我们需要在……之前决定", + "ttft_ms": 145, + "e2e_ms": 258, + "gen_tokens": 5, + "tps": 44.4, + "prompt_tokens": 131, + "prompt_ms": 143, + "output": "我们需要做出决定。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.337 + }, + { + "engine": "llamacpp-metal", + "category": "B_asr_drafts", + "id": "B07", + "run": 1, + "input": "which brings me to", + "reference": "这就引出了", + "ttft_ms": 144, + "e2e_ms": 309, + "gen_tokens": 7, + "tps": 42.5, + "prompt_tokens": 130, + "prompt_ms": 141, + "output": "这把我带到了", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.096 + }, + { + "engine": "llamacpp-metal", + "category": "B_asr_drafts", + "id": "B08", + "run": 1, + "input": "the data shows a clear", + "reference": "数据显示出明显的", + "ttft_ms": 146, + "e2e_ms": 293, + "gen_tokens": 6, + "tps": 40.7, + "prompt_tokens": 131, + "prompt_ms": 142, + "output": "数据显示，情况明显。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.438 + }, + { + "engine": "llamacpp-metal", + "category": "C_asr_noisy", + "id": "C01", + "run": 0, + "input": "um so yeah basically what we're gonna do is we're gonna ship it next week", + "reference": "所以基本上我们要做的就是下周发布。", + "ttft_ms": 182, + "e2e_ms": 392, + "gen_tokens": 9, + "tps": 42.7, + "prompt_tokens": 143, + "prompt_ms": 173, + "output": "所以基本上，我们下周就发货。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.37 + }, + { + "engine": "llamacpp-metal", + "category": "C_asr_noisy", + "id": "C02", + "run": 0, + "input": "i mean it's it's not perfect but it works you know", + "reference": "我是说它不完美，但能用，你懂的。", + "ttft_ms": 143, + "e2e_ms": 540, + "gen_tokens": 16, + "tps": 40.3, + "prompt_tokens": 139, + "prompt_ms": 140, + "output": "我的意思是，它虽然不完美，但能工作，你知道的。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.411 + }, + { + "engine": "llamacpp-metal", + "category": "C_asr_noisy", + "id": "C03", + "run": 0, + "input": "okay okay let me let me just pull that up real quick", + "reference": "好好，让我快速调出来看一下。", + "ttft_ms": 146, + "e2e_ms": 412, + "gen_tokens": 11, + "tps": 41.4, + "prompt_tokens": 138, + "prompt_ms": 143, + "output": "好的，好的，让我快速把它调出来。", + "compliant": false, + "problems": [ + "preamble:好的" + ], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.427 + }, + { + "engine": "llamacpp-metal", + "category": "C_asr_noisy", + "id": "C04", + "run": 0, + "input": "right so the the thing is we don't actually have the budget for that", + "reference": "对，问题是我们其实没有这方面的预算。", + "ttft_ms": 146, + "e2e_ms": 334, + "gen_tokens": 8, + "tps": 42.4, + "prompt_tokens": 141, + "prompt_ms": 142, + "output": "所以，实际上我们没有这笔预算。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.252 + }, + { + "engine": "llamacpp-metal", + "category": "C_asr_noisy", + "id": "C05", + "run": 0, + "input": "like honestly I don't think anyone has looked at this since like march", + "reference": "说实话我觉得从三月以来就没人看过这个了。", + "ttft_ms": 152, + "e2e_ms": 470, + "gen_tokens": 13, + "tps": 40.9, + "prompt_tokens": 140, + "prompt_ms": 148, + "output": "说实话，我觉得自三月以来，没人关注过这个。", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 1, + "terms_missing": [], + "chrf": 0.411 + }, + { + "engine": "llamacpp-metal", + "category": "C_asr_noisy", + "id": "C06", + "run": 0, + "input": "and uh we also need to loop in the legal team before we sign anything", + "reference": "另外在签任何东西之前我们还需要让法务团队参与进来。", + "ttft_ms": 147, + "e2e_ms": 529, + "gen_tokens": 15, + "tps": 39.3, + "prompt_tokens": 141, + "prompt_ms": 143, + "output": "而且，在我们签署任何文件之前，还需要让法律团队介入。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.296 + }, + { + "engine": "llamacpp-metal", + "category": "C_asr_noisy", + "id": "C07", + "run": 0, + "input": "yeah no that's fair that's totally fair", + "reference": "对，没错，这很合理。", + "ttft_ms": 146, + "e2e_ms": 385, + "gen_tokens": 10, + "tps": 41.8, + "prompt_tokens": 135, + "prompt_ms": 141, + "output": "是啊，那很公平，完全公平。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.094 + }, + { + "engine": "llamacpp-metal", + "category": "C_asr_noisy", + "id": "C08", + "run": 0, + "input": "so hold on are we saying the deadline moved or not", + "reference": "等等，我们是说截止日期改了还是没改？", + "ttft_ms": 142, + "e2e_ms": 475, + "gen_tokens": 13, + "tps": 39.0, + "prompt_tokens": 137, + "prompt_ms": 138, + "output": "所以，我们要说的是截止日期变动了，还是没变？", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.36 + }, + { + "engine": "llamacpp-metal", + "category": "C_asr_noisy", + "id": "C01", + "run": 1, + "input": "um so yeah basically what we're gonna do is we're gonna ship it next week", + "reference": "所以基本上我们要做的就是下周发布。", + "ttft_ms": 160, + "e2e_ms": 427, + "gen_tokens": 10, + "tps": 37.4, + "prompt_tokens": 143, + "prompt_ms": 153, + "output": "所以，基本上，我们下周就会发货。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.275 + }, + { + "engine": "llamacpp-metal", + "category": "C_asr_noisy", + "id": "C02", + "run": 1, + "input": "i mean it's it's not perfect but it works you know", + "reference": "我是说它不完美，但能用，你懂的。", + "ttft_ms": 153, + "e2e_ms": 584, + "gen_tokens": 16, + "tps": 37.2, + "prompt_tokens": 139, + "prompt_ms": 148, + "output": "我的意思是，虽然它不完美，但它是能用的，你知道。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.42 + }, + { + "engine": "llamacpp-metal", + "category": "C_asr_noisy", + "id": "C03", + "run": 1, + "input": "okay okay let me let me just pull that up real quick", + "reference": "好好，让我快速调出来看一下。", + "ttft_ms": 151, + "e2e_ms": 425, + "gen_tokens": 11, + "tps": 40.1, + "prompt_tokens": 138, + "prompt_ms": 146, + "output": "好的，好的，我马上把它调出来。", + "compliant": false, + "problems": [ + "preamble:好的" + ], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.199 + }, + { + "engine": "llamacpp-metal", + "category": "C_asr_noisy", + "id": "C04", + "run": 1, + "input": "right so the the thing is we don't actually have the budget for that", + "reference": "对，问题是我们其实没有这方面的预算。", + "ttft_ms": 145, + "e2e_ms": 328, + "gen_tokens": 8, + "tps": 43.7, + "prompt_tokens": 141, + "prompt_ms": 141, + "output": "所以，实际上我们没有那个预算。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.207 + }, + { + "engine": "llamacpp-metal", + "category": "C_asr_noisy", + "id": "C05", + "run": 1, + "input": "like honestly I don't think anyone has looked at this since like march", + "reference": "说实话我觉得从三月以来就没人看过这个了。", + "ttft_ms": 145, + "e2e_ms": 517, + "gen_tokens": 15, + "tps": 40.4, + "prompt_tokens": 140, + "prompt_ms": 142, + "output": "说实话，我觉得自三月以来，没人再关注过这个了。", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 1, + "terms_missing": [], + "chrf": 0.496 + }, + { + "engine": "llamacpp-metal", + "category": "C_asr_noisy", + "id": "C06", + "run": 1, + "input": "and uh we also need to loop in the legal team before we sign anything", + "reference": "另外在签任何东西之前我们还需要让法务团队参与进来。", + "ttft_ms": 144, + "e2e_ms": 537, + "gen_tokens": 16, + "tps": 40.7, + "prompt_tokens": 141, + "prompt_ms": 140, + "output": "而且，在签署任何文件之前，我们还需要让法务团队介入。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.458 + }, + { + "engine": "llamacpp-metal", + "category": "C_asr_noisy", + "id": "C07", + "run": 1, + "input": "yeah no that's fair that's totally fair", + "reference": "对，没错，这很合理。", + "ttft_ms": 144, + "e2e_ms": 385, + "gen_tokens": 10, + "tps": 41.5, + "prompt_tokens": 135, + "prompt_ms": 141, + "output": "是啊，这很公平，完全公平。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.199 + }, + { + "engine": "llamacpp-metal", + "category": "C_asr_noisy", + "id": "C08", + "run": 1, + "input": "so hold on are we saying the deadline moved or not", + "reference": "等等，我们是说截止日期改了还是没改？", + "ttft_ms": 148, + "e2e_ms": 476, + "gen_tokens": 13, + "tps": 39.7, + "prompt_tokens": 137, + "prompt_ms": 144, + "output": "所以，我们要说的是截止日期变动了，还是没变动？", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.356 + }, + { + "engine": "llamacpp-metal", + "category": "D_numbers_names", + "id": "D01", + "run": 0, + "input": "Revenue grew 12% year over year to $4.3 billion.", + "reference": "收入同比增长12%，达到43亿美元。", + "ttft_ms": 146, + "e2e_ms": 492, + "gen_tokens": 14, + "tps": 40.5, + "prompt_tokens": 142, + "prompt_ms": 142, + "output": "收入同比增长 12%，达到 43 亿美元。", + "compliant": true, + "problems": [], + "terms_total": 2, + "terms_kept": 2, + "terms_missing": [], + "chrf": 1.0 + }, + { + "engine": "llamacpp-metal", + "category": "D_numbers_names", + "id": "D02", + "run": 0, + "input": "Sarah from Stripe will join us on March 15th at 3 p.m.", + "reference": "Stripe的Sarah将在3月15日下午3点加入我们。", + "ttft_ms": 475, + "e2e_ms": 1062, + "gen_tokens": 23, + "tps": 39.1, + "prompt_tokens": 144, + "prompt_ms": 470, + "output": "来自 Stripe 的 Sarah 将于 3 月 15 日下午 3 点加入我们。", + "compliant": true, + "problems": [], + "terms_total": 3, + "terms_kept": 3, + "terms_missing": [], + "chrf": 0.89 + }, + { + "engine": "llamacpp-metal", + "category": "D_numbers_names", + "id": "D03", + "run": 0, + "input": "We processed 2.7 million requests with a p99 latency of 180 milliseconds.", + "reference": "我们处理了270万个请求，p99延迟为180毫秒。", + "ttft_ms": 220, + "e2e_ms": 876, + "gen_tokens": 25, + "tps": 38.1, + "prompt_tokens": 147, + "prompt_ms": 217, + "output": "我们处理了 270 万次请求，p99 延迟为 180 毫秒。", + "compliant": true, + "problems": [], + "terms_total": 3, + "terms_kept": 3, + "terms_missing": [], + "chrf": 0.891 + }, + { + "engine": "llamacpp-metal", + "category": "D_numbers_names", + "id": "D04", + "run": 0, + "input": "Microsoft and OpenAI announced the partnership in Q2 2024.", + "reference": "Microsoft和OpenAI在2024年第二季度宣布了合作。", + "ttft_ms": 144, + "e2e_ms": 624, + "gen_tokens": 19, + "tps": 39.6, + "prompt_tokens": 142, + "prompt_ms": 141, + "output": "微软和 OpenAI 于 2024 年第二季度宣布建立合作关系。", + "compliant": true, + "problems": [], + "terms_total": 3, + "terms_kept": 3, + "terms_missing": [], + "chrf": 0.535 + }, + { + "engine": "llamacpp-metal", + "category": "D_numbers_names", + "id": "D05", + "run": 0, + "input": "The budget is 850 thousand euros, split 60-40 between the two teams.", + "reference": "预算是85万欧元，两个团队按60比40分配。", + "ttft_ms": 204, + "e2e_ms": 873, + "gen_tokens": 25, + "tps": 37.4, + "prompt_tokens": 148, + "prompt_ms": 201, + "output": "预算为 85 万欧元，在两个团队之间按 60 比 40 分配。", + "compliant": true, + "problems": [], + "terms_total": 3, + "terms_kept": 3, + "terms_missing": [], + "chrf": 0.714 + }, + { + "engine": "llamacpp-metal", + "category": "D_numbers_names", + "id": "D06", + "run": 0, + "input": "Dr. Nakamura's team at Stanford published the results on arXiv last Tuesday.", + "reference": "Nakamura博士在斯坦福的团队上周二在arXiv上发表了结果。", + "ttft_ms": 173, + "e2e_ms": 715, + "gen_tokens": 21, + "tps": 38.8, + "prompt_tokens": 144, + "prompt_ms": 161, + "output": "中村博士在斯坦福大学的研究团队于上周二在 arXiv 上发布了研究成果。", + "compliant": true, + "problems": [], + "terms_total": 2, + "terms_kept": 2, + "terms_missing": [], + "chrf": 0.5 + }, + { + "engine": "llamacpp-metal", + "category": "D_numbers_names", + "id": "D07", + "run": 0, + "input": "Version 3.2.1 fixes the bug reported in ticket LCT-418.", + "reference": "3.2.1版本修复了工单LCT-418中报告的问题。", + "ttft_ms": 212, + "e2e_ms": 814, + "gen_tokens": 24, + "tps": 39.9, + "prompt_tokens": 146, + "prompt_ms": 206, + "output": "版本 3.2.1 修复了 LCT-418 工单中报告的错误。", + "compliant": true, + "problems": [], + "terms_total": 2, + "terms_kept": 2, + "terms_missing": [], + "chrf": 0.591 + }, + { + "engine": "llamacpp-metal", + "category": "D_numbers_names", + "id": "D08", + "run": 0, + "input": "Our NPS went from 31 to 47 after the redesign.", + "reference": "改版后我们的NPS从31升到了47。", + "ttft_ms": 143, + "e2e_ms": 651, + "gen_tokens": 21, + "tps": 41.3, + "prompt_tokens": 142, + "prompt_ms": 139, + "output": "我们的净推荐值从 31 提升到 47，这是经过重新设计之后。", + "compliant": true, + "problems": [], + "terms_total": 3, + "terms_kept": 2, + "terms_missing": [ + "NPS" + ], + "chrf": 0.259 + }, + { + "engine": "llamacpp-metal", + "category": "D_numbers_names", + "id": "D01", + "run": 1, + "input": "Revenue grew 12% year over year to $4.3 billion.", + "reference": "收入同比增长12%，达到43亿美元。", + "ttft_ms": 145, + "e2e_ms": 493, + "gen_tokens": 14, + "tps": 40.2, + "prompt_tokens": 142, + "prompt_ms": 142, + "output": "收入同比增长 12%，达到 43 亿美元。", + "compliant": true, + "problems": [], + "terms_total": 2, + "terms_kept": 2, + "terms_missing": [], + "chrf": 1.0 + }, + { + "engine": "llamacpp-metal", + "category": "D_numbers_names", + "id": "D02", + "run": 1, + "input": "Sarah from Stripe will join us on March 15th at 3 p.m.", + "reference": "Stripe的Sarah将在3月15日下午3点加入我们。", + "ttft_ms": 152, + "e2e_ms": 704, + "gen_tokens": 23, + "tps": 41.7, + "prompt_tokens": 144, + "prompt_ms": 149, + "output": "来自 Stripe 的 Sarah 将于 3 月 15 日下午 3 点加入我们。", + "compliant": true, + "problems": [], + "terms_total": 3, + "terms_kept": 3, + "terms_missing": [], + "chrf": 0.89 + }, + { + "engine": "llamacpp-metal", + "category": "D_numbers_names", + "id": "D03", + "run": 1, + "input": "We processed 2.7 million requests with a p99 latency of 180 milliseconds.", + "reference": "我们处理了270万个请求，p99延迟为180毫秒。", + "ttft_ms": 200, + "e2e_ms": 789, + "gen_tokens": 25, + "tps": 42.5, + "prompt_tokens": 147, + "prompt_ms": 197, + "output": "我们处理了 270 万次请求，p99 延迟为 180 毫秒。", + "compliant": true, + "problems": [], + "terms_total": 3, + "terms_kept": 3, + "terms_missing": [], + "chrf": 0.891 + }, + { + "engine": "llamacpp-metal", + "category": "D_numbers_names", + "id": "D04", + "run": 1, + "input": "Microsoft and OpenAI announced the partnership in Q2 2024.", + "reference": "Microsoft和OpenAI在2024年第二季度宣布了合作。", + "ttft_ms": 141, + "e2e_ms": 599, + "gen_tokens": 19, + "tps": 41.5, + "prompt_tokens": 142, + "prompt_ms": 139, + "output": "微软和 OpenAI 于 2024 年第二季度宣布建立合作关系。", + "compliant": true, + "problems": [], + "terms_total": 3, + "terms_kept": 3, + "terms_missing": [], + "chrf": 0.535 + }, + { + "engine": "llamacpp-metal", + "category": "D_numbers_names", + "id": "D05", + "run": 1, + "input": "The budget is 850 thousand euros, split 60-40 between the two teams.", + "reference": "预算是85万欧元，两个团队按60比40分配。", + "ttft_ms": 195, + "e2e_ms": 801, + "gen_tokens": 25, + "tps": 41.2, + "prompt_tokens": 148, + "prompt_ms": 193, + "output": "预算为 85 万欧元，按 60 比 40 的比例分配给两个团队。", + "compliant": true, + "problems": [], + "terms_total": 3, + "terms_kept": 3, + "terms_missing": [], + "chrf": 0.611 + }, + { + "engine": "llamacpp-metal", + "category": "D_numbers_names", + "id": "D06", + "run": 1, + "input": "Dr. Nakamura's team at Stanford published the results on arXiv last Tuesday.", + "reference": "Nakamura博士在斯坦福的团队上周二在arXiv上发表了结果。", + "ttft_ms": 144, + "e2e_ms": 643, + "gen_tokens": 21, + "tps": 42.1, + "prompt_tokens": 144, + "prompt_ms": 140, + "output": "仲村博士在斯坦福大学的研究团队于上周二在 arXiv 上发布了研究成果。", + "compliant": true, + "problems": [], + "terms_total": 2, + "terms_kept": 1, + "terms_missing": [ + "Nakamura/中村" + ], + "chrf": 0.5 + }, + { + "engine": "llamacpp-metal", + "category": "D_numbers_names", + "id": "D07", + "run": 1, + "input": "Version 3.2.1 fixes the bug reported in ticket LCT-418.", + "reference": "3.2.1版本修复了工单LCT-418中报告的问题。", + "ttft_ms": 212, + "e2e_ms": 788, + "gen_tokens": 24, + "tps": 41.7, + "prompt_tokens": 146, + "prompt_ms": 207, + "output": "版本 3.2.1 修复了 LCT-418 工单中报告的缺陷。", + "compliant": true, + "problems": [], + "terms_total": 2, + "terms_kept": 2, + "terms_missing": [], + "chrf": 0.591 + }, + { + "engine": "llamacpp-metal", + "category": "D_numbers_names", + "id": "D08", + "run": 1, + "input": "Our NPS went from 31 to 47 after the redesign.", + "reference": "改版后我们的NPS从31升到了47。", + "ttft_ms": 138, + "e2e_ms": 583, + "gen_tokens": 19, + "tps": 42.7, + "prompt_tokens": 142, + "prompt_ms": 134, + "output": "我们的 NPS 在重新设计后从 31 提升到 47。", + "compliant": true, + "problems": [], + "terms_total": 3, + "terms_kept": 3, + "terms_missing": [], + "chrf": 0.482 + }, + { + "engine": "llamacpp-metal", + "category": "E_tech_terms", + "id": "E01", + "run": 0, + "input": "We migrated the service from Kubernetes to a single Rust binary on bare metal.", + "reference": "我们把这个服务从Kubernetes迁移到了裸金属上的单个Rust二进制。", + "ttft_ms": 136, + "e2e_ms": 529, + "gen_tokens": 17, + "tps": 43.3, + "prompt_tokens": 141, + "prompt_ms": 134, + "output": "我们将服务从 Kubernetes 迁移到了裸金属上的单个 Rust 二进制文件。", + "compliant": true, + "problems": [], + "terms_total": 2, + "terms_kept": 2, + "terms_missing": [], + "chrf": 0.858 + }, + { + "engine": "llamacpp-metal", + "category": "E_tech_terms", + "id": "E02", + "run": 0, + "input": "The model runs fully on-device with Metal acceleration and a 4-bit quantized Qwen checkpoint.", + "reference": "模型完全在设备端运行，使用Metal加速和4位量化的Qwen检查点。", + "ttft_ms": 189, + "e2e_ms": 755, + "gen_tokens": 24, + "tps": 42.4, + "prompt_tokens": 146, + "prompt_ms": 187, + "output": "该模型完全基于设备运行，采用 Metal 加速和 4 位量化 Qwen 检查点。", + "compliant": true, + "problems": [], + "terms_total": 3, + "terms_kept": 3, + "terms_missing": [], + "chrf": 0.695 + }, + { + "engine": "llamacpp-metal", + "category": "E_tech_terms", + "id": "E03", + "run": 0, + "input": "Enable WASAPI loopback so we capture system audio without a virtual device.", + "reference": "启用WASAPI环回，这样不需要虚拟设备也能采集系统音频。", + "ttft_ms": 136, + "e2e_ms": 561, + "gen_tokens": 18, + "tps": 42.4, + "prompt_tokens": 141, + "prompt_ms": 134, + "output": "启用 WASAPI 回环，以便在不使用虚拟设备的情况下捕获系统音频。", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 1, + "terms_missing": [], + "chrf": 0.475 + }, + { + "engine": "llamacpp-metal", + "category": "E_tech_terms", + "id": "E04", + "run": 0, + "input": "The API returns a 429 when you exceed the rate limit, so add exponential backoff.", + "reference": "超过速率限制时API会返回429，所以要加指数退避。", + "ttft_ms": 192, + "e2e_ms": 744, + "gen_tokens": 23, + "tps": 41.6, + "prompt_tokens": 147, + "prompt_ms": 190, + "output": "当超过速率限制时，API 会返回 429，因此请添加指数退避。", + "compliant": true, + "problems": [], + "terms_total": 2, + "terms_kept": 2, + "terms_missing": [], + "chrf": 0.733 + }, + { + "engine": "llamacpp-metal", + "category": "E_tech_terms", + "id": "E05", + "run": 0, + "input": "We're seeing GPU memory pressure because num_ctx defaults to 32K.", + "reference": "我们遇到了GPU内存压力，因为num_ctx默认是32K。", + "ttft_ms": 136, + "e2e_ms": 557, + "gen_tokens": 18, + "tps": 42.8, + "prompt_tokens": 142, + "prompt_ms": 134, + "output": "我们看到 GPU 内存压力，因为 num_ctx 默认值为 32K。", + "compliant": true, + "problems": [], + "terms_total": 3, + "terms_kept": 3, + "terms_missing": [], + "chrf": 0.76 + }, + { + "engine": "llamacpp-metal", + "category": "E_tech_terms", + "id": "E06", + "run": 0, + "input": "The PR adds a TranslationEngine protocol with an Ollama adapter and an embedded llama.cpp runtime.", + "reference": "这个PR增加了TranslationEngine协议，带有一个Ollama适配器和一个内置的llama.cpp运行时。", + "ttft_ms": 189, + "e2e_ms": 761, + "gen_tokens": 24, + "tps": 42.0, + "prompt_tokens": 146, + "prompt_ms": 187, + "output": "该 PR 添加了一个 TranslationEngine 协议，包含 Ollama 适配器及嵌入式 llama.cpp 运行时。", + "compliant": true, + "problems": [], + "terms_total": 4, + "terms_kept": 4, + "terms_missing": [], + "chrf": 0.697 + }, + { + "engine": "llamacpp-metal", + "category": "E_tech_terms", + "id": "E07", + "run": 0, + "input": "Notarization requires a Developer ID certificate and the hardened runtime.", + "reference": "公证需要Developer ID证书和强化运行时。", + "ttft_ms": 140, + "e2e_ms": 391, + "gen_tokens": 11, + "tps": 43.8, + "prompt_tokens": 139, + "prompt_ms": 138, + "output": "公证需要开发者 ID 证书和加固运行时。", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 0, + "terms_missing": [ + "Developer ID" + ], + "chrf": 0.392 + }, + { + "engine": "llamacpp-metal", + "category": "E_tech_terms", + "id": "E08", + "run": 0, + "input": "Our CI runs on windows-latest, which no longer ships NSIS.", + "reference": "我们的CI跑在windows-latest上，它不再自带NSIS。", + "ttft_ms": 138, + "e2e_ms": 616, + "gen_tokens": 19, + "tps": 39.7, + "prompt_tokens": 141, + "prompt_ms": 136, + "output": "我们的 CI 运行在 windows-latest 上，该镜像不再包含 NSIS。", + "compliant": true, + "problems": [], + "terms_total": 3, + "terms_kept": 3, + "terms_missing": [], + "chrf": 0.721 + }, + { + "engine": "llamacpp-metal", + "category": "E_tech_terms", + "id": "E01", + "run": 1, + "input": "We migrated the service from Kubernetes to a single Rust binary on bare metal.", + "reference": "我们把这个服务从Kubernetes迁移到了裸金属上的单个Rust二进制。", + "ttft_ms": 164, + "e2e_ms": 564, + "gen_tokens": 17, + "tps": 42.5, + "prompt_tokens": 141, + "prompt_ms": 158, + "output": "我们将服务从 Kubernetes 迁移到了裸金属上的单一 Rust 二进制文件。", + "compliant": true, + "problems": [], + "terms_total": 2, + "terms_kept": 2, + "terms_missing": [], + "chrf": 0.787 + }, + { + "engine": "llamacpp-metal", + "category": "E_tech_terms", + "id": "E02", + "run": 1, + "input": "The model runs fully on-device with Metal acceleration and a 4-bit quantized Qwen checkpoint.", + "reference": "模型完全在设备端运行，使用Metal加速和4位量化的Qwen检查点。", + "ttft_ms": 200, + "e2e_ms": 797, + "gen_tokens": 25, + "tps": 41.8, + "prompt_tokens": 146, + "prompt_ms": 196, + "output": "该模型完全基于设备运行，利用 Metal 加速，并使用 4 位量化 Qwen 检查点。", + "compliant": true, + "problems": [], + "terms_total": 3, + "terms_kept": 3, + "terms_missing": [], + "chrf": 0.621 + }, + { + "engine": "llamacpp-metal", + "category": "E_tech_terms", + "id": "E03", + "run": 1, + "input": "Enable WASAPI loopback so we capture system audio without a virtual device.", + "reference": "启用WASAPI环回，这样不需要虚拟设备也能采集系统音频。", + "ttft_ms": 141, + "e2e_ms": 560, + "gen_tokens": 18, + "tps": 43.0, + "prompt_tokens": 141, + "prompt_ms": 137, + "output": "启用 WASAPI 回环，以便在不使用虚拟设备的情况下捕获系统音频。", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 1, + "terms_missing": [], + "chrf": 0.475 + }, + { + "engine": "llamacpp-metal", + "category": "E_tech_terms", + "id": "E04", + "run": 1, + "input": "The API returns a 429 when you exceed the rate limit, so add exponential backoff.", + "reference": "超过速率限制时API会返回429，所以要加指数退避。", + "ttft_ms": 190, + "e2e_ms": 738, + "gen_tokens": 23, + "tps": 41.9, + "prompt_tokens": 147, + "prompt_ms": 187, + "output": "当超过速率限制时，API 会返回 429，因此请添加指数退避。", + "compliant": true, + "problems": [], + "terms_total": 2, + "terms_kept": 2, + "terms_missing": [], + "chrf": 0.733 + }, + { + "engine": "llamacpp-metal", + "category": "E_tech_terms", + "id": "E05", + "run": 1, + "input": "We're seeing GPU memory pressure because num_ctx defaults to 32K.", + "reference": "我们遇到了GPU内存压力，因为num_ctx默认是32K。", + "ttft_ms": 138, + "e2e_ms": 564, + "gen_tokens": 18, + "tps": 42.2, + "prompt_tokens": 142, + "prompt_ms": 135, + "output": "我们看到 GPU 内存压力，因为 num_ctx 默认为 32K。", + "compliant": true, + "problems": [], + "terms_total": 3, + "terms_kept": 3, + "terms_missing": [], + "chrf": 0.766 + }, + { + "engine": "llamacpp-metal", + "category": "E_tech_terms", + "id": "E06", + "run": 1, + "input": "The PR adds a TranslationEngine protocol with an Ollama adapter and an embedded llama.cpp runtime.", + "reference": "这个PR增加了TranslationEngine协议，带有一个Ollama适配器和一个内置的llama.cpp运行时。", + "ttft_ms": 193, + "e2e_ms": 766, + "gen_tokens": 24, + "tps": 41.8, + "prompt_tokens": 146, + "prompt_ms": 190, + "output": "该 PR 添加了一个 TranslationEngine 协议，包含 Ollama 适配器及嵌入式 llama.cpp 运行时。", + "compliant": true, + "problems": [], + "terms_total": 4, + "terms_kept": 4, + "terms_missing": [], + "chrf": 0.697 + }, + { + "engine": "llamacpp-metal", + "category": "E_tech_terms", + "id": "E07", + "run": 1, + "input": "Notarization requires a Developer ID certificate and the hardened runtime.", + "reference": "公证需要Developer ID证书和强化运行时。", + "ttft_ms": 134, + "e2e_ms": 387, + "gen_tokens": 11, + "tps": 43.5, + "prompt_tokens": 139, + "prompt_ms": 132, + "output": "公证需要开发者 ID 证书和加固运行时。", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 0, + "terms_missing": [ + "Developer ID" + ], + "chrf": 0.392 + }, + { + "engine": "llamacpp-metal", + "category": "E_tech_terms", + "id": "E08", + "run": 1, + "input": "Our CI runs on windows-latest, which no longer ships NSIS.", + "reference": "我们的CI跑在windows-latest上，它不再自带NSIS。", + "ttft_ms": 136, + "e2e_ms": 583, + "gen_tokens": 19, + "tps": 42.5, + "prompt_tokens": 141, + "prompt_ms": 134, + "output": "我们的 CI 运行在 windows-latest 上，该镜像不再包含 NSIS。", + "compliant": true, + "problems": [], + "terms_total": 3, + "terms_kept": 3, + "terms_missing": [], + "chrf": 0.721 + }, + { + "engine": "llamacpp-metal", + "category": "F_long_sentences", + "id": "F01", + "run": 0, + "input": "What I want to emphasize today is that the transition to on-device inference is not primarily about cost, it's about latency and privacy, and those two things are what actually change user behavior.", + "reference": "我今天想强调的是，向设备端推理的转变主要不是为了成本，而是为了延迟和隐私，而这两点才是真正改变用户行为的因素。", + "ttft_ms": 197, + "e2e_ms": 943, + "gen_tokens": 31, + "tps": 41.5, + "prompt_tokens": 164, + "prompt_ms": 193, + "output": "今天我特别想强调的是，向端侧推理的转型并非主要关乎成本，而是关乎延迟与隐私，正是这两点真正改变了用户行为。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.429 + }, + { + "engine": "llamacpp-metal", + "category": "F_long_sentences", + "id": "F02", + "run": 0, + "input": "Before we commit to the Q3 roadmap, I'd like each team lead to send me a one-page summary of their top three risks and what support they need from leadership to mitigate them.", + "reference": "在我们确定第三季度路线图之前，我希望每位团队负责人给我发一页纸的摘要，说明他们最主要的三个风险，以及需要管理层提供什么支持来缓解这些风险。", + "ttft_ms": 196, + "e2e_ms": 1095, + "gen_tokens": 37, + "tps": 41.2, + "prompt_tokens": 164, + "prompt_ms": 191, + "output": "在我们确定第三季度路线图之前，我想请每位团队负责人向我发送一份单页总结，列出他们面临的三大主要风险，以及需要管理层提供哪些支持以缓解这些风险。", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 1, + "terms_missing": [], + "chrf": 0.594 + }, + { + "engine": "llamacpp-metal", + "category": "F_long_sentences", + "id": "F03", + "run": 0, + "input": "If the benchmark shows that the embedded runtime is within twenty percent of the Ollama path on first-token latency, I'm comfortable making it the default and keeping Ollama as an advanced option.", + "reference": "如果基准测试显示内置运行时的首字延迟与Ollama路径相差在百分之二十以内，我可以接受把它设为默认，并把Ollama保留为高级选项。", + "ttft_ms": 196, + "e2e_ms": 1250, + "gen_tokens": 43, + "tps": 40.8, + "prompt_tokens": 166, + "prompt_ms": 192, + "output": "如果基准测试显示，在首字延迟方面，嵌入式运行时距离 Ollama 路径在二十个百分点以内，那么我乐意将其设为默认选项，并将 Ollama 保留为高级选项。", + "compliant": true, + "problems": [], + "terms_total": 2, + "terms_kept": 2, + "terms_missing": [], + "chrf": 0.547 + }, + { + "engine": "llamacpp-metal", + "category": "F_long_sentences", + "id": "F04", + "run": 0, + "input": "The reason the microphone channel picks up the speaker audio is simply acoustic feedback, so with headphones the duplicate lines disappear, but we should still say that in onboarding.", + "reference": "麦克风通道会拾取扬声器声音的原因就是声学回授，所以戴上耳机重复的字幕就会消失，但我们仍然应该在引导里说明这一点。", + "ttft_ms": 207, + "e2e_ms": 979, + "gen_tokens": 32, + "tps": 41.4, + "prompt_tokens": 159, + "prompt_ms": 204, + "output": "麦克风通道拾取到扬声器声音的原因仅仅是声学反馈，因此戴上耳机后重复的音频就会消失，但我们仍应在入职培训中说明这一点。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.525 + }, + { + "engine": "llamacpp-metal", + "category": "F_long_sentences", + "id": "F05", + "run": 0, + "input": "Historically every attempt to bundle a second inference server inside a desktop app has failed on maintenance cost, so the question is whether a statically linked library changes that calculus.", + "reference": "历史上每一次在桌面应用里捆绑第二个推理服务的尝试都败在了维护成本上，所以问题在于静态链接的库是否改变了这个算计。", + "ttft_ms": 194, + "e2e_ms": 1069, + "gen_tokens": 36, + "tps": 41.1, + "prompt_tokens": 159, + "prompt_ms": 191, + "output": "历史上，任何试图在桌面应用程序中捆绑第二个推理服务器的尝试，都因维护成本过高而失败，因此问题在于：静态链接库是否会改变这一成本计算。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.456 + }, + { + "engine": "llamacpp-metal", + "category": "F_long_sentences", + "id": "F06", + "run": 0, + "input": "Let's park the pricing discussion, because until we know whether enterprise customers actually need SSO and audit logs, any number we put on the slide is going to be a guess.", + "reference": "我们先把定价讨论放一放，因为在知道企业客户是否真的需要SSO和审计日志之前，我们放在幻灯片上的任何数字都只是猜测。", + "ttft_ms": 194, + "e2e_ms": 1034, + "gen_tokens": 35, + "tps": 41.7, + "prompt_tokens": 162, + "prompt_ms": 191, + "output": "让我们先搁置定价讨论，因为在我们弄清楚企业客户是否真的需要 SSO 和审计日志之前，我们在幻灯片上列出的任何数字都只能是猜测。", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 1, + "terms_missing": [], + "chrf": 0.699 + }, + { + "engine": "llamacpp-metal", + "category": "F_long_sentences", + "id": "F01", + "run": 1, + "input": "What I want to emphasize today is that the transition to on-device inference is not primarily about cost, it's about latency and privacy, and those two things are what actually change user behavior.", + "reference": "我今天想强调的是，向设备端推理的转变主要不是为了成本，而是为了延迟和隐私，而这两点才是真正改变用户行为的因素。", + "ttft_ms": 212, + "e2e_ms": 921, + "gen_tokens": 30, + "tps": 42.3, + "prompt_tokens": 164, + "prompt_ms": 208, + "output": "今天我想要强调的是，向端侧推理的过渡主要并非关乎成本，而是关乎延迟和隐私，正是这两者真正改变了用户行为。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.426 + }, + { + "engine": "llamacpp-metal", + "category": "F_long_sentences", + "id": "F02", + "run": 1, + "input": "Before we commit to the Q3 roadmap, I'd like each team lead to send me a one-page summary of their top three risks and what support they need from leadership to mitigate them.", + "reference": "在我们确定第三季度路线图之前，我希望每位团队负责人给我发一页纸的摘要，说明他们最主要的三个风险，以及需要管理层提供什么支持来缓解这些风险。", + "ttft_ms": 204, + "e2e_ms": 1066, + "gen_tokens": 36, + "tps": 41.7, + "prompt_tokens": 164, + "prompt_ms": 199, + "output": "在我们承诺第三季度路线图之前，我希望每位团队负责人向我发送一份单页摘要，列出他们面临的三大主要风险，以及需要管理层提供何种支持以减轻这些风险。", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 1, + "terms_missing": [], + "chrf": 0.583 + }, + { + "engine": "llamacpp-metal", + "category": "F_long_sentences", + "id": "F03", + "run": 1, + "input": "If the benchmark shows that the embedded runtime is within twenty percent of the Ollama path on first-token latency, I'm comfortable making it the default and keeping Ollama as an advanced option.", + "reference": "如果基准测试显示内置运行时的首字延迟与Ollama路径相差在百分之二十以内，我可以接受把它设为默认，并把Ollama保留为高级选项。", + "ttft_ms": 202, + "e2e_ms": 1245, + "gen_tokens": 43, + "tps": 41.2, + "prompt_tokens": 166, + "prompt_ms": 197, + "output": "如果基准测试显示，在首字延迟方面，嵌入式运行时距离 Ollama 路径在二十个百分点以内，我就有把握将其设为默认选项，并将 Ollama 保留为高级选项。", + "compliant": true, + "problems": [], + "terms_total": 2, + "terms_kept": 2, + "terms_missing": [], + "chrf": 0.563 + }, + { + "engine": "llamacpp-metal", + "category": "F_long_sentences", + "id": "F04", + "run": 1, + "input": "The reason the microphone channel picks up the speaker audio is simply acoustic feedback, so with headphones the duplicate lines disappear, but we should still say that in onboarding.", + "reference": "麦克风通道会拾取扬声器声音的原因就是声学回授，所以戴上耳机重复的字幕就会消失，但我们仍然应该在引导里说明这一点。", + "ttft_ms": 203, + "e2e_ms": 968, + "gen_tokens": 32, + "tps": 41.9, + "prompt_tokens": 159, + "prompt_ms": 197, + "output": "麦克风通道拾取到扬声器声音的原因仅仅是声学反馈，因此佩戴耳机后重复的音频就会消失，但我们仍应在入职培训中说明这一点。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.498 + }, + { + "engine": "llamacpp-metal", + "category": "F_long_sentences", + "id": "F05", + "run": 1, + "input": "Historically every attempt to bundle a second inference server inside a desktop app has failed on maintenance cost, so the question is whether a statically linked library changes that calculus.", + "reference": "历史上每一次在桌面应用里捆绑第二个推理服务的尝试都败在了维护成本上，所以问题在于静态链接的库是否改变了这个算计。", + "ttft_ms": 198, + "e2e_ms": 1065, + "gen_tokens": 36, + "tps": 41.5, + "prompt_tokens": 159, + "prompt_ms": 194, + "output": "历史上，每次试图在桌面应用程序中捆绑第二个推理服务器时，都因维护成本过高而失败，那么问题是，静态链接的库是否会改变这一计算结果。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.46 + }, + { + "engine": "llamacpp-metal", + "category": "F_long_sentences", + "id": "F06", + "run": 1, + "input": "Let's park the pricing discussion, because until we know whether enterprise customers actually need SSO and audit logs, any number we put on the slide is going to be a guess.", + "reference": "我们先把定价讨论放一放，因为在知道企业客户是否真的需要SSO和审计日志之前，我们放在幻灯片上的任何数字都只是猜测。", + "ttft_ms": 199, + "e2e_ms": 1042, + "gen_tokens": 35, + "tps": 41.5, + "prompt_tokens": 162, + "prompt_ms": 195, + "output": "让我们暂时搁置定价讨论，因为在我们尚未明确企业客户是否真正需要 SSO 和审计日志之前，幻灯片上列出的任何数字都将是猜测。", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 1, + "terms_missing": [], + "chrf": 0.587 + }, + { + "engine": "llamacpp-metal", + "category": "G_zh_to_en", + "id": "G01", + "run": 0, + "input": "这个功能我们下周三之前能上线吗", + "reference": "Can we ship this feature before next Wednesday?", + "ttft_ms": 438, + "e2e_ms": 660, + "gen_tokens": 10, + "tps": 45.0, + "prompt_tokens": 132, + "prompt_ms": 413, + "output": "Can this feature be launched before next Wednesday?", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 1, + "terms_missing": [], + "chrf": 0.793 + }, + { + "engine": "llamacpp-metal", + "category": "G_zh_to_en", + "id": "G02", + "run": 0, + "input": "那个就是说我们现在有两个方案一个是自己做一个是买第三方的", + "reference": "So we have two options now: build it ourselves, or buy a third-party solution.", + "ttft_ms": 135, + "e2e_ms": 859, + "gen_tokens": 30, + "tps": 41.4, + "prompt_tokens": 135, + "prompt_ms": 132, + "output": "That is to say, we currently have two options: one is to build it ourselves, and the other is to purchase a third-party solution.", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 1, + "terms_missing": [], + "chrf": 0.703 + }, + { + "engine": "llamacpp-metal", + "category": "G_zh_to_en", + "id": "G03", + "run": 0, + "input": "上个季度我们的留存率从百分之三十八涨到了百分之四十五", + "reference": "Last quarter our retention rate went from 38% to 45%.", + "ttft_ms": 136, + "e2e_ms": 589, + "gen_tokens": 19, + "tps": 41.9, + "prompt_tokens": 137, + "prompt_ms": 133, + "output": "Last quarter, our retention rate rose from 38 percent to 45 percent.", + "compliant": true, + "problems": [], + "terms_total": 2, + "terms_kept": 2, + "terms_missing": [], + "chrf": 0.74 + }, + { + "engine": "llamacpp-metal", + "category": "G_zh_to_en", + "id": "G04", + "run": 0, + "input": "我觉得这个风险有点大，要不我们先小范围灰度一下", + "reference": "I think the risk is a bit high. How about a small-scale gradual rollout first?", + "ttft_ms": 134, + "e2e_ms": 605, + "gen_tokens": 20, + "tps": 42.5, + "prompt_tokens": 136, + "prompt_ms": 132, + "output": "I think this risk is quite high; shall we start with a small-scale gray release first?", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.576 + }, + { + "engine": "llamacpp-metal", + "category": "G_zh_to_en", + "id": "G05", + "run": 0, + "input": "张总说预算最多给到五十万，超了要重新审批", + "reference": "Mr. Zhang said the budget is capped at 500 thousand; anything above that needs re-approval.", + "ttft_ms": 139, + "e2e_ms": 788, + "gen_tokens": 27, + "tps": 41.6, + "prompt_tokens": 138, + "prompt_ms": 136, + "output": "Mr. Zhang said the budget is capped at 500,000; anything exceeding that requires re-approval.", + "compliant": true, + "problems": [], + "terms_total": 2, + "terms_kept": 2, + "terms_missing": [], + "chrf": 0.738 + }, + { + "engine": "llamacpp-metal", + "category": "G_zh_to_en", + "id": "G06", + "run": 0, + "input": "识别延迟大概一点五秒，翻译再加一秒左右，用户基本感觉不到", + "reference": "Recognition latency is about 1.5 seconds, translation adds roughly another second, so users barely notice.", + "ttft_ms": 142, + "e2e_ms": 837, + "gen_tokens": 29, + "tps": 41.7, + "prompt_tokens": 139, + "prompt_ms": 139, + "output": "The latency for recognition is approximately 1.5 seconds, and translation adds about one more second, so users generally don't notice it.", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 1, + "terms_missing": [], + "chrf": 0.705 + }, + { + "engine": "llamacpp-metal", + "category": "G_zh_to_en", + "id": "G07", + "run": 0, + "input": "等一下我这边网不太好你们先聊", + "reference": "Hold on, my connection isn't great. You go ahead.", + "ttft_ms": 134, + "e2e_ms": 509, + "gen_tokens": 16, + "tps": 42.6, + "prompt_tokens": 131, + "prompt_ms": 132, + "output": "Wait, my internet connection is poor here; you go ahead and chat.", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.578 + }, + { + "engine": "llamacpp-metal", + "category": "G_zh_to_en", + "id": "G08", + "run": 0, + "input": "这个PR先不要合，Codex还有两条评论没处理", + "reference": "Don't merge this PR yet; there are still two Codex comments to address.", + "ttft_ms": 135, + "e2e_ms": 562, + "gen_tokens": 18, + "tps": 42.1, + "prompt_tokens": 136, + "prompt_ms": 132, + "output": "Let's not merge this PR yet; Codex still has two comments to address.", + "compliant": true, + "problems": [], + "terms_total": 3, + "terms_kept": 3, + "terms_missing": [], + "chrf": 0.742 + }, + { + "engine": "llamacpp-metal", + "category": "G_zh_to_en", + "id": "G01", + "run": 1, + "input": "这个功能我们下周三之前能上线吗", + "reference": "Can we ship this feature before next Wednesday?", + "ttft_ms": 138, + "e2e_ms": 363, + "gen_tokens": 10, + "tps": 44.6, + "prompt_tokens": 132, + "prompt_ms": 135, + "output": "Can this feature be launched by next Wednesday?", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 1, + "terms_missing": [], + "chrf": 0.673 + }, + { + "engine": "llamacpp-metal", + "category": "G_zh_to_en", + "id": "G02", + "run": 1, + "input": "那个就是说我们现在有两个方案一个是自己做一个是买第三方的", + "reference": "So we have two options now: build it ourselves, or buy a third-party solution.", + "ttft_ms": 139, + "e2e_ms": 869, + "gen_tokens": 30, + "tps": 41.1, + "prompt_tokens": 135, + "prompt_ms": 137, + "output": "That is to say, we currently have two options: one is to build it ourselves, and the other is to purchase a third-party solution.", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 1, + "terms_missing": [], + "chrf": 0.703 + }, + { + "engine": "llamacpp-metal", + "category": "G_zh_to_en", + "id": "G03", + "run": 1, + "input": "上个季度我们的留存率从百分之三十八涨到了百分之四十五", + "reference": "Last quarter our retention rate went from 38% to 45%.", + "ttft_ms": 152, + "e2e_ms": 600, + "gen_tokens": 19, + "tps": 42.4, + "prompt_tokens": 137, + "prompt_ms": 145, + "output": "Last quarter, our retention rate rose from 38 percent to 45 percent.", + "compliant": true, + "problems": [], + "terms_total": 2, + "terms_kept": 2, + "terms_missing": [], + "chrf": 0.74 + }, + { + "engine": "llamacpp-metal", + "category": "G_zh_to_en", + "id": "G04", + "run": 1, + "input": "我觉得这个风险有点大，要不我们先小范围灰度一下", + "reference": "I think the risk is a bit high. How about a small-scale gradual rollout first?", + "ttft_ms": 133, + "e2e_ms": 569, + "gen_tokens": 19, + "tps": 43.6, + "prompt_tokens": 136, + "prompt_ms": 131, + "output": "I think the risk is quite high; shall we start with a small-scale rollout first?", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.662 + }, + { + "engine": "llamacpp-metal", + "category": "G_zh_to_en", + "id": "G05", + "run": 1, + "input": "张总说预算最多给到五十万，超了要重新审批", + "reference": "Mr. Zhang said the budget is capped at 500 thousand; anything above that needs re-approval.", + "ttft_ms": 136, + "e2e_ms": 780, + "gen_tokens": 27, + "tps": 41.9, + "prompt_tokens": 138, + "prompt_ms": 134, + "output": "Mr. Zhang said the budget is capped at 500,000; anything exceeding that requires re-approval.", + "compliant": true, + "problems": [], + "terms_total": 2, + "terms_kept": 2, + "terms_missing": [], + "chrf": 0.738 + }, + { + "engine": "llamacpp-metal", + "category": "G_zh_to_en", + "id": "G06", + "run": 1, + "input": "识别延迟大概一点五秒，翻译再加一秒左右，用户基本感觉不到", + "reference": "Recognition latency is about 1.5 seconds, translation adds roughly another second, so users barely notice.", + "ttft_ms": 145, + "e2e_ms": 783, + "gen_tokens": 26, + "tps": 40.7, + "prompt_tokens": 139, + "prompt_ms": 141, + "output": "Recognition delay is approximately 1.5 seconds, plus about one second for translation, and users generally do not notice it.", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 1, + "terms_missing": [], + "chrf": 0.622 + }, + { + "engine": "llamacpp-metal", + "category": "G_zh_to_en", + "id": "G07", + "run": 1, + "input": "等一下我这边网不太好你们先聊", + "reference": "Hold on, my connection isn't great. You go ahead.", + "ttft_ms": 140, + "e2e_ms": 614, + "gen_tokens": 20, + "tps": 42.2, + "prompt_tokens": 131, + "prompt_ms": 137, + "output": "Wait, my internet connection isn't very good right now; you guys go ahead and talk.", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.567 + }, + { + "engine": "llamacpp-metal", + "category": "G_zh_to_en", + "id": "G08", + "run": 1, + "input": "这个PR先不要合，Codex还有两条评论没处理", + "reference": "Don't merge this PR yet; there are still two Codex comments to address.", + "ttft_ms": 136, + "e2e_ms": 560, + "gen_tokens": 18, + "tps": 42.4, + "prompt_tokens": 136, + "prompt_ms": 133, + "output": "Let's not merge this PR yet; Codex still has two comments to address.", + "compliant": true, + "problems": [], + "terms_total": 3, + "terms_kept": 3, + "terms_missing": [], + "chrf": 0.742 + }, + { + "engine": "llamacpp-metal", + "category": "H_context_followup", + "id": "H01", + "run": 0, + "input": "We benchmarked the MLX build last week.", + "reference": "我们上周对MLX版本做了基准测试。", + "ttft_ms": 202, + "e2e_ms": 478, + "gen_tokens": 12, + "tps": 43.4, + "prompt_tokens": 136, + "prompt_ms": 148, + "output": "我们上周对 MLX 版本进行了基准测试。", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 1, + "terms_missing": [], + "chrf": 0.822 + }, + { + "engine": "llamacpp-metal", + "category": "H_context_followup", + "id": "H02", + "run": 0, + "input": "It was about twice as fast on first token.", + "reference": "它的首字速度大约快了一倍。", + "ttft_ms": 228, + "e2e_ms": 454, + "gen_tokens": 10, + "tps": 44.2, + "prompt_tokens": 173, + "prompt_ms": 224, + "output": "首字生成速度大约快了两倍。", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 1, + "terms_missing": [], + "chrf": 0.504 + }, + { + "engine": "llamacpp-metal", + "category": "H_context_followup", + "id": "H03", + "run": 0, + "input": "But it only works on Apple Silicon.", + "reference": "但它只能在Apple Silicon上运行。", + "ttft_ms": 205, + "e2e_ms": 407, + "gen_tokens": 9, + "tps": 44.6, + "prompt_tokens": 206, + "prompt_ms": 198, + "output": "但它仅在 Apple Silicon 上运行。", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 1, + "terms_missing": [], + "chrf": 0.839 + }, + { + "engine": "llamacpp-metal", + "category": "H_context_followup", + "id": "H04", + "run": 0, + "input": "So for Intel Macs we'd still need the other one.", + "reference": "所以对于Intel的Mac我们仍然需要另一个。", + "ttft_ms": 364, + "e2e_ms": 667, + "gen_tokens": 13, + "tps": 43.0, + "prompt_tokens": 206, + "prompt_ms": 361, + "output": "因此，对于 Intel Mac 我们仍然需要另一个版本。", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 1, + "terms_missing": [], + "chrf": 0.738 + }, + { + "engine": "llamacpp-metal", + "category": "H_context_followup", + "id": "H05", + "run": 0, + "input": "She asked whether that doubles the maintenance work.", + "reference": "她问这是否会让维护工作翻倍。", + "ttft_ms": 363, + "e2e_ms": 586, + "gen_tokens": 10, + "tps": 44.9, + "prompt_tokens": 208, + "prompt_ms": 355, + "output": "她问这是否会使维护工作量翻倍。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.651 + }, + { + "engine": "llamacpp-metal", + "category": "H_context_followup", + "id": "H06", + "run": 0, + "input": "Honestly, it probably does.", + "reference": "说实话，可能会。", + "ttft_ms": 356, + "e2e_ms": 543, + "gen_tokens": 8, + "tps": 42.7, + "prompt_tokens": 207, + "prompt_ms": 352, + "output": "老实说，很可能确实如此。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.203 + }, + { + "engine": "llamacpp-metal", + "category": "H_context_followup", + "id": "H01", + "run": 1, + "input": "We benchmarked the MLX build last week.", + "reference": "我们上周对MLX版本做了基准测试。", + "ttft_ms": 161, + "e2e_ms": 435, + "gen_tokens": 12, + "tps": 43.7, + "prompt_tokens": 136, + "prompt_ms": 136, + "output": "我们上周对 MLX 版本进行了基准测试。", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 1, + "terms_missing": [], + "chrf": 0.822 + }, + { + "engine": "llamacpp-metal", + "category": "H_context_followup", + "id": "H02", + "run": 1, + "input": "It was about twice as fast on first token.", + "reference": "它的首字速度大约快了一倍。", + "ttft_ms": 209, + "e2e_ms": 433, + "gen_tokens": 10, + "tps": 44.6, + "prompt_tokens": 173, + "prompt_ms": 206, + "output": "首词生成速度大约快了两倍。", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 1, + "terms_missing": [], + "chrf": 0.464 + }, + { + "engine": "llamacpp-metal", + "category": "H_context_followup", + "id": "H03", + "run": 1, + "input": "But it only works on Apple Silicon.", + "reference": "但它只能在Apple Silicon上运行。", + "ttft_ms": 210, + "e2e_ms": 359, + "gen_tokens": 7, + "tps": 47.2, + "prompt_tokens": 206, + "prompt_ms": 206, + "output": "但它仅适用于 Apple Silicon。", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 1, + "terms_missing": [], + "chrf": 0.597 + }, + { + "engine": "llamacpp-metal", + "category": "H_context_followup", + "id": "H04", + "run": 1, + "input": "So for Intel Macs we'd still need the other one.", + "reference": "所以对于Intel的Mac我们仍然需要另一个。", + "ttft_ms": 369, + "e2e_ms": 664, + "gen_tokens": 13, + "tps": 44.0, + "prompt_tokens": 204, + "prompt_ms": 366, + "output": "因此，对于 Intel Mac，我们仍然需要另一个版本。", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 1, + "terms_missing": [], + "chrf": 0.661 + }, + { + "engine": "llamacpp-metal", + "category": "H_context_followup", + "id": "H05", + "run": 1, + "input": "She asked whether that doubles the maintenance work.", + "reference": "她问这是否会让维护工作翻倍。", + "ttft_ms": 353, + "e2e_ms": 578, + "gen_tokens": 10, + "tps": 44.5, + "prompt_tokens": 206, + "prompt_ms": 350, + "output": "她问这是否会使维护工作量翻倍。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.651 + }, + { + "engine": "llamacpp-metal", + "category": "H_context_followup", + "id": "H06", + "run": 1, + "input": "Honestly, it probably does.", + "reference": "说实话，可能会。", + "ttft_ms": 352, + "e2e_ms": 528, + "gen_tokens": 8, + "tps": 45.5, + "prompt_tokens": 207, + "prompt_ms": 349, + "output": "老实说，很可能确实如此。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.203 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "A_short_turns", + "id": "A01", + "run": 0, + "input": "Let's get started.", + "reference": "我们开始吧。", + "ttft_ms": 104, + "e2e_ms": 135, + "gen_tokens": 4, + "tps": 126.9, + "prompt_tokens": 121, + "prompt_ms": 102, + "output": "让我们开始。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.567 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "A_short_turns", + "id": "A02", + "run": 0, + "input": "Can everyone see my screen?", + "reference": "大家能看到我的屏幕吗？", + "ttft_ms": 28, + "e2e_ms": 96, + "gen_tokens": 7, + "tps": 103.3, + "prompt_tokens": 122, + "prompt_ms": 26, + "output": "能否所有人能看到我的屏幕？", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.572 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "A_short_turns", + "id": "A03", + "run": 0, + "input": "Sorry, you cut out for a second.", + "reference": "抱歉，你刚才断了一下。", + "ttft_ms": 28, + "e2e_ms": 98, + "gen_tokens": 8, + "tps": 113.3, + "prompt_tokens": 125, + "prompt_ms": 26, + "output": "抱歉，我短暂中断了一下。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.495 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "A_short_turns", + "id": "A04", + "run": 0, + "input": "I'll take that offline.", + "reference": "这个我们线下再聊。", + "ttft_ms": 26, + "e2e_ms": 99, + "gen_tokens": 8, + "tps": 109.6, + "prompt_tokens": 122, + "prompt_ms": 24, + "output": "我将把它转到线下。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.142 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "A_short_turns", + "id": "A05", + "run": 0, + "input": "Any questions before we move on?", + "reference": "在继续之前有什么问题吗？", + "ttft_ms": 27, + "e2e_ms": 99, + "gen_tokens": 8, + "tps": 111.0, + "prompt_tokens": 123, + "prompt_ms": 25, + "output": "任何在我们继续之前的问题？", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.329 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "A_short_turns", + "id": "A06", + "run": 0, + "input": "That makes sense, thanks for clarifying.", + "reference": "明白了，谢谢你的说明。", + "ttft_ms": 27, + "e2e_ms": 79, + "gen_tokens": 6, + "tps": 115.3, + "prompt_tokens": 125, + "prompt_ms": 26, + "output": "明白了，谢谢澄清。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.51 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "A_short_turns", + "id": "A07", + "run": 0, + "input": "We're running about five minutes behind.", + "reference": "我们大概晚了五分钟。", + "ttft_ms": 27, + "e2e_ms": 88, + "gen_tokens": 7, + "tps": 114.6, + "prompt_tokens": 124, + "prompt_ms": 25, + "output": "我们大约落后五分钟左右。", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 1, + "terms_missing": [], + "chrf": 0.334 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "A_short_turns", + "id": "A08", + "run": 0, + "input": "Let me share the link in the chat.", + "reference": "我把链接发到聊天里。", + "ttft_ms": 27, + "e2e_ms": 109, + "gen_tokens": 9, + "tps": 110.1, + "prompt_tokens": 125, + "prompt_ms": 26, + "output": "请让我在聊天中分享链接。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.197 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "A_short_turns", + "id": "A01", + "run": 1, + "input": "Let's get started.", + "reference": "我们开始吧。", + "ttft_ms": 26, + "e2e_ms": 59, + "gen_tokens": 4, + "tps": 122.0, + "prompt_tokens": 121, + "prompt_ms": 25, + "output": "让我们开始。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.567 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "A_short_turns", + "id": "A02", + "run": 1, + "input": "Can everyone see my screen?", + "reference": "大家能看到我的屏幕吗？", + "ttft_ms": 26, + "e2e_ms": 89, + "gen_tokens": 7, + "tps": 111.8, + "prompt_tokens": 122, + "prompt_ms": 25, + "output": "能否 everyone see my screen？", + "compliant": false, + "problems": [ + "wrong-script" + ], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.038 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "A_short_turns", + "id": "A03", + "run": 1, + "input": "Sorry, you cut out for a second.", + "reference": "抱歉，你刚才断了一下。", + "ttft_ms": 28, + "e2e_ms": 100, + "gen_tokens": 8, + "tps": 111.3, + "prompt_tokens": 125, + "prompt_ms": 26, + "output": "抱歉，我短暂中断了一下。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.495 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "A_short_turns", + "id": "A04", + "run": 1, + "input": "I'll take that offline.", + "reference": "这个我们线下再聊。", + "ttft_ms": 26, + "e2e_ms": 90, + "gen_tokens": 7, + "tps": 108.6, + "prompt_tokens": 122, + "prompt_ms": 24, + "output": "我会把它放到线下处理。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.136 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "A_short_turns", + "id": "A05", + "run": 1, + "input": "Any questions before we move on?", + "reference": "在继续之前有什么问题吗？", + "ttft_ms": 28, + "e2e_ms": 103, + "gen_tokens": 8, + "tps": 106.9, + "prompt_tokens": 123, + "prompt_ms": 26, + "output": "任何问题在我们继续之前？", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.335 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "A_short_turns", + "id": "A06", + "run": 1, + "input": "That makes sense, thanks for clarifying.", + "reference": "明白了，谢谢你的说明。", + "ttft_ms": 28, + "e2e_ms": 81, + "gen_tokens": 6, + "tps": 112.9, + "prompt_tokens": 125, + "prompt_ms": 26, + "output": "明白了，谢谢澄清。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.51 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "A_short_turns", + "id": "A07", + "run": 1, + "input": "We're running about five minutes behind.", + "reference": "我们大概晚了五分钟。", + "ttft_ms": 28, + "e2e_ms": 81, + "gen_tokens": 6, + "tps": 112.3, + "prompt_tokens": 124, + "prompt_ms": 26, + "output": "我们大约落后五分钟。", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 1, + "terms_missing": [], + "chrf": 0.443 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "A_short_turns", + "id": "A08", + "run": 1, + "input": "Let me share the link in the chat.", + "reference": "我把链接发到聊天里。", + "ttft_ms": 28, + "e2e_ms": 110, + "gen_tokens": 9, + "tps": 109.4, + "prompt_tokens": 125, + "prompt_ms": 26, + "output": "请让我在聊天中分享链接。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.197 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "B_asr_drafts", + "id": "B01", + "run": 0, + "input": "so what we found was that the", + "reference": "所以我们发现的是", + "ttft_ms": 27, + "e2e_ms": 82, + "gen_tokens": 6, + "tps": 108.9, + "prompt_tokens": 123, + "prompt_ms": 25, + "output": "我们发现的情况是：", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.539 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "B_asr_drafts", + "id": "B02", + "run": 0, + "input": "if you look at the second chart you can see", + "reference": "如果你看第二张图，你可以看到", + "ttft_ms": 31, + "e2e_ms": 116, + "gen_tokens": 9, + "tps": 105.1, + "prompt_tokens": 126, + "prompt_ms": 29, + "output": "如果你查看第二张图表，可以看到。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.58 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "B_asr_drafts", + "id": "B03", + "run": 0, + "input": "the main reason we chose this approach is", + "reference": "我们选择这个方案的主要原因是", + "ttft_ms": 29, + "e2e_ms": 79, + "gen_tokens": 6, + "tps": 120.2, + "prompt_tokens": 124, + "prompt_ms": 27, + "output": "我们选择这种方法的主要原因是", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.647 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "B_asr_drafts", + "id": "B04", + "run": 0, + "input": "and then the customer said that they", + "reference": "然后客户说他们", + "ttft_ms": 27, + "e2e_ms": 68, + "gen_tokens": 5, + "tps": 122.4, + "prompt_tokens": 123, + "prompt_ms": 26, + "output": "然后客户说他们", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 1.0 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "B_asr_drafts", + "id": "B05", + "run": 0, + "input": "I think the biggest risk here", + "reference": "我觉得这里最大的风险", + "ttft_ms": 25, + "e2e_ms": 86, + "gen_tokens": 7, + "tps": 114.9, + "prompt_tokens": 122, + "prompt_ms": 24, + "output": "我认为这里最大的风险是。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.636 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "B_asr_drafts", + "id": "B06", + "run": 0, + "input": "we need to decide by", + "reference": "我们需要在……之前决定", + "ttft_ms": 25, + "e2e_ms": 56, + "gen_tokens": 4, + "tps": 128.6, + "prompt_tokens": 121, + "prompt_ms": 24, + "output": "我们需要决定。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.352 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "B_asr_drafts", + "id": "B07", + "run": 0, + "input": "which brings me to", + "reference": "这就引出了", + "ttft_ms": 24, + "e2e_ms": 65, + "gen_tokens": 5, + "tps": 123.5, + "prompt_tokens": 120, + "prompt_ms": 23, + "output": "哪一句让我想到", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.0 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "B_asr_drafts", + "id": "B08", + "run": 0, + "input": "the data shows a clear", + "reference": "数据显示出明显的", + "ttft_ms": 25, + "e2e_ms": 76, + "gen_tokens": 6, + "tps": 117.6, + "prompt_tokens": 121, + "prompt_ms": 24, + "output": "数据显示有明显的趋势。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.526 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "B_asr_drafts", + "id": "B01", + "run": 1, + "input": "so what we found was that the", + "reference": "所以我们发现的是", + "ttft_ms": 26, + "e2e_ms": 78, + "gen_tokens": 6, + "tps": 115.6, + "prompt_tokens": 123, + "prompt_ms": 24, + "output": "我们发现的情况是：", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.539 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "B_asr_drafts", + "id": "B02", + "run": 1, + "input": "if you look at the second chart you can see", + "reference": "如果你看第二张图，你可以看到", + "ttft_ms": 28, + "e2e_ms": 111, + "gen_tokens": 9, + "tps": 108.4, + "prompt_tokens": 126, + "prompt_ms": 26, + "output": "如果你查看第二张图表，可以看到。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.58 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "B_asr_drafts", + "id": "B03", + "run": 1, + "input": "the main reason we chose this approach is", + "reference": "我们选择这个方案的主要原因是", + "ttft_ms": 27, + "e2e_ms": 79, + "gen_tokens": 6, + "tps": 114.7, + "prompt_tokens": 124, + "prompt_ms": 25, + "output": "我们选择这种方法的主要原因是", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.647 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "B_asr_drafts", + "id": "B04", + "run": 1, + "input": "and then the customer said that they", + "reference": "然后客户说他们", + "ttft_ms": 26, + "e2e_ms": 68, + "gen_tokens": 5, + "tps": 118.2, + "prompt_tokens": 123, + "prompt_ms": 24, + "output": "然后客户说他们", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 1.0 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "B_asr_drafts", + "id": "B05", + "run": 1, + "input": "I think the biggest risk here", + "reference": "我觉得这里最大的风险", + "ttft_ms": 25, + "e2e_ms": 88, + "gen_tokens": 7, + "tps": 111.9, + "prompt_tokens": 122, + "prompt_ms": 24, + "output": "我觉得这里最大的风险是。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.954 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "B_asr_drafts", + "id": "B06", + "run": 1, + "input": "we need to decide by", + "reference": "我们需要在……之前决定", + "ttft_ms": 26, + "e2e_ms": 58, + "gen_tokens": 4, + "tps": 126.3, + "prompt_tokens": 121, + "prompt_ms": 25, + "output": "我们需要决定。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.352 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "B_asr_drafts", + "id": "B07", + "run": 1, + "input": "which brings me to", + "reference": "这就引出了", + "ttft_ms": 25, + "e2e_ms": 56, + "gen_tokens": 4, + "tps": 126.5, + "prompt_tokens": 120, + "prompt_ms": 23, + "output": "哪一个让我想到", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.0 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "B_asr_drafts", + "id": "B08", + "run": 1, + "input": "the data shows a clear", + "reference": "数据显示出明显的", + "ttft_ms": 26, + "e2e_ms": 77, + "gen_tokens": 6, + "tps": 117.9, + "prompt_tokens": 121, + "prompt_ms": 24, + "output": "数据显示情况非常清楚。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.336 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "C_asr_noisy", + "id": "C01", + "run": 0, + "input": "um so yeah basically what we're gonna do is we're gonna ship it next week", + "reference": "所以基本上我们要做的就是下周发布。", + "ttft_ms": 122, + "e2e_ms": 172, + "gen_tokens": 6, + "tps": 117.9, + "prompt_tokens": 133, + "prompt_ms": 120, + "output": "我们下周会发货。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.134 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "C_asr_noisy", + "id": "C02", + "run": 0, + "input": "i mean it's it's not perfect but it works you know", + "reference": "我是说它不完美，但能用，你懂的。", + "ttft_ms": 42, + "e2e_ms": 160, + "gen_tokens": 13, + "tps": 110.5, + "prompt_tokens": 129, + "prompt_ms": 40, + "output": "那就是，虽然它并不完美，但确实能用。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.338 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "C_asr_noisy", + "id": "C03", + "run": 0, + "input": "okay okay let me let me just pull that up real quick", + "reference": "好好，让我快速调出来看一下。", + "ttft_ms": 41, + "e2e_ms": 146, + "gen_tokens": 11, + "tps": 104.8, + "prompt_tokens": 128, + "prompt_ms": 39, + "output": "好的，好的，我来快速找一下。", + "compliant": false, + "problems": [ + "preamble:好的" + ], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.257 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "C_asr_noisy", + "id": "C04", + "run": 0, + "input": "right so the the thing is we don't actually have the budget for that", + "reference": "对，问题是我们其实没有这方面的预算。", + "ttft_ms": 40, + "e2e_ms": 121, + "gen_tokens": 9, + "tps": 110.2, + "prompt_tokens": 131, + "prompt_ms": 38, + "output": "好的，我们实际上没有那个预算。", + "compliant": false, + "problems": [ + "preamble:好的" + ], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.221 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "C_asr_noisy", + "id": "C05", + "run": 0, + "input": "like honestly I don't think anyone has looked at this since like march", + "reference": "说实话我觉得从三月以来就没人看过这个了。", + "ttft_ms": 39, + "e2e_ms": 170, + "gen_tokens": 14, + "tps": 106.7, + "prompt_tokens": 130, + "prompt_ms": 38, + "output": "看起来没有人在这方面进行过研究，自从三月以来。", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 1, + "terms_missing": [], + "chrf": 0.253 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "C_asr_noisy", + "id": "C06", + "run": 0, + "input": "and uh we also need to loop in the legal team before we sign anything", + "reference": "另外在签任何东西之前我们还需要让法务团队参与进来。", + "ttft_ms": 40, + "e2e_ms": 173, + "gen_tokens": 14, + "tps": 105.5, + "prompt_tokens": 131, + "prompt_ms": 39, + "output": "我们还需要在签署任何文件之前将法律团队纳入进来。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.343 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "C_asr_noisy", + "id": "C07", + "run": 0, + "input": "yeah no that's fair that's totally fair", + "reference": "对，没错，这很合理。", + "ttft_ms": 27, + "e2e_ms": 159, + "gen_tokens": 14, + "tps": 106.7, + "prompt_tokens": 125, + "prompt_ms": 26, + "output": "嗯，那算是公平的，那真的是完全公平的。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.064 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "C_asr_noisy", + "id": "C08", + "run": 0, + "input": "so hold on are we saying the deadline moved or not", + "reference": "等等，我们是说截止日期改了还是没改？", + "ttft_ms": 43, + "e2e_ms": 154, + "gen_tokens": 11, + "tps": 99.1, + "prompt_tokens": 127, + "prompt_ms": 41, + "output": "我们说截止日期是提前还是推迟了？", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.343 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "C_asr_noisy", + "id": "C01", + "run": 1, + "input": "um so yeah basically what we're gonna do is we're gonna ship it next week", + "reference": "所以基本上我们要做的就是下周发布。", + "ttft_ms": 41, + "e2e_ms": 94, + "gen_tokens": 6, + "tps": 112.6, + "prompt_tokens": 133, + "prompt_ms": 39, + "output": "我们下周会发货。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.134 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "C_asr_noisy", + "id": "C02", + "run": 1, + "input": "i mean it's it's not perfect but it works you know", + "reference": "我是说它不完美，但能用，你懂的。", + "ttft_ms": 43, + "e2e_ms": 165, + "gen_tokens": 13, + "tps": 106.4, + "prompt_tokens": 129, + "prompt_ms": 41, + "output": "嗯，虽然它并不完美，但确实能用。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.332 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "C_asr_noisy", + "id": "C03", + "run": 1, + "input": "okay okay let me let me just pull that up real quick", + "reference": "好好，让我快速调出来看一下。", + "ttft_ms": 41, + "e2e_ms": 110, + "gen_tokens": 8, + "tps": 115.3, + "prompt_tokens": 128, + "prompt_ms": 40, + "output": "好的，让我快速找一下。", + "compliant": false, + "problems": [ + "preamble:好的" + ], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.425 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "C_asr_noisy", + "id": "C04", + "run": 1, + "input": "right so the the thing is we don't actually have the budget for that", + "reference": "对，问题是我们其实没有这方面的预算。", + "ttft_ms": 39, + "e2e_ms": 118, + "gen_tokens": 9, + "tps": 114.9, + "prompt_tokens": 131, + "prompt_ms": 38, + "output": "好的，我们实际并没有那个预算。", + "compliant": false, + "problems": [ + "preamble:好的" + ], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.221 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "C_asr_noisy", + "id": "C05", + "run": 1, + "input": "like honestly I don't think anyone has looked at this since like march", + "reference": "说实话我觉得从三月以来就没人看过这个了。", + "ttft_ms": 39, + "e2e_ms": 148, + "gen_tokens": 12, + "tps": 110.4, + "prompt_tokens": 130, + "prompt_ms": 38, + "output": "看起来没有人从3月份以来对此做过任何分析。", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 1, + "terms_missing": [], + "chrf": 0.124 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "C_asr_noisy", + "id": "C06", + "run": 1, + "input": "and uh we also need to loop in the legal team before we sign anything", + "reference": "另外在签任何东西之前我们还需要让法务团队参与进来。", + "ttft_ms": 39, + "e2e_ms": 168, + "gen_tokens": 14, + "tps": 109.0, + "prompt_tokens": 131, + "prompt_ms": 38, + "output": "我们还需要在签署任何文件之前将法律团队纳入进来。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.343 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "C_asr_noisy", + "id": "C07", + "run": 1, + "input": "yeah no that's fair that's totally fair", + "reference": "对，没错，这很合理。", + "ttft_ms": 27, + "e2e_ms": 149, + "gen_tokens": 13, + "tps": 107.2, + "prompt_tokens": 125, + "prompt_ms": 26, + "output": "哦，那算是公平的，那完全公平的。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.067 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "C_asr_noisy", + "id": "C08", + "run": 1, + "input": "so hold on are we saying the deadline moved or not", + "reference": "等等，我们是说截止日期改了还是没改？", + "ttft_ms": 42, + "e2e_ms": 145, + "gen_tokens": 11, + "tps": 107.2, + "prompt_tokens": 127, + "prompt_ms": 41, + "output": "我们说截止日期是提前还是推迟了？", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.343 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "D_numbers_names", + "id": "D01", + "run": 0, + "input": "Revenue grew 12% year over year to $4.3 billion.", + "reference": "收入同比增长12%，达到43亿美元。", + "ttft_ms": 39, + "e2e_ms": 141, + "gen_tokens": 11, + "tps": 108.4, + "prompt_tokens": 132, + "prompt_ms": 38, + "output": "营业收入同比增长12%至43亿元。", + "compliant": true, + "problems": [], + "terms_total": 2, + "terms_kept": 2, + "terms_missing": [], + "chrf": 0.588 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "D_numbers_names", + "id": "D02", + "run": 0, + "input": "Sarah from Stripe will join us on March 15th at 3 p.m.", + "reference": "Stripe的Sarah将在3月15日下午3点加入我们。", + "ttft_ms": 39, + "e2e_ms": 219, + "gen_tokens": 19, + "tps": 105.5, + "prompt_tokens": 134, + "prompt_ms": 38, + "output": "Sarah from Stripe will join us on March 15th at 3 p.m.", + "compliant": false, + "problems": [ + "echo", + "wrong-script" + ], + "terms_total": 3, + "terms_kept": 3, + "terms_missing": [], + "chrf": 0.301 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "D_numbers_names", + "id": "D03", + "run": 0, + "input": "We processed 2.7 million requests with a p99 latency of 180 milliseconds.", + "reference": "我们处理了270万个请求，p99延迟为180毫秒。", + "ttft_ms": 40, + "e2e_ms": 261, + "gen_tokens": 23, + "tps": 103.9, + "prompt_tokens": 137, + "prompt_ms": 38, + "output": "我们处理了270万请求，99百分位的延迟为180毫秒。", + "compliant": true, + "problems": [], + "terms_total": 3, + "terms_kept": 2, + "terms_missing": [ + "p99" + ], + "chrf": 0.715 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "D_numbers_names", + "id": "D04", + "run": 0, + "input": "Microsoft and OpenAI announced the partnership in Q2 2024.", + "reference": "Microsoft和OpenAI在2024年第二季度宣布了合作。", + "ttft_ms": 40, + "e2e_ms": 200, + "gen_tokens": 17, + "tps": 105.9, + "prompt_tokens": 132, + "prompt_ms": 38, + "output": "微软和OpenAI在2024年第二季度宣布了合作。", + "compliant": true, + "problems": [], + "terms_total": 3, + "terms_kept": 3, + "terms_missing": [], + "chrf": 0.738 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "D_numbers_names", + "id": "D05", + "run": 0, + "input": "The budget is 850 thousand euros, split 60-40 between the two teams.", + "reference": "预算是85万欧元，两个团队按60比40分配。", + "ttft_ms": 40, + "e2e_ms": 234, + "gen_tokens": 20, + "tps": 102.9, + "prompt_tokens": 138, + "prompt_ms": 38, + "output": "预算为85万欧元，两队各分60%和40%。", + "compliant": true, + "problems": [], + "terms_total": 3, + "terms_kept": 3, + "terms_missing": [], + "chrf": 0.408 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "D_numbers_names", + "id": "D06", + "run": 0, + "input": "Dr. Nakamura's team at Stanford published the results on arXiv last Tuesday.", + "reference": "Nakamura博士在斯坦福的团队上周二在arXiv上发表了结果。", + "ttft_ms": 40, + "e2e_ms": 251, + "gen_tokens": 22, + "tps": 104.2, + "prompt_tokens": 134, + "prompt_ms": 39, + "output": "纳木扎教授领导的斯坦福团队在上周二将结果发布在arXiv上。", + "compliant": true, + "problems": [], + "terms_total": 2, + "terms_kept": 1, + "terms_missing": [ + "Nakamura/中村" + ], + "chrf": 0.349 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "D_numbers_names", + "id": "D07", + "run": 0, + "input": "Version 3.2.1 fixes the bug reported in ticket LCT-418.", + "reference": "3.2.1版本修复了工单LCT-418中报告的问题。", + "ttft_ms": 41, + "e2e_ms": 242, + "gen_tokens": 21, + "tps": 104.3, + "prompt_tokens": 136, + "prompt_ms": 39, + "output": "版本3.2.1修复了票号LCT-418报告的bug。", + "compliant": true, + "problems": [], + "terms_total": 2, + "terms_kept": 2, + "terms_missing": [], + "chrf": 0.521 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "D_numbers_names", + "id": "D08", + "run": 0, + "input": "Our NPS went from 31 to 47 after the redesign.", + "reference": "改版后我们的NPS从31升到了47。", + "ttft_ms": 40, + "e2e_ms": 195, + "gen_tokens": 16, + "tps": 103.5, + "prompt_tokens": 132, + "prompt_ms": 38, + "output": "我们的NPS从31变为47，经过了重新设计。", + "compliant": true, + "problems": [], + "terms_total": 3, + "terms_kept": 3, + "terms_missing": [], + "chrf": 0.498 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "D_numbers_names", + "id": "D01", + "run": 1, + "input": "Revenue grew 12% year over year to $4.3 billion.", + "reference": "收入同比增长12%，达到43亿美元。", + "ttft_ms": 39, + "e2e_ms": 140, + "gen_tokens": 11, + "tps": 108.9, + "prompt_tokens": 132, + "prompt_ms": 38, + "output": "营业收入同比增长12%至43亿元。", + "compliant": true, + "problems": [], + "terms_total": 2, + "terms_kept": 2, + "terms_missing": [], + "chrf": 0.588 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "D_numbers_names", + "id": "D02", + "run": 1, + "input": "Sarah from Stripe will join us on March 15th at 3 p.m.", + "reference": "Stripe的Sarah将在3月15日下午3点加入我们。", + "ttft_ms": 39, + "e2e_ms": 220, + "gen_tokens": 19, + "tps": 105.4, + "prompt_tokens": 134, + "prompt_ms": 38, + "output": "Sarah from Stripe will join us on March 15th at 3 p.m.", + "compliant": false, + "problems": [ + "echo", + "wrong-script" + ], + "terms_total": 3, + "terms_kept": 3, + "terms_missing": [], + "chrf": 0.301 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "D_numbers_names", + "id": "D03", + "run": 1, + "input": "We processed 2.7 million requests with a p99 latency of 180 milliseconds.", + "reference": "我们处理了270万个请求，p99延迟为180毫秒。", + "ttft_ms": 40, + "e2e_ms": 266, + "gen_tokens": 23, + "tps": 101.6, + "prompt_tokens": 137, + "prompt_ms": 38, + "output": "我们处理了270万请求，99百分位的延迟为180毫秒。", + "compliant": true, + "problems": [], + "terms_total": 3, + "terms_kept": 2, + "terms_missing": [ + "p99" + ], + "chrf": 0.715 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "D_numbers_names", + "id": "D04", + "run": 1, + "input": "Microsoft and OpenAI announced the partnership in Q2 2024.", + "reference": "Microsoft和OpenAI在2024年第二季度宣布了合作。", + "ttft_ms": 41, + "e2e_ms": 203, + "gen_tokens": 17, + "tps": 104.7, + "prompt_tokens": 132, + "prompt_ms": 39, + "output": "微软和OpenAI在2024年第二季度宣布了合作。", + "compliant": true, + "problems": [], + "terms_total": 3, + "terms_kept": 3, + "terms_missing": [], + "chrf": 0.738 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "D_numbers_names", + "id": "D05", + "run": 1, + "input": "The budget is 850 thousand euros, split 60-40 between the two teams.", + "reference": "预算是85万欧元，两个团队按60比40分配。", + "ttft_ms": 40, + "e2e_ms": 234, + "gen_tokens": 20, + "tps": 103.1, + "prompt_tokens": 138, + "prompt_ms": 39, + "output": "预算为850万欧元，两队各占60%和40%。", + "compliant": true, + "problems": [], + "terms_total": 3, + "terms_kept": 3, + "terms_missing": [], + "chrf": 0.33 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "D_numbers_names", + "id": "D06", + "run": 1, + "input": "Dr. Nakamura's team at Stanford published the results on arXiv last Tuesday.", + "reference": "Nakamura博士在斯坦福的团队上周二在arXiv上发表了结果。", + "ttft_ms": 40, + "e2e_ms": 221, + "gen_tokens": 19, + "tps": 104.7, + "prompt_tokens": 134, + "prompt_ms": 38, + "output": "斯坦福大学纳冈博士团队在上周二发布了在arXiv上的结果。", + "compliant": true, + "problems": [], + "terms_total": 2, + "terms_kept": 1, + "terms_missing": [ + "Nakamura/中村" + ], + "chrf": 0.399 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "D_numbers_names", + "id": "D07", + "run": 1, + "input": "Version 3.2.1 fixes the bug reported in ticket LCT-418.", + "reference": "3.2.1版本修复了工单LCT-418中报告的问题。", + "ttft_ms": 40, + "e2e_ms": 235, + "gen_tokens": 20, + "tps": 103.0, + "prompt_tokens": 136, + "prompt_ms": 39, + "output": "版本3.2.1修复了报告在LCT-418中的bug。", + "compliant": true, + "problems": [], + "terms_total": 2, + "terms_kept": 2, + "terms_missing": [], + "chrf": 0.542 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "D_numbers_names", + "id": "D08", + "run": 1, + "input": "Our NPS went from 31 to 47 after the redesign.", + "reference": "改版后我们的NPS从31升到了47。", + "ttft_ms": 40, + "e2e_ms": 154, + "gen_tokens": 12, + "tps": 104.6, + "prompt_tokens": 132, + "prompt_ms": 38, + "output": "我们的NPS从31提升到了47。", + "compliant": true, + "problems": [], + "terms_total": 3, + "terms_kept": 3, + "terms_missing": [], + "chrf": 0.739 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "E_tech_terms", + "id": "E01", + "run": 0, + "input": "We migrated the service from Kubernetes to a single Rust binary on bare metal.", + "reference": "我们把这个服务从Kubernetes迁移到了裸金属上的单个Rust二进制。", + "ttft_ms": 40, + "e2e_ms": 273, + "gen_tokens": 21, + "tps": 90.0, + "prompt_tokens": 131, + "prompt_ms": 38, + "output": "我们将服务从 Kubernetes 迁移到了裸金属上的一个 Rust 二进制文件。", + "compliant": true, + "problems": [], + "terms_total": 2, + "terms_kept": 2, + "terms_missing": [], + "chrf": 0.787 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "E_tech_terms", + "id": "E02", + "run": 0, + "input": "The model runs fully on-device with Metal acceleration and a 4-bit quantized Qwen checkpoint.", + "reference": "模型完全在设备端运行，使用Metal加速和4位量化的Qwen检查点。", + "ttft_ms": 42, + "e2e_ms": 243, + "gen_tokens": 21, + "tps": 104.3, + "prompt_tokens": 136, + "prompt_ms": 40, + "output": "该模型全由设备运行，使用Metal加速，并采用4位量化Qwen检查点。", + "compliant": true, + "problems": [], + "terms_total": 3, + "terms_kept": 3, + "terms_missing": [], + "chrf": 0.645 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "E_tech_terms", + "id": "E03", + "run": 0, + "input": "Enable WASAPI loopback so we capture system audio without a virtual device.", + "reference": "启用WASAPI环回，这样不需要虚拟设备也能采集系统音频。", + "ttft_ms": 41, + "e2e_ms": 265, + "gen_tokens": 23, + "tps": 102.6, + "prompt_tokens": 131, + "prompt_ms": 39, + "output": "启用 WASAPI 循环回路，以便我们可以在不使用虚拟设备的情况下捕获系统音频。", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 1, + "terms_missing": [], + "chrf": 0.464 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "E_tech_terms", + "id": "E04", + "run": 0, + "input": "The API returns a 429 when you exceed the rate limit, so add exponential backoff.", + "reference": "超过速率限制时API会返回429，所以要加指数退避。", + "ttft_ms": 40, + "e2e_ms": 229, + "gen_tokens": 20, + "tps": 105.5, + "prompt_tokens": 137, + "prompt_ms": 38, + "output": "当超出速率限制时，API 返回 429，因此添加指数退避。", + "compliant": true, + "problems": [], + "terms_total": 2, + "terms_kept": 2, + "terms_missing": [], + "chrf": 0.569 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "E_tech_terms", + "id": "E05", + "run": 0, + "input": "We're seeing GPU memory pressure because num_ctx defaults to 32K.", + "reference": "我们遇到了GPU内存压力，因为num_ctx默认是32K。", + "ttft_ms": 40, + "e2e_ms": 224, + "gen_tokens": 19, + "tps": 103.1, + "prompt_tokens": 132, + "prompt_ms": 38, + "output": "我们看到 GPU 内存压力是因为 num_ctx 默认值为 32K。", + "compliant": true, + "problems": [], + "terms_total": 3, + "terms_kept": 3, + "terms_missing": [], + "chrf": 0.676 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "E_tech_terms", + "id": "E06", + "run": 0, + "input": "The PR adds a TranslationEngine protocol with an Ollama adapter and an embedded llama.cpp runtime.", + "reference": "这个PR增加了TranslationEngine协议，带有一个Ollama适配器和一个内置的llama.cpp运行时。", + "ttft_ms": 40, + "e2e_ms": 334, + "gen_tokens": 30, + "tps": 101.7, + "prompt_tokens": 136, + "prompt_ms": 38, + "output": "PR 添加了一个 TranslationEngine 协议，使用了 Ollama 适配器，并嵌入了 llama.cpp 运行时。", + "compliant": true, + "problems": [], + "terms_total": 4, + "terms_kept": 4, + "terms_missing": [], + "chrf": 0.694 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "E_tech_terms", + "id": "E07", + "run": 0, + "input": "Notarization requires a Developer ID certificate and the hardened runtime.", + "reference": "公证需要Developer ID证书和强化运行时。", + "ttft_ms": 42, + "e2e_ms": 187, + "gen_tokens": 15, + "tps": 103.1, + "prompt_tokens": 129, + "prompt_ms": 40, + "output": "注：需要开发者 ID 证书和硬编码运行时环境。", + "compliant": false, + "problems": [ + "preamble:注：" + ], + "terms_total": 1, + "terms_kept": 0, + "terms_missing": [ + "Developer ID" + ], + "chrf": 0.267 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "E_tech_terms", + "id": "E08", + "run": 0, + "input": "Our CI runs on windows-latest, which no longer ships NSIS.", + "reference": "我们的CI跑在windows-latest上，它不再自带NSIS。", + "ttft_ms": 41, + "e2e_ms": 192, + "gen_tokens": 16, + "tps": 106.2, + "prompt_tokens": 131, + "prompt_ms": 39, + "output": "我们的 CI 运行在 windows-latest，不再提供 NSIS。", + "compliant": true, + "problems": [], + "terms_total": 3, + "terms_kept": 3, + "terms_missing": [], + "chrf": 0.683 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "E_tech_terms", + "id": "E01", + "run": 1, + "input": "We migrated the service from Kubernetes to a single Rust binary on bare metal.", + "reference": "我们把这个服务从Kubernetes迁移到了裸金属上的单个Rust二进制。", + "ttft_ms": 41, + "e2e_ms": 287, + "gen_tokens": 25, + "tps": 101.8, + "prompt_tokens": 131, + "prompt_ms": 39, + "output": "我们将服务从 Kubernetes 迁移到了单个 Rust 二进制文件，部署在裸金属上。", + "compliant": true, + "problems": [], + "terms_total": 2, + "terms_kept": 2, + "terms_missing": [], + "chrf": 0.731 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "E_tech_terms", + "id": "E02", + "run": 1, + "input": "The model runs fully on-device with Metal acceleration and a 4-bit quantized Qwen checkpoint.", + "reference": "模型完全在设备端运行，使用Metal加速和4位量化的Qwen检查点。", + "ttft_ms": 39, + "e2e_ms": 292, + "gen_tokens": 26, + "tps": 102.8, + "prompt_tokens": 136, + "prompt_ms": 38, + "output": "该模型全量运行在设备上，使用Metal加速，并且使用了一个4位量化Qwen的检查点。", + "compliant": true, + "problems": [], + "terms_total": 3, + "terms_kept": 3, + "terms_missing": [], + "chrf": 0.565 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "E_tech_terms", + "id": "E03", + "run": 1, + "input": "Enable WASAPI loopback so we capture system audio without a virtual device.", + "reference": "启用WASAPI环回，这样不需要虚拟设备也能采集系统音频。", + "ttft_ms": 39, + "e2e_ms": 248, + "gen_tokens": 21, + "tps": 100.5, + "prompt_tokens": 131, + "prompt_ms": 38, + "output": "启用 WASAPI 循环回路，以便我们可以通过系统音频捕获无虚拟设备。", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 1, + "terms_missing": [], + "chrf": 0.445 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "E_tech_terms", + "id": "E04", + "run": 1, + "input": "The API returns a 429 when you exceed the rate limit, so add exponential backoff.", + "reference": "超过速率限制时API会返回429，所以要加指数退避。", + "ttft_ms": 40, + "e2e_ms": 247, + "gen_tokens": 21, + "tps": 101.2, + "prompt_tokens": 137, + "prompt_ms": 38, + "output": "API 返回 429 时，表示已超出速率限制，请添加指数退避。", + "compliant": true, + "problems": [], + "terms_total": 2, + "terms_kept": 2, + "terms_missing": [], + "chrf": 0.503 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "E_tech_terms", + "id": "E05", + "run": 1, + "input": "We're seeing GPU memory pressure because num_ctx defaults to 32K.", + "reference": "我们遇到了GPU内存压力，因为num_ctx默认是32K。", + "ttft_ms": 40, + "e2e_ms": 223, + "gen_tokens": 19, + "tps": 103.8, + "prompt_tokens": 132, + "prompt_ms": 39, + "output": "我们看到 GPU 内存压力是因为 num_ctx 默认值为 32K。", + "compliant": true, + "problems": [], + "terms_total": 3, + "terms_kept": 3, + "terms_missing": [], + "chrf": 0.676 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "E_tech_terms", + "id": "E06", + "run": 1, + "input": "The PR adds a TranslationEngine protocol with an Ollama adapter and an embedded llama.cpp runtime.", + "reference": "这个PR增加了TranslationEngine协议，带有一个Ollama适配器和一个内置的llama.cpp运行时。", + "ttft_ms": 40, + "e2e_ms": 328, + "gen_tokens": 29, + "tps": 100.6, + "prompt_tokens": 136, + "prompt_ms": 38, + "output": "PR 添加了 TranslationEngine 协议，使用 Ollama 适配器和嵌入的 llama.cpp 运行时。", + "compliant": true, + "problems": [], + "terms_total": 4, + "terms_kept": 4, + "terms_missing": [], + "chrf": 0.749 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "E_tech_terms", + "id": "E07", + "run": 1, + "input": "Notarization requires a Developer ID certificate and the hardened runtime.", + "reference": "公证需要Developer ID证书和强化运行时。", + "ttft_ms": 42, + "e2e_ms": 177, + "gen_tokens": 14, + "tps": 104.1, + "prompt_tokens": 129, + "prompt_ms": 41, + "output": "Notarization requires a Developer ID certificate and the hardened runtime.", + "compliant": false, + "problems": [ + "echo", + "wrong-script" + ], + "terms_total": 1, + "terms_kept": 1, + "terms_missing": [], + "chrf": 0.309 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "E_tech_terms", + "id": "E08", + "run": 1, + "input": "Our CI runs on windows-latest, which no longer ships NSIS.", + "reference": "我们的CI跑在windows-latest上，它不再自带NSIS。", + "ttft_ms": 40, + "e2e_ms": 174, + "gen_tokens": 14, + "tps": 104.2, + "prompt_tokens": 131, + "prompt_ms": 38, + "output": "我们的 CI 使用的是 windows-latest，不再提供 NSIS。", + "compliant": true, + "problems": [], + "terms_total": 3, + "terms_kept": 3, + "terms_missing": [], + "chrf": 0.647 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "F_long_sentences", + "id": "F01", + "run": 0, + "input": "What I want to emphasize today is that the transition to on-device inference is not primarily about cost, it's about latency and privacy, and those two things are what actually change user behavior.", + "reference": "我今天想强调的是，向设备端推理的转变主要不是为了成本，而是为了延迟和隐私，而这两点才是真正改变用户行为的因素。", + "ttft_ms": 56, + "e2e_ms": 344, + "gen_tokens": 29, + "tps": 100.7, + "prompt_tokens": 154, + "prompt_ms": 55, + "output": "今天我想强调的是，从设备端推理的过渡不是主要关于成本，而是关于延迟和隐私，这实际上是改变了用户行为。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.448 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "F_long_sentences", + "id": "F02", + "run": 0, + "input": "Before we commit to the Q3 roadmap, I'd like each team lead to send me a one-page summary of their top three risks and what support they need from leadership to mitigate them.", + "reference": "在我们确定第三季度路线图之前，我希望每位团队负责人给我发一页纸的摘要，说明他们最主要的三个风险，以及需要管理层提供什么支持来缓解这些风险。", + "ttft_ms": 58, + "e2e_ms": 439, + "gen_tokens": 38, + "tps": 100.0, + "prompt_tokens": 154, + "prompt_ms": 57, + "output": "在我们决定Q3路线图之前，我希望每位团队负责人能给我一份包含他们认为最重要的三个风险及其需要领导层支持来减轻这些风险的一页简要总结。", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 1, + "terms_missing": [], + "chrf": 0.435 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "F_long_sentences", + "id": "F03", + "run": 0, + "input": "If the benchmark shows that the embedded runtime is within twenty percent of the Ollama path on first-token latency, I'm comfortable making it the default and keeping Ollama as an advanced option.", + "reference": "如果基准测试显示内置运行时的首字延迟与Ollama路径相差在百分之二十以内，我可以接受把它设为默认，并把Ollama保留为高级选项。", + "ttft_ms": 57, + "e2e_ms": 474, + "gen_tokens": 42, + "tps": 100.9, + "prompt_tokens": 156, + "prompt_ms": 56, + "output": "如果基准显示嵌入式运行时在首次令牌延迟上与Ollama路径的误差不超过20%，我有信心将其设置为默认选项，同时保留Ollama作为高级选项。", + "compliant": true, + "problems": [], + "terms_total": 2, + "terms_kept": 2, + "terms_missing": [], + "chrf": 0.407 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "F_long_sentences", + "id": "F04", + "run": 0, + "input": "The reason the microphone channel picks up the speaker audio is simply acoustic feedback, so with headphones the duplicate lines disappear, but we should still say that in onboarding.", + "reference": "麦克风通道会拾取扬声器声音的原因就是声学回授，所以戴上耳机重复的字幕就会消失，但我们仍然应该在引导里说明这一点。", + "ttft_ms": 58, + "e2e_ms": 419, + "gen_tokens": 37, + "tps": 102.4, + "prompt_tokens": 149, + "prompt_ms": 56, + "output": "麦克风通道拾取到发言人音频的原因是简单的声学反馈，因此在使用耳机时，重复的线路会消失，但在入职时我们仍然应该提及这一点。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.363 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "F_long_sentences", + "id": "F05", + "run": 0, + "input": "Historically every attempt to bundle a second inference server inside a desktop app has failed on maintenance cost, so the question is whether a statically linked library changes that calculus.", + "reference": "历史上每一次在桌面应用里捆绑第二个推理服务的尝试都败在了维护成本上，所以问题在于静态链接的库是否改变了这个算计。", + "ttft_ms": 56, + "e2e_ms": 400, + "gen_tokens": 35, + "tps": 101.7, + "prompt_tokens": 149, + "prompt_ms": 54, + "output": "历史上，将第二个推理服务器捆绑到桌面应用程序中的任何尝试都因维护成本而失败，因此问题在于是否静态链接的库会改变这个计算。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.452 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "F_long_sentences", + "id": "F06", + "run": 0, + "input": "Let's park the pricing discussion, because until we know whether enterprise customers actually need SSO and audit logs, any number we put on the slide is going to be a guess.", + "reference": "我们先把定价讨论放一放，因为在知道企业客户是否真的需要SSO和审计日志之前，我们放在幻灯片上的任何数字都只是猜测。", + "ttft_ms": 56, + "e2e_ms": 386, + "gen_tokens": 33, + "tps": 100.1, + "prompt_tokens": 152, + "prompt_ms": 55, + "output": "让我们暂停价格讨论，因为我们直到知道企业客户是否真的需要 SSO 和审计日志，任何我们在幻灯片上列出的数字都是猜测。", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 1, + "terms_missing": [], + "chrf": 0.562 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "F_long_sentences", + "id": "F01", + "run": 1, + "input": "What I want to emphasize today is that the transition to on-device inference is not primarily about cost, it's about latency and privacy, and those two things are what actually change user behavior.", + "reference": "我今天想强调的是，向设备端推理的转变主要不是为了成本，而是为了延迟和隐私，而这两点才是真正改变用户行为的因素。", + "ttft_ms": 57, + "e2e_ms": 359, + "gen_tokens": 31, + "tps": 102.6, + "prompt_tokens": 154, + "prompt_ms": 55, + "output": "今天我想强调的是，从设备端推理的过渡不仅仅是关于成本，而是关于延迟和隐私，而这两个因素实际上改变了用户的使用行为。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.465 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "F_long_sentences", + "id": "F02", + "run": 1, + "input": "Before we commit to the Q3 roadmap, I'd like each team lead to send me a one-page summary of their top three risks and what support they need from leadership to mitigate them.", + "reference": "在我们确定第三季度路线图之前，我希望每位团队负责人给我发一页纸的摘要，说明他们最主要的三个风险，以及需要管理层提供什么支持来缓解这些风险。", + "ttft_ms": 59, + "e2e_ms": 451, + "gen_tokens": 39, + "tps": 99.4, + "prompt_tokens": 154, + "prompt_ms": 57, + "output": "在我们确定Q3路线图之前，我希望每个团队领导发送我一份包含他们认为的前三大风险及其从管理层获得支持以减轻这些风险的说明的一页总结。", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 1, + "terms_missing": [], + "chrf": 0.338 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "F_long_sentences", + "id": "F03", + "run": 1, + "input": "If the benchmark shows that the embedded runtime is within twenty percent of the Ollama path on first-token latency, I'm comfortable making it the default and keeping Ollama as an advanced option.", + "reference": "如果基准测试显示内置运行时的首字延迟与Ollama路径相差在百分之二十以内，我可以接受把它设为默认，并把Ollama保留为高级选项。", + "ttft_ms": 57, + "e2e_ms": 486, + "gen_tokens": 43, + "tps": 100.3, + "prompt_tokens": 156, + "prompt_ms": 55, + "output": "如果基准显示嵌入式运行时在首次令牌延迟上与Ollama路径相差不超过20%，我就可以放心地将其设为默认选项，同时保留Ollama作为高级选项。", + "compliant": true, + "problems": [], + "terms_total": 2, + "terms_kept": 2, + "terms_missing": [], + "chrf": 0.447 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "F_long_sentences", + "id": "F04", + "run": 1, + "input": "The reason the microphone channel picks up the speaker audio is simply acoustic feedback, so with headphones the duplicate lines disappear, but we should still say that in onboarding.", + "reference": "麦克风通道会拾取扬声器声音的原因就是声学回授，所以戴上耳机重复的字幕就会消失，但我们仍然应该在引导里说明这一点。", + "ttft_ms": 58, + "e2e_ms": 423, + "gen_tokens": 37, + "tps": 101.5, + "prompt_tokens": 149, + "prompt_ms": 57, + "output": "麦克风通道拾取到说话人音频的原因是简单的声学反馈，因此当使用耳机时，重复的线路会消失，但我们仍应在入职时强调这一点。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.347 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "F_long_sentences", + "id": "F05", + "run": 1, + "input": "Historically every attempt to bundle a second inference server inside a desktop app has failed on maintenance cost, so the question is whether a statically linked library changes that calculus.", + "reference": "历史上每一次在桌面应用里捆绑第二个推理服务的尝试都败在了维护成本上，所以问题在于静态链接的库是否改变了这个算计。", + "ttft_ms": 56, + "e2e_ms": 382, + "gen_tokens": 33, + "tps": 101.2, + "prompt_tokens": 149, + "prompt_ms": 54, + "output": "历史上，将第二个推理服务器捆绑到桌面应用程序中一直因维护成本而失败，所以问题在于是否静态链接的库会改变这个计算。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.482 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "F_long_sentences", + "id": "F06", + "run": 1, + "input": "Let's park the pricing discussion, because until we know whether enterprise customers actually need SSO and audit logs, any number we put on the slide is going to be a guess.", + "reference": "我们先把定价讨论放一放，因为在知道企业客户是否真的需要SSO和审计日志之前，我们放在幻灯片上的任何数字都只是猜测。", + "ttft_ms": 56, + "e2e_ms": 394, + "gen_tokens": 34, + "tps": 100.5, + "prompt_tokens": 152, + "prompt_ms": 55, + "output": "让我们暂停定价讨论，因为直到我们确定企业客户是否真的需要 SSO 和审计日志，我们在幻灯片上所使用的任何数字都是猜测。", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 1, + "terms_missing": [], + "chrf": 0.596 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "G_zh_to_en", + "id": "G01", + "run": 0, + "input": "这个功能我们下周三之前能上线吗", + "reference": "Can we ship this feature before next Wednesday?", + "ttft_ms": 100, + "e2e_ms": 222, + "gen_tokens": 13, + "tps": 106.1, + "prompt_tokens": 122, + "prompt_ms": 97, + "output": "Can this feature be available before the end of next week?", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 0, + "terms_missing": [ + "Wednesday" + ], + "chrf": 0.588 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "G_zh_to_en", + "id": "G02", + "run": 0, + "input": "那个就是说我们现在有两个方案一个是自己做一个是买第三方的", + "reference": "So we have two options now: build it ourselves, or buy a third-party solution.", + "ttft_ms": 43, + "e2e_ms": 230, + "gen_tokens": 19, + "tps": 101.9, + "prompt_tokens": 126, + "prompt_ms": 42, + "output": "That means we have two options: either we do it ourselves or buy a third party.", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 1, + "terms_missing": [], + "chrf": 0.622 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "G_zh_to_en", + "id": "G03", + "run": 0, + "input": "上个季度我们的留存率从百分之三十八涨到了百分之四十五", + "reference": "Last quarter our retention rate went from 38% to 45%.", + "ttft_ms": 39, + "e2e_ms": 235, + "gen_tokens": 20, + "tps": 102.0, + "prompt_tokens": 128, + "prompt_ms": 38, + "output": "Last quarter, our retention rate increased from 3.8% to 4.5%.", + "compliant": true, + "problems": [], + "terms_total": 2, + "terms_kept": 0, + "terms_missing": [ + "38", + "45" + ], + "chrf": 0.745 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "G_zh_to_en", + "id": "G04", + "run": 0, + "input": "我觉得这个风险有点大，要不我们先小范围灰度一下", + "reference": "I think the risk is a bit high. How about a small-scale gradual rollout first?", + "ttft_ms": 40, + "e2e_ms": 224, + "gen_tokens": 19, + "tps": 103.1, + "prompt_tokens": 128, + "prompt_ms": 38, + "output": "I think this risk is a bit high. Let's do a small range test first.", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.533 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "G_zh_to_en", + "id": "G05", + "run": 0, + "input": "张总说预算最多给到五十万，超了要重新审批", + "reference": "Mr. Zhang said the budget is capped at 500 thousand; anything above that needs re-approval.", + "ttft_ms": 41, + "e2e_ms": 277, + "gen_tokens": 24, + "tps": 101.6, + "prompt_tokens": 128, + "prompt_ms": 39, + "output": "The CEO said the budget can be given up to fifty thousand, and exceeding it will require a re-approval.", + "compliant": true, + "problems": [], + "terms_total": 2, + "terms_kept": 0, + "terms_missing": [ + "Zhang", + "500/500,000/half a million" + ], + "chrf": 0.503 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "G_zh_to_en", + "id": "G06", + "run": 0, + "input": "识别延迟大概一点五秒，翻译再加一秒左右，用户基本感觉不到", + "reference": "Recognition latency is about 1.5 seconds, translation adds roughly another second, so users barely notice.", + "ttft_ms": 41, + "e2e_ms": 336, + "gen_tokens": 30, + "tps": 101.8, + "prompt_tokens": 130, + "prompt_ms": 40, + "output": "The recognition delay is about 1.5 seconds, and the translation adds another second or so, making the user feel it's not noticeable.", + "compliant": false, + "problems": [ + "preamble:The translation" + ], + "terms_total": 1, + "terms_kept": 1, + "terms_missing": [], + "chrf": 0.688 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "G_zh_to_en", + "id": "G07", + "run": 0, + "input": "等一下我这边网不太好你们先聊", + "reference": "Hold on, my connection isn't great. You go ahead.", + "ttft_ms": 27, + "e2e_ms": 228, + "gen_tokens": 21, + "tps": 104.7, + "prompt_tokens": 122, + "prompt_ms": 26, + "output": "Wait a moment; my internet connection is poor here. You can continue chatting while I fix it.", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.413 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "G_zh_to_en", + "id": "G08", + "run": 0, + "input": "这个PR先不要合，Codex还有两条评论没处理", + "reference": "Don't merge this PR yet; there are still two Codex comments to address.", + "ttft_ms": 42, + "e2e_ms": 214, + "gen_tokens": 18, + "tps": 104.3, + "prompt_tokens": 126, + "prompt_ms": 40, + "output": "This PR should not be merged yet, Codex has two more comments to process.", + "compliant": true, + "problems": [], + "terms_total": 3, + "terms_kept": 3, + "terms_missing": [], + "chrf": 0.507 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "G_zh_to_en", + "id": "G01", + "run": 1, + "input": "这个功能我们下周三之前能上线吗", + "reference": "Can we ship this feature before next Wednesday?", + "ttft_ms": 27, + "e2e_ms": 151, + "gen_tokens": 13, + "tps": 104.8, + "prompt_tokens": 122, + "prompt_ms": 26, + "output": "This feature will be available before the end of next Wednesday.", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 1, + "terms_missing": [], + "chrf": 0.675 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "G_zh_to_en", + "id": "G02", + "run": 1, + "input": "那个就是说我们现在有两个方案一个是自己做一个是买第三方的", + "reference": "So we have two options now: build it ourselves, or buy a third-party solution.", + "ttft_ms": 41, + "e2e_ms": 233, + "gen_tokens": 20, + "tps": 104.6, + "prompt_tokens": 126, + "prompt_ms": 40, + "output": "That means we have two options: either we do it ourselves or buy a third-party solution.", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 1, + "terms_missing": [], + "chrf": 0.774 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "G_zh_to_en", + "id": "G03", + "run": 1, + "input": "上个季度我们的留存率从百分之三十八涨到了百分之四十五", + "reference": "Last quarter our retention rate went from 38% to 45%.", + "ttft_ms": 39, + "e2e_ms": 233, + "gen_tokens": 20, + "tps": 103.2, + "prompt_tokens": 128, + "prompt_ms": 38, + "output": "Our retention rate increased from 3.8% to 4.5% last quarter.", + "compliant": true, + "problems": [], + "terms_total": 2, + "terms_kept": 0, + "terms_missing": [ + "38", + "45" + ], + "chrf": 0.737 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "G_zh_to_en", + "id": "G04", + "run": 1, + "input": "我觉得这个风险有点大，要不我们先小范围灰度一下", + "reference": "I think the risk is a bit high. How about a small-scale gradual rollout first?", + "ttft_ms": 40, + "e2e_ms": 247, + "gen_tokens": 20, + "tps": 96.6, + "prompt_tokens": 128, + "prompt_ms": 38, + "output": "I think this risk is a bit high, let's do a small-scale gray area first.", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.622 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "G_zh_to_en", + "id": "G05", + "run": 1, + "input": "张总说预算最多给到五十万，超了要重新审批", + "reference": "Mr. Zhang said the budget is capped at 500 thousand; anything above that needs re-approval.", + "ttft_ms": 39, + "e2e_ms": 311, + "gen_tokens": 28, + "tps": 102.9, + "prompt_tokens": 128, + "prompt_ms": 38, + "output": "Mr. Zhang said the budget can be approved at most up to fifty thousand, and any overage will require a re-approval.", + "compliant": true, + "problems": [], + "terms_total": 2, + "terms_kept": 1, + "terms_missing": [ + "500/500,000/half a million" + ], + "chrf": 0.608 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "G_zh_to_en", + "id": "G06", + "run": 1, + "input": "识别延迟大概一点五秒，翻译再加一秒左右，用户基本感觉不到", + "reference": "Recognition latency is about 1.5 seconds, translation adds roughly another second, so users barely notice.", + "ttft_ms": 39, + "e2e_ms": 335, + "gen_tokens": 30, + "tps": 101.3, + "prompt_tokens": 130, + "prompt_ms": 37, + "output": "The recognition delay is approximately 1.5 seconds, and the translation should add another second or so. The user will generally not feel it.", + "compliant": false, + "problems": [ + "preamble:The translation" + ], + "terms_total": 1, + "terms_kept": 1, + "terms_missing": [], + "chrf": 0.611 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "G_zh_to_en", + "id": "G07", + "run": 1, + "input": "等一下我这边网不太好你们先聊", + "reference": "Hold on, my connection isn't great. You go ahead.", + "ttft_ms": 27, + "e2e_ms": 204, + "gen_tokens": 18, + "tps": 101.9, + "prompt_tokens": 122, + "prompt_ms": 26, + "output": "Wait a moment, my internet connection is not good here. You can talk first.", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.475 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "G_zh_to_en", + "id": "G08", + "run": 1, + "input": "这个PR先不要合，Codex还有两条评论没处理", + "reference": "Don't merge this PR yet; there are still two Codex comments to address.", + "ttft_ms": 42, + "e2e_ms": 215, + "gen_tokens": 18, + "tps": 104.2, + "prompt_tokens": 126, + "prompt_ms": 40, + "output": "This PR should not be merged yet, Codex has two more comments to process.", + "compliant": true, + "problems": [], + "terms_total": 3, + "terms_kept": 3, + "terms_missing": [], + "chrf": 0.507 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "H_context_followup", + "id": "H01", + "run": 0, + "input": "We benchmarked the MLX build last week.", + "reference": "我们上周对MLX版本做了基准测试。", + "ttft_ms": 31, + "e2e_ms": 151, + "gen_tokens": 13, + "tps": 107.7, + "prompt_tokens": 126, + "prompt_ms": 27, + "output": "我们上周对 MLX 构建进行了基准测试。", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 1, + "terms_missing": [], + "chrf": 0.694 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "H_context_followup", + "id": "H02", + "run": 0, + "input": "It was about twice as fast on first token.", + "reference": "它的首字速度大约快了一倍。", + "ttft_ms": 40, + "e2e_ms": 133, + "gen_tokens": 10, + "tps": 106.6, + "prompt_tokens": 160, + "prompt_ms": 38, + "output": "第一次 token 上它快了两倍。", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 1, + "terms_missing": [], + "chrf": 0.152 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "H_context_followup", + "id": "H03", + "run": 0, + "input": "But it only works on Apple Silicon.", + "reference": "但它只能在Apple Silicon上运行。", + "ttft_ms": 40, + "e2e_ms": 104, + "gen_tokens": 7, + "tps": 110.3, + "prompt_tokens": 189, + "prompt_ms": 38, + "output": "但只适用于 Apple Silicon。", + "compliant": false, + "problems": [ + "wrong-script" + ], + "terms_total": 1, + "terms_kept": 1, + "terms_missing": [], + "chrf": 0.591 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "H_context_followup", + "id": "H04", + "run": 0, + "input": "So for Intel Macs we'd still need the other one.", + "reference": "所以对于Intel的Mac我们仍然需要另一个。", + "ttft_ms": 81, + "e2e_ms": 204, + "gen_tokens": 13, + "tps": 105.6, + "prompt_tokens": 186, + "prompt_ms": 79, + "output": "所以对于 Intel Macs 我们仍然需要另一个。", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 1, + "terms_missing": [], + "chrf": 0.808 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "H_context_followup", + "id": "H05", + "run": 0, + "input": "She asked whether that doubles the maintenance work.", + "reference": "她问这是否会让维护工作翻倍。", + "ttft_ms": 81, + "e2e_ms": 171, + "gen_tokens": 10, + "tps": 110.4, + "prompt_tokens": 188, + "prompt_ms": 79, + "output": "她问那是否增加了维护工作量。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.321 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "H_context_followup", + "id": "H06", + "run": 0, + "input": "Honestly, it probably does.", + "reference": "说实话，可能会。", + "ttft_ms": 81, + "e2e_ms": 156, + "gen_tokens": 8, + "tps": 106.2, + "prompt_tokens": 189, + "prompt_ms": 79, + "output": "说实话，那大概会增加。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.394 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "H_context_followup", + "id": "H01", + "run": 1, + "input": "We benchmarked the MLX build last week.", + "reference": "我们上周对MLX版本做了基准测试。", + "ttft_ms": 29, + "e2e_ms": 152, + "gen_tokens": 13, + "tps": 105.2, + "prompt_tokens": 126, + "prompt_ms": 27, + "output": "我们上周对 MLX 构建进行了基准测试。", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 1, + "terms_missing": [], + "chrf": 0.694 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "H_context_followup", + "id": "H02", + "run": 1, + "input": "It was about twice as fast on first token.", + "reference": "它的首字速度大约快了一倍。", + "ttft_ms": 40, + "e2e_ms": 134, + "gen_tokens": 10, + "tps": 106.7, + "prompt_tokens": 160, + "prompt_ms": 38, + "output": "第一次 token 上它快了两倍。", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 1, + "terms_missing": [], + "chrf": 0.152 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "H_context_followup", + "id": "H03", + "run": 1, + "input": "But it only works on Apple Silicon.", + "reference": "但它只能在Apple Silicon上运行。", + "ttft_ms": 42, + "e2e_ms": 126, + "gen_tokens": 9, + "tps": 106.8, + "prompt_tokens": 189, + "prompt_ms": 40, + "output": "但它只在 Apple Silicon 上有效。", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 1, + "terms_missing": [], + "chrf": 0.733 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "H_context_followup", + "id": "H04", + "run": 1, + "input": "So for Intel Macs we'd still need the other one.", + "reference": "所以对于Intel的Mac我们仍然需要另一个。", + "ttft_ms": 81, + "e2e_ms": 202, + "gen_tokens": 13, + "tps": 107.0, + "prompt_tokens": 188, + "prompt_ms": 78, + "output": "所以对于 Intel Macs 我们仍然需要另一个。", + "compliant": true, + "problems": [], + "terms_total": 1, + "terms_kept": 1, + "terms_missing": [], + "chrf": 0.808 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "H_context_followup", + "id": "H05", + "run": 1, + "input": "She asked whether that doubles the maintenance work.", + "reference": "她问这是否会让维护工作翻倍。", + "ttft_ms": 84, + "e2e_ms": 175, + "gen_tokens": 10, + "tps": 108.8, + "prompt_tokens": 190, + "prompt_ms": 82, + "output": "她问那是否增加了维护工作量。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.321 + }, + { + "engine": "llamacpp-qwen2.5-1.5b(PR27)", + "category": "H_context_followup", + "id": "H06", + "run": 1, + "input": "Honestly, it probably does.", + "reference": "说实话，可能会。", + "ttft_ms": 81, + "e2e_ms": 155, + "gen_tokens": 8, + "tps": 107.9, + "prompt_tokens": 189, + "prompt_ms": 78, + "output": "老实说，可能确实如此。", + "compliant": true, + "problems": [], + "terms_total": 0, + "terms_kept": 0, + "terms_missing": [], + "chrf": 0.278 + } + ] +} \ No newline at end of file diff --git a/benchmark/realtime_eval_set.json b/benchmark/realtime_eval_set.json new file mode 100644 index 0000000..7d5dd27 --- /dev/null +++ b/benchmark/realtime_eval_set.json @@ -0,0 +1,548 @@ +{ + "metadata": { + "version": "1.0", + "description": "LCT realtime caption-translation eval set: inputs mimic what on-device ASR (SpeechAnalyzer / Live Captions) actually emits: short turns, unfinished drafts, no punctuation, fillers, numbers and names.", + "created": "2026-11-07", + "scoring": "latency (TTFT/E2E/TPS, client-side, streaming), instruction compliance (no preamble / think / markers / echo), term preservation (numbers, names, tech terms), hallucination guard (length ratio), chrF vs reference" + }, + "system_prompt": "You are a professional simultaneous interpreter. Translate the text enclosed in 🔤 markers to {TARGET_LANGUAGE}.\n\nCRITICAL RULES:\n1. Output ONLY the translated text, never the original text\n2. Handle incomplete sentences naturally and professionally\n3. Preserve technical terms, company names, and proper nouns\n4. Keep numbers, dates and units exactly as spoken\n5. Never add explanations, notes, or commentary\n6. If the text is a fragment, translate the fragment as-is without completing it", + "categories": { + "A_short_turns": { + "description": "短对话轮次，会议里最常见", + "direction": "en->zh", + "cases": [ + { + "id": "A01", + "input": "Let's get started.", + "reference": "我们开始吧。" + }, + { + "id": "A02", + "input": "Can everyone see my screen?", + "reference": "大家能看到我的屏幕吗？" + }, + { + "id": "A03", + "input": "Sorry, you cut out for a second.", + "reference": "抱歉，你刚才断了一下。" + }, + { + "id": "A04", + "input": "I'll take that offline.", + "reference": "这个我们线下再聊。" + }, + { + "id": "A05", + "input": "Any questions before we move on?", + "reference": "在继续之前有什么问题吗？" + }, + { + "id": "A06", + "input": "That makes sense, thanks for clarifying.", + "reference": "明白了，谢谢你的说明。" + }, + { + "id": "A07", + "input": "We're running about five minutes behind.", + "reference": "我们大概晚了五分钟。", + "terms_any": [ + [ + "5", + "五" + ] + ] + }, + { + "id": "A08", + "input": "Let me share the link in the chat.", + "reference": "我把链接发到聊天里。" + } + ] + }, + "B_asr_drafts": { + "description": "ASR 草稿：句子说到一半就被切出来，模型不能脑补结尾", + "direction": "en->zh", + "cases": [ + { + "id": "B01", + "input": "so what we found was that the", + "reference": "所以我们发现的是", + "max_ratio": 2.0 + }, + { + "id": "B02", + "input": "if you look at the second chart you can see", + "reference": "如果你看第二张图，你可以看到", + "max_ratio": 2.0 + }, + { + "id": "B03", + "input": "the main reason we chose this approach is", + "reference": "我们选择这个方案的主要原因是", + "max_ratio": 2.0 + }, + { + "id": "B04", + "input": "and then the customer said that they", + "reference": "然后客户说他们", + "max_ratio": 2.0 + }, + { + "id": "B05", + "input": "I think the biggest risk here", + "reference": "我觉得这里最大的风险", + "max_ratio": 2.0 + }, + { + "id": "B06", + "input": "we need to decide by", + "reference": "我们需要在……之前决定", + "max_ratio": 2.0 + }, + { + "id": "B07", + "input": "which brings me to", + "reference": "这就引出了", + "max_ratio": 2.0 + }, + { + "id": "B08", + "input": "the data shows a clear", + "reference": "数据显示出明显的", + "max_ratio": 2.0 + } + ] + }, + "C_asr_noisy": { + "description": "无标点、全小写、口头禅、重复词，模拟真实识别输出", + "direction": "en->zh", + "cases": [ + { + "id": "C01", + "input": "um so yeah basically what we're gonna do is we're gonna ship it next week", + "reference": "所以基本上我们要做的就是下周发布。" + }, + { + "id": "C02", + "input": "i mean it's it's not perfect but it works you know", + "reference": "我是说它不完美，但能用，你懂的。" + }, + { + "id": "C03", + "input": "okay okay let me let me just pull that up real quick", + "reference": "好好，让我快速调出来看一下。" + }, + { + "id": "C04", + "input": "right so the the thing is we don't actually have the budget for that", + "reference": "对，问题是我们其实没有这方面的预算。" + }, + { + "id": "C05", + "input": "like honestly I don't think anyone has looked at this since like march", + "reference": "说实话我觉得从三月以来就没人看过这个了。", + "terms_any": [ + [ + "三月", + "3月" + ] + ] + }, + { + "id": "C06", + "input": "and uh we also need to loop in the legal team before we sign anything", + "reference": "另外在签任何东西之前我们还需要让法务团队参与进来。" + }, + { + "id": "C07", + "input": "yeah no that's fair that's totally fair", + "reference": "对，没错，这很合理。" + }, + { + "id": "C08", + "input": "so hold on are we saying the deadline moved or not", + "reference": "等等，我们是说截止日期改了还是没改？" + } + ] + }, + "D_numbers_names": { + "description": "数字、日期、货币、人名公司名必须原样保留", + "direction": "en->zh", + "cases": [ + { + "id": "D01", + "input": "Revenue grew 12% year over year to $4.3 billion.", + "reference": "收入同比增长12%，达到43亿美元。", + "terms": [ + "12" + ], + "terms_any": [ + [ + "4.3", + "43" + ] + ] + }, + { + "id": "D02", + "input": "Sarah from Stripe will join us on March 15th at 3 p.m.", + "reference": "Stripe的Sarah将在3月15日下午3点加入我们。", + "terms": [ + "Sarah", + "Stripe", + "15" + ] + }, + { + "id": "D03", + "input": "We processed 2.7 million requests with a p99 latency of 180 milliseconds.", + "reference": "我们处理了270万个请求，p99延迟为180毫秒。", + "terms": [ + "180", + "p99" + ], + "terms_any": [ + [ + "2.7", + "270" + ] + ] + }, + { + "id": "D04", + "input": "Microsoft and OpenAI announced the partnership in Q2 2024.", + "reference": "Microsoft和OpenAI在2024年第二季度宣布了合作。", + "terms": [ + "OpenAI", + "2024" + ], + "terms_any": [ + [ + "Microsoft", + "微软" + ] + ] + }, + { + "id": "D05", + "input": "The budget is 850 thousand euros, split 60-40 between the two teams.", + "reference": "预算是85万欧元，两个团队按60比40分配。", + "terms": [ + "60", + "40" + ], + "terms_any": [ + [ + "850", + "85" + ] + ] + }, + { + "id": "D06", + "input": "Dr. Nakamura's team at Stanford published the results on arXiv last Tuesday.", + "reference": "Nakamura博士在斯坦福的团队上周二在arXiv上发表了结果。", + "terms": [ + "arXiv" + ], + "terms_any": [ + [ + "Nakamura", + "中村" + ] + ] + }, + { + "id": "D07", + "input": "Version 3.2.1 fixes the bug reported in ticket LCT-418.", + "reference": "3.2.1版本修复了工单LCT-418中报告的问题。", + "terms": [ + "3.2.1", + "LCT-418" + ] + }, + { + "id": "D08", + "input": "Our NPS went from 31 to 47 after the redesign.", + "reference": "改版后我们的NPS从31升到了47。", + "terms": [ + "NPS", + "31", + "47" + ] + } + ] + }, + "E_tech_terms": { + "description": "技术/产品术语，保留英文或用标准译法", + "direction": "en->zh", + "cases": [ + { + "id": "E01", + "input": "We migrated the service from Kubernetes to a single Rust binary on bare metal.", + "reference": "我们把这个服务从Kubernetes迁移到了裸金属上的单个Rust二进制。", + "terms": [ + "Kubernetes", + "Rust" + ] + }, + { + "id": "E02", + "input": "The model runs fully on-device with Metal acceleration and a 4-bit quantized Qwen checkpoint.", + "reference": "模型完全在设备端运行，使用Metal加速和4位量化的Qwen检查点。", + "terms": [ + "Metal", + "Qwen", + "4" + ] + }, + { + "id": "E03", + "input": "Enable WASAPI loopback so we capture system audio without a virtual device.", + "reference": "启用WASAPI环回，这样不需要虚拟设备也能采集系统音频。", + "terms": [ + "WASAPI" + ] + }, + { + "id": "E04", + "input": "The API returns a 429 when you exceed the rate limit, so add exponential backoff.", + "reference": "超过速率限制时API会返回429，所以要加指数退避。", + "terms": [ + "API", + "429" + ] + }, + { + "id": "E05", + "input": "We're seeing GPU memory pressure because num_ctx defaults to 32K.", + "reference": "我们遇到了GPU内存压力，因为num_ctx默认是32K。", + "terms": [ + "GPU", + "num_ctx", + "32K" + ] + }, + { + "id": "E06", + "input": "The PR adds a TranslationEngine protocol with an Ollama adapter and an embedded llama.cpp runtime.", + "reference": "这个PR增加了TranslationEngine协议，带有一个Ollama适配器和一个内置的llama.cpp运行时。", + "terms": [ + "TranslationEngine", + "Ollama", + "llama.cpp", + "PR" + ] + }, + { + "id": "E07", + "input": "Notarization requires a Developer ID certificate and the hardened runtime.", + "reference": "公证需要Developer ID证书和强化运行时。", + "terms": [ + "Developer ID" + ] + }, + { + "id": "E08", + "input": "Our CI runs on windows-latest, which no longer ships NSIS.", + "reference": "我们的CI跑在windows-latest上，它不再自带NSIS。", + "terms": [ + "CI", + "windows-latest", + "NSIS" + ] + } + ] + }, + "F_long_sentences": { + "description": "讲座/会议里的长句，30-45 词", + "direction": "en->zh", + "cases": [ + { + "id": "F01", + "input": "What I want to emphasize today is that the transition to on-device inference is not primarily about cost, it's about latency and privacy, and those two things are what actually change user behavior.", + "reference": "我今天想强调的是，向设备端推理的转变主要不是为了成本，而是为了延迟和隐私，而这两点才是真正改变用户行为的因素。" + }, + { + "id": "F02", + "input": "Before we commit to the Q3 roadmap, I'd like each team lead to send me a one-page summary of their top three risks and what support they need from leadership to mitigate them.", + "reference": "在我们确定第三季度路线图之前，我希望每位团队负责人给我发一页纸的摘要，说明他们最主要的三个风险，以及需要管理层提供什么支持来缓解这些风险。", + "terms_any": [ + [ + "3", + "三" + ] + ] + }, + { + "id": "F03", + "input": "If the benchmark shows that the embedded runtime is within twenty percent of the Ollama path on first-token latency, I'm comfortable making it the default and keeping Ollama as an advanced option.", + "reference": "如果基准测试显示内置运行时的首字延迟与Ollama路径相差在百分之二十以内，我可以接受把它设为默认，并把Ollama保留为高级选项。", + "terms": [ + "Ollama" + ], + "terms_any": [ + [ + "20", + "二十" + ] + ] + }, + { + "id": "F04", + "input": "The reason the microphone channel picks up the speaker audio is simply acoustic feedback, so with headphones the duplicate lines disappear, but we should still say that in onboarding.", + "reference": "麦克风通道会拾取扬声器声音的原因就是声学回授，所以戴上耳机重复的字幕就会消失，但我们仍然应该在引导里说明这一点。" + }, + { + "id": "F05", + "input": "Historically every attempt to bundle a second inference server inside a desktop app has failed on maintenance cost, so the question is whether a statically linked library changes that calculus.", + "reference": "历史上每一次在桌面应用里捆绑第二个推理服务的尝试都败在了维护成本上，所以问题在于静态链接的库是否改变了这个算计。" + }, + { + "id": "F06", + "input": "Let's park the pricing discussion, because until we know whether enterprise customers actually need SSO and audit logs, any number we put on the slide is going to be a guess.", + "reference": "我们先把定价讨论放一放，因为在知道企业客户是否真的需要SSO和审计日志之前，我们放在幻灯片上的任何数字都只是猜测。", + "terms": [ + "SSO" + ] + } + ] + }, + "G_zh_to_en": { + "description": "中文会议发言译成英文，含口语和数字", + "direction": "zh->en", + "cases": [ + { + "id": "G01", + "input": "这个功能我们下周三之前能上线吗", + "reference": "Can we ship this feature before next Wednesday?", + "terms": [ + "Wednesday" + ] + }, + { + "id": "G02", + "input": "那个就是说我们现在有两个方案一个是自己做一个是买第三方的", + "reference": "So we have two options now: build it ourselves, or buy a third-party solution.", + "terms_any": [ + [ + "two", + "2" + ] + ] + }, + { + "id": "G03", + "input": "上个季度我们的留存率从百分之三十八涨到了百分之四十五", + "reference": "Last quarter our retention rate went from 38% to 45%.", + "terms": [ + "38", + "45" + ] + }, + { + "id": "G04", + "input": "我觉得这个风险有点大，要不我们先小范围灰度一下", + "reference": "I think the risk is a bit high. How about a small-scale gradual rollout first?" + }, + { + "id": "G05", + "input": "张总说预算最多给到五十万，超了要重新审批", + "reference": "Mr. Zhang said the budget is capped at 500 thousand; anything above that needs re-approval.", + "terms": [ + "Zhang" + ], + "terms_any": [ + [ + "500", + "500,000", + "half a million" + ] + ] + }, + { + "id": "G06", + "input": "识别延迟大概一点五秒，翻译再加一秒左右，用户基本感觉不到", + "reference": "Recognition latency is about 1.5 seconds, translation adds roughly another second, so users barely notice.", + "terms": [ + "1.5" + ] + }, + { + "id": "G07", + "input": "等一下我这边网不太好你们先聊", + "reference": "Hold on, my connection isn't great. You go ahead." + }, + { + "id": "G08", + "input": "这个PR先不要合，Codex还有两条评论没处理", + "reference": "Don't merge this PR yet; there are still two Codex comments to address.", + "terms": [ + "PR", + "Codex" + ], + "terms_any": [ + [ + "two", + "2" + ] + ] + } + ] + }, + "H_context_followup": { + "description": "依赖上文的代词和省略，带 2 条上下文历史", + "direction": "en->zh", + "context": true, + "cases": [ + { + "id": "H01", + "input": "We benchmarked the MLX build last week.", + "reference": "我们上周对MLX版本做了基准测试。", + "terms": [ + "MLX" + ] + }, + { + "id": "H02", + "input": "It was about twice as fast on first token.", + "reference": "它的首字速度大约快了一倍。", + "terms_any": [ + [ + "一倍", + "2", + "两", + "二" + ] + ] + }, + { + "id": "H03", + "input": "But it only works on Apple Silicon.", + "reference": "但它只能在Apple Silicon上运行。", + "terms": [ + "Apple Silicon" + ] + }, + { + "id": "H04", + "input": "So for Intel Macs we'd still need the other one.", + "reference": "所以对于Intel的Mac我们仍然需要另一个。", + "terms": [ + "Intel" + ] + }, + { + "id": "H05", + "input": "She asked whether that doubles the maintenance work.", + "reference": "她问这是否会让维护工作翻倍。" + }, + { + "id": "H06", + "input": "Honestly, it probably does.", + "reference": "说实话，可能会。" + } + ] + } + } +} \ No newline at end of file diff --git a/benchmark/run_engine_compare.py b/benchmark/run_engine_compare.py new file mode 100644 index 0000000..2116d07 --- /dev/null +++ b/benchmark/run_engine_compare.py @@ -0,0 +1,331 @@ +#!/usr/bin/env python3 +""" +LCT engine comparison: same model weights, different inference runtimes. + +Measures what a live caption translator actually feels: time to first token, +end-to-end time per utterance, generation speed, and whether the output is a +clean translation (no preamble, no echo, numbers and names kept, no invented +endings for half sentences). + +Engines are given as name=kind:target where kind is + ollama target = model name (uses http://localhost:11434/api/chat) + openai target = base_url|model (any OpenAI-compatible server, e.g. llama-server) + +Usage: + python3 run_engine_compare.py \ + --engine ollama-mlx=ollama:qwen3.5:4b-mlx \ + --engine ollama-gguf=ollama:qwen3.5:4b \ + --engine llamacpp-gguf="openai:http://127.0.0.1:8089|qwen3.5-4b" \ + --runs 2 --output engine_compare_results.json --report engine_compare_report.md +""" +import argparse, json, math, re, statistics, sys, time, urllib.request +from collections import Counter +from datetime import datetime + +OLLAMA_URL = "http://localhost:11434" +OPENER = urllib.request.build_opener(urllib.request.ProxyHandler({})) +TEMPERATURE = 0.3 +NUM_CTX = 4096 +MAX_TOKENS = 256 +TARGET_NAMES = {"zh": "Chinese (Simplified)", "en": "English", "ja": "Japanese"} + +VIOLATION_PATTERNS = [ + "好的", "当然", "以下是", "翻译如下", "翻译：", "Translation:", "Here is", "Sure", "Of course", + "The translation", "译文：", "翻译结果", "I'll translate", "Let me translate", "```", "", + "[thinking]", "🔤", "Note:", "注：", "（注", "(Note", +] + +# ─── engines ──────────────────────────────────────────────────────────────── + +def _post_stream(url, payload, headers=None): + data = json.dumps(payload).encode() + req = urllib.request.Request(url, data=data, headers={"Content-Type": "application/json", **(headers or {})}) + return OPENER.open(req, timeout=180) + +def call_ollama(model, messages): + t0 = time.perf_counter(); ttft = None; text = []; final = {} + resp = _post_stream(f"{OLLAMA_URL}/api/chat", { + "model": model, "messages": messages, "stream": True, "think": False, + "keep_alive": "10m", + "options": {"temperature": TEMPERATURE, "num_ctx": NUM_CTX, "num_predict": MAX_TOKENS}, + }) + for line in resp: + if not line.strip(): + continue + chunk = json.loads(line) + piece = chunk.get("message", {}).get("content", "") + if piece and ttft is None: + ttft = time.perf_counter() - t0 + text.append(piece) + if chunk.get("done"): + final = chunk + e2e = time.perf_counter() - t0 + gen_tokens = final.get("eval_count") or len(text) + prompt_ms = (final.get("prompt_eval_duration") or 0) / 1e6 + return {"text": "".join(text), "ttft_s": ttft or e2e, "e2e_s": e2e, "gen_tokens": gen_tokens, + "prompt_tokens": final.get("prompt_eval_count"), "prompt_ms": prompt_ms or None} + +def call_openai(base_url, model, messages): + t0 = time.perf_counter(); ttft = None; text = []; usage = None; timings = None; n_chunks = 0 + resp = _post_stream(f"{base_url}/v1/chat/completions", { + "model": model, "messages": messages, "stream": True, "temperature": TEMPERATURE, + "max_tokens": MAX_TOKENS, "stream_options": {"include_usage": True}, + "chat_template_kwargs": {"enable_thinking": False}, + }) + for raw in resp: + line = raw.decode("utf-8", "replace").strip() + if not line.startswith("data:"): + continue + body = line[5:].strip() + if body == "[DONE]": + break + chunk = json.loads(body) + if chunk.get("usage"): + usage = chunk["usage"] + if chunk.get("timings"): + timings = chunk["timings"] + for choice in chunk.get("choices", []): + piece = (choice.get("delta") or {}).get("content") or "" + if piece: + n_chunks += 1 + if ttft is None: + ttft = time.perf_counter() - t0 + text.append(piece) + e2e = time.perf_counter() - t0 + gen_tokens = (usage or {}).get("completion_tokens") or (timings or {}).get("predicted_n") or n_chunks + prompt_ms = (timings or {}).get("prompt_ms") + return {"text": "".join(text), "ttft_s": ttft or e2e, "e2e_s": e2e, "gen_tokens": gen_tokens, + "prompt_tokens": (usage or {}).get("prompt_tokens") or (timings or {}).get("prompt_n"), + "prompt_ms": prompt_ms} + +def make_engine(spec): + name, _, rest = spec.partition("=") + kind, _, target = rest.partition(":") + if kind == "ollama": + return name, (lambda msgs, m=target: call_ollama(m, msgs)) + if kind == "openai": + base, _, model = target.partition("|") + return name, (lambda msgs, b=base.rstrip("/"), m=model or "default": call_openai(b, m, msgs)) + raise SystemExit(f"unknown engine kind in {spec!r}") + +# ─── scoring ──────────────────────────────────────────────────────────────── + +def clean(text): + text = re.sub(r".*?", "", text, flags=re.DOTALL) + text = re.sub(r".*", "", text, flags=re.DOTALL) + return text.replace("🔤", "").strip().strip('"').strip() + +def cjk_ratio(s): + letters = [c for c in s if c.isalpha()] + if not letters: + return 0.0 + return sum(1 for c in letters if "一" <= c <= "鿿") / len(letters) + +def chrf(hyp, ref, n_max=4, beta=2.0): + hyp = re.sub(r"\s+", "", hyp.lower()); ref = re.sub(r"\s+", "", ref.lower()) + if not hyp or not ref: + return 0.0 + precs, recs = [], [] + for n in range(1, n_max + 1): + h = Counter(hyp[i:i+n] for i in range(len(hyp) - n + 1)) + r = Counter(ref[i:i+n] for i in range(len(ref) - n + 1)) + if not h or not r: + continue + common = sum((h & r).values()) + precs.append(common / max(1, sum(h.values()))); recs.append(common / max(1, sum(r.values()))) + if not precs: + return 0.0 + p, r = statistics.mean(precs), statistics.mean(recs) + if p + r == 0: + return 0.0 + return (1 + beta**2) * p * r / (beta**2 * p + r) + +def score(case, direction, output): + out = clean(output) + problems = [] + low = out.lower() + for pat in VIOLATION_PATTERNS: + if pat.lower() in low: + problems.append(f"preamble:{pat}"); break + if len(case["input"]) >= 12 and case["input"].lower() in low: + problems.append("echo") + if "\n" in out.strip(): + problems.append("multiline") + target = direction.split("->")[1] + ratio = cjk_ratio(out) + if target == "zh" and ratio < 0.3: + problems.append("wrong-script") + if target == "en" and ratio > 0.05: + problems.append("wrong-script") + ref = case.get("reference", "") + max_ratio = case.get("max_ratio", 3.0) + if ref and len(out) > max_ratio * max(len(ref), 6): + problems.append("too-long") + missing = [] + for t in case.get("terms", []): + if t.lower() not in low: + missing.append(t) + for group in case.get("terms_any", []): + if not any(t.lower() in low for t in group): + missing.append("/".join(group)) + n_terms = len(case.get("terms", [])) + len(case.get("terms_any", [])) + return { + "output": out, "compliant": not problems, "problems": problems, + "terms_total": n_terms, "terms_kept": n_terms - len(missing), "terms_missing": missing, + "chrf": round(chrf(out, ref), 3) if ref else None, + } + +# ─── run ──────────────────────────────────────────────────────────────────── + +def build_messages(system_prompt, direction, text, history): + target = TARGET_NAMES[direction.split("->")[1]] + msgs = [{"role": "system", "content": system_prompt.replace("{TARGET_LANGUAGE}", target)}] + for src, out in history: + msgs.append({"role": "user", "content": f"🔤{src}🔤"}) + msgs.append({"role": "assistant", "content": out}) + msgs.append({"role": "user", "content": f"🔤{text}🔤"}) + return msgs + +def run_engine(name, call, data, categories, runs): + results = [] + sp = data["system_prompt"] + # warm-up (not recorded) + for _ in range(2): + call(build_messages(sp, "en->zh", "Let's begin.", [])) + for ckey in categories: + cat = data["categories"][ckey] + direction = cat["direction"]; use_ctx = cat.get("context", False) + for run in range(runs): + history = [] + for case in cat["cases"]: + msgs = build_messages(sp, direction, case["input"], history[-2:] if use_ctx else []) + try: + r = call(msgs) + except Exception as e: # noqa: BLE001 + print(f" {name} {case['id']} run{run}: ERROR {e}", file=sys.stderr) + continue + s = score(case, direction, r["text"]) + gen_s = max(1e-6, r["e2e_s"] - r["ttft_s"]) + rec = {"engine": name, "category": ckey, "id": case["id"], "run": run, + "input": case["input"], "reference": case.get("reference"), + "ttft_ms": round(r["ttft_s"] * 1000), "e2e_ms": round(r["e2e_s"] * 1000), + "gen_tokens": r["gen_tokens"], "tps": round(r["gen_tokens"] / gen_s, 1) if r["gen_tokens"] else None, + "prompt_tokens": r["prompt_tokens"], "prompt_ms": round(r["prompt_ms"]) if r["prompt_ms"] else None, + **s} + results.append(rec) + if use_ctx: + history.append((case["input"], s["output"])) + flag = "" if s["compliant"] and not s["terms_missing"] else f" !! {s['problems']} missing={s['terms_missing']}" + print(f" {name} {case['id']} r{run}: ttft {rec['ttft_ms']}ms e2e {rec['e2e_ms']}ms | {s['output'][:60]}{flag}") + return results + +def pct(v, p): + if not v: + return float("nan") + v = sorted(v); k = (len(v) - 1) * p / 100; f = math.floor(k); c = min(f + 1, len(v) - 1) + return v[f] + (v[c] - v[f]) * (k - f) + +def summarize(rows): + ttft = [r["ttft_ms"] for r in rows]; e2e = [r["e2e_ms"] for r in rows] + tps = [r["tps"] for r in rows if r["tps"]] + chrfs = [r["chrf"] for r in rows if r["chrf"] is not None] + tt = sum(r["terms_total"] for r in rows); tk = sum(r["terms_kept"] for r in rows) + return { + "n": len(rows), + "ttft_avg": statistics.mean(ttft), "ttft_p50": pct(ttft, 50), "ttft_p95": pct(ttft, 95), + "e2e_avg": statistics.mean(e2e), "e2e_p50": pct(e2e, 50), "e2e_p95": pct(e2e, 95), + "tps_avg": statistics.mean(tps) if tps else float("nan"), + "compliance": 100 * sum(r["compliant"] for r in rows) / len(rows), + "terms": 100 * tk / tt if tt else float("nan"), + "too_long": sum("too-long" in r["problems"] for r in rows), + "chrf": 100 * statistics.mean(chrfs) if chrfs else float("nan"), + } + +def report(results, engines, data, args): + lines = ["# LCT engine comparison: same weights, different runtimes", "", + f"Generated {datetime.now():%Y-%m-%d %H:%M}. Eval set `{args.data}` " + f"({sum(len(c['cases']) for c in data['categories'].values())} cases × {args.runs} runs), " + f"temperature {TEMPERATURE}, num_ctx {NUM_CTX}, max_tokens {MAX_TOKENS}, thinking off, streaming; " + "latency measured client-side from request start.", ""] + if args.note: + lines += [args.note, ""] + lines += ["## Overall", "", "| Metric | " + " | ".join(engines) + " |", "|---|" + "---:|" * len(engines)] + sums = {e: summarize([r for r in results if r["engine"] == e]) for e in engines} + def row(label, key, fmt): + lines.append(f"| {label} | " + " | ".join(fmt.format(sums[e][key]) for e in engines) + " |") + row("Cases scored", "n", "{}") + row("TTFT avg", "ttft_avg", "{:.0f} ms"); row("TTFT p50", "ttft_p50", "{:.0f} ms"); row("TTFT p95", "ttft_p95", "{:.0f} ms") + row("End-to-end avg", "e2e_avg", "{:.0f} ms"); row("End-to-end p50", "e2e_p50", "{:.0f} ms"); row("End-to-end p95", "e2e_p95", "{:.0f} ms") + row("Generation speed avg", "tps_avg", "{:.1f} tok/s") + row("Instruction compliance", "compliance", "{:.0f}%"); row("Numbers / names kept", "terms", "{:.0f}%") + row("Over-long outputs (drafts completed, commentary)", "too_long", "{}") + row("chrF vs reference", "chrf", "{:.1f}") + lines += ["", "## By category (end-to-end p50 / chrF)", "", "| Category | " + " | ".join(engines) + " |", "|---|" + "---:|" * len(engines)] + for ckey, cat in data["categories"].items(): + cells = [] + for e in engines: + rows = [r for r in results if r["engine"] == e and r["category"] == ckey] + if not rows: + cells.append("—"); continue + s = summarize(rows); cells.append(f"{s['e2e_p50']:.0f} ms / {s['chrf']:.0f}") + lines.append(f"| {ckey} — {cat['description']} | " + " | ".join(cells) + " |") + lines += ["", "## Problems", ""] + for e in engines: + probs = Counter(p for r in results if r["engine"] == e for p in r["problems"]) + miss = [(r["id"], r["terms_missing"]) for r in results if r["engine"] == e and r["terms_missing"] and r["run"] == 0] + lines.append(f"- **{e}**: " + (", ".join(f"{k} ×{v}" for k, v in probs.most_common()) or "none") + + (f"; missing terms (run 0): {miss}" if miss else "")) + lines += ["", "## Side-by-side samples (run 0)", ""] + sample_ids = ["B01", "B06", "C01", "D01", "D03", "D05", "G03", "G05", "H02", "F03"] + for sid in sample_ids: + rows = {r["engine"]: r for r in results if r["id"] == sid and r["run"] == 0} + if not rows: + continue + any_row = next(iter(rows.values())) + lines.append(f"**{sid}** `{any_row['input']}` ") + if any_row.get("reference"): + lines.append(f"ref: {any_row['reference']} ") + for e in engines: + if e in rows: + lines.append(f"- {e} ({rows[e]['e2e_ms']} ms): {rows[e]['output']}") + lines.append("") + return "\n".join(lines) + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--engine", action="append", default=[], help="name=ollama:model or name=openai:base_url|model") + ap.add_argument("--report-only", action="store_true", help="rebuild the report from --output without running anything") + ap.add_argument("--data", default="realtime_eval_set.json") + ap.add_argument("--categories", nargs="*") + ap.add_argument("--runs", type=int, default=2) + ap.add_argument("--output", default="engine_compare_results.json") + ap.add_argument("--report", default="engine_compare_report.md") + ap.add_argument("--append", action="store_true", help="merge into an existing results file (re-run one engine)") + ap.add_argument("--note", default="", help="free text placed at the top of the report (hardware, versions)") + args = ap.parse_args() + + data = json.load(open(args.data)) + categories = args.categories or list(data["categories"]) + engines = []; results = [] + if args.append or args.report_only: + try: + results = json.load(open(args.output))["results"] + except FileNotFoundError: + pass + if not args.engine and not args.report_only: + ap.error("give at least one --engine, or --report-only") + for spec in args.engine: + name, call = make_engine(spec) + print(f"=== {name} ===") + results = [r for r in results if r["engine"] != name] + results += run_engine(name, call, data, categories, args.runs) + engines.append(name) + all_engines = list(dict.fromkeys([r["engine"] for r in results])) + json.dump({"generated": datetime.now().isoformat(), "config": vars(args), "results": results}, + open(args.output, "w"), ensure_ascii=False, indent=1) + md = report(results, all_engines, data, args) + open(args.report, "w").write(md) + print(md) + +if __name__ == "__main__": + main()