diff --git a/fcitx5-engine/CMakeLists.txt b/fcitx5-engine/CMakeLists.txt index 8326f18..c4f39b3 100644 --- a/fcitx5-engine/CMakeLists.txt +++ b/fcitx5-engine/CMakeLists.txt @@ -41,6 +41,11 @@ add_library(config_loader_obj OBJECT src/config_loader.cpp) target_include_directories(config_loader_obj PUBLIC src) set_target_properties(config_loader_obj PROPERTIES POSITION_INDEPENDENT_CODE ON) +# ollama_request — pure request-body builder, consumed by the addon and tests +add_library(ollama_request_obj OBJECT src/ollama_request.cpp) +target_include_directories(ollama_request_obj PUBLIC src) +set_target_properties(ollama_request_obj PROPERTIES POSITION_INDEPENDENT_CODE ON) + add_library(xtype-fcitx5 SHARED src/xtype.cpp src/inference_client.cpp @@ -53,6 +58,7 @@ target_link_libraries(xtype-fcitx5 PRIVATE style_profile_obj prompt_builder_obj config_loader_obj + ollama_request_obj Fcitx5::Core CURL::libcurl ) diff --git a/fcitx5-engine/src/caret_context.h b/fcitx5-engine/src/caret_context.h new file mode 100644 index 0000000..287af89 --- /dev/null +++ b/fcitx5-engine/src/caret_context.h @@ -0,0 +1,79 @@ +#pragma once + +// Pure helpers for the text a suggestion continues. Header-only and +// Fcitx5-free so tests can use them directly. + +#include +#include + +namespace caret_context { + +inline bool isContinuation(char c) { // UTF-8 continuation byte (10xxxxxx) + return (static_cast(c) & 0xC0) == 0x80; +} + +// The app's own text before the caret, from Fcitx5 surrounding text: `text` +// with the caret `cursorChars` code points in. nullopt when it can't be +// trusted — the caret is past the end, or the text before it doesn't end with +// `lastTyped` (the app hasn't reported the keystroke we just saw yet). +inline std::optional beforeCaret(const std::string &text, + unsigned cursorChars, + const std::string &lastTyped) { + size_t bytes = 0; + for (unsigned n = 0; n < cursorChars; ++n) { + if (bytes >= text.size()) return std::nullopt; + ++bytes; + while (bytes < text.size() && isContinuation(text[bytes])) ++bytes; + } + std::string before = text.substr(0, bytes); + if (before.size() < lastTyped.size() || + before.compare(before.size() - lastTyped.size(), lastTyped.size(), lastTyped) != 0) + return std::nullopt; + return before; +} + +// The last code point of s, or "" if s is empty. +inline std::string lastChar(const std::string &s) { + if (s.empty()) return {}; + size_t start = s.size() - 1; + while (start > 0 && isContinuation(s[start])) --start; + return s.substr(start); +} + +inline bool isBlank(char c) { return c == ' ' || c == '\t'; } + +// The prompt to send for `context`: without trailing spaces/tabs. Tokenizers +// attach a word's leading space to the word, so a prompt ending on a bare +// space pushes the model toward digits and word fragments ("ritten", "0800"). +// The model then emits " word"; trimLeadingBlanksAfter() drops that space +// against the real context. +inline std::string promptFor(const std::string &context) { + size_t end = context.size(); + while (end > 0 && isBlank(context[end - 1])) --end; + return context.substr(0, end); +} + +// The model's continuation from its raw streamed output. Some chat templates +// (e.g. Qwen3 in Ollama) return the assistant prefill in front of the +// continuation; that echo is cut off. While the stream is still inside the +// echo, nothing is shown yet. +inline std::string withoutPromptEcho(const std::string &raw, const std::string &prompt) { + if (prompt.empty()) return raw; + if (raw.size() >= prompt.size()) + return raw.compare(0, prompt.size(), prompt) == 0 ? raw.substr(prompt.size()) : raw; + return prompt.compare(0, raw.size(), raw) == 0 ? std::string{} : raw; +} + +// A suggestion's leading spaces/tabs when the context already ends in +// whitespace — accepting them would type a double space. Mid-word and +// after-word contexts keep the model's spacing as is. +inline void trimLeadingBlanksAfter(std::string &suggestion, const std::string &context) { + if (context.empty()) return; + const char last = context.back(); + if (!isBlank(last) && last != '\n') return; + size_t n = 0; + while (n < suggestion.size() && isBlank(suggestion[n])) ++n; + suggestion.erase(0, n); +} + +} // namespace caret_context diff --git a/fcitx5-engine/src/inference_client.cpp b/fcitx5-engine/src/inference_client.cpp index 4fe66a5..1c65a8a 100644 --- a/fcitx5-engine/src/inference_client.cpp +++ b/fcitx5-engine/src/inference_client.cpp @@ -2,7 +2,6 @@ #include #include -#include #include #include #include @@ -10,6 +9,8 @@ #include #include +#include "ollama_request.h" + static FILE *ic_logfile() { static FILE *f = []() -> FILE* { const char* home = std::getenv("HOME"); @@ -35,22 +36,6 @@ static constexpr char kBaseSystemPrompt[] = // ── JSON helpers ────────────────────────────────────────────────────────────── -static std::string json_escape(std::string_view s) { - std::string out; - out.reserve(s.size()); - for (char c : s) { - switch (c) { - case '"': out += "\\\""; break; - case '\\': out += "\\\\"; break; - case '\n': out += "\\n"; break; - case '\r': out += "\\r"; break; - case '\t': out += "\\t"; break; - default: out += c; - } - } - return out; -} - // Extract a JSON string value for `key` from a flat JSON object. // Returns empty string if not found. static std::string json_str(std::string_view json, std::string_view key) { @@ -81,56 +66,6 @@ static std::string json_str(std::string_view json, std::string_view key) { return result; } -// ── Payload builder ─────────────────────────────────────────────────────────── - -static std::string build_payload(const InferenceConfig& cfg, - const std::string& context, - const std::string& system_prompt) { - std::string stop_arr = "["; - for (size_t i = 0; i < cfg.stop_tokens.size(); ++i) { - if (i) stop_arr += ','; - stop_arr += '"'; - stop_arr += json_escape(cfg.stop_tokens[i]); - stop_arr += '"'; - } - stop_arr += ']'; - - // std::to_chars is locale-independent — snprintf respects LC_NUMERIC and would - // emit "0,3" instead of "0.3" under European locales, breaking Ollama's JSON parser. - char temp_buf[32] = {}, top_p_buf[32] = {}; - { - auto r = std::to_chars(temp_buf, temp_buf + sizeof(temp_buf) - 1, - static_cast(cfg.temperature), - std::chars_format::general, 6); - *r.ptr = '\0'; - } - { - auto r = std::to_chars(top_p_buf, top_p_buf + sizeof(top_p_buf) - 1, - static_cast(cfg.top_p), - std::chars_format::general, 6); - *r.ptr = '\0'; - } - - // Assistant-prefill: context is placed in an incomplete assistant turn. - // The model continues its own text, bypassing the chat-response pattern that - // fires when conversational text appears in the user role. - std::string payload = std::string(R"({"model":")") + json_escape(cfg.model) - + R"(","messages":[)" - + R"({"role":"system","content":")" + json_escape(system_prompt) + R"("},)" - + R"({"role":"assistant","content":")" + json_escape(context) + R"("})" - + R"(],"stream":true,"options":{"num_predict":)" - + std::to_string(cfg.num_predict) - + R"(,"temperature":)" + temp_buf - + R"(,"top_p":)" + top_p_buf - + R"(,"stop":)" + stop_arr; - - if (cfg.threads.has_value()) - payload += R"(,"num_thread":)" + std::to_string(*cfg.threads); - - payload += "}}"; - return payload; -} - // ── CURL write callbacks ────────────────────────────────────────────────────── struct WriteState { @@ -289,7 +224,7 @@ void InferenceClient::execute(Req& req) { prompt_snapshot = _system_prompt; // frozen for this request } - std::string payload = build_payload(req.cfg, req.context, prompt_snapshot); + std::string payload = buildChatPayload(req.cfg, req.context, prompt_snapshot); std::string url = req.cfg.ollama_host + "/api/chat"; WriteState ws(req.gen, _gen, req.on_token); diff --git a/fcitx5-engine/src/ollama_request.cpp b/fcitx5-engine/src/ollama_request.cpp new file mode 100644 index 0000000..3bb7af2 --- /dev/null +++ b/fcitx5-engine/src/ollama_request.cpp @@ -0,0 +1,67 @@ +#include "ollama_request.h" + +#include +#include + +namespace { + +std::string json_escape(std::string_view s) { + std::string out; + out.reserve(s.size()); + for (char c : s) { + switch (c) { + case '"': out += "\\\""; break; + case '\\': out += "\\\\"; break; + case '\n': out += "\\n"; break; + case '\r': out += "\\r"; break; + case '\t': out += "\\t"; break; + default: out += c; + } + } + return out; +} + +// std::to_chars is locale-independent — snprintf respects LC_NUMERIC and would +// emit "0,3" instead of "0.3" under European locales, breaking Ollama's JSON parser. +std::string number(float v) { + char buf[32] = {}; + auto r = std::to_chars(buf, buf + sizeof(buf) - 1, static_cast(v), + std::chars_format::general, 6); + *r.ptr = '\0'; + return buf; +} + +std::string options(const InferenceConfig& cfg) { + std::string stop_arr = "["; + for (size_t i = 0; i < cfg.stop_tokens.size(); ++i) { + if (i) stop_arr += ','; + stop_arr += '"'; + stop_arr += json_escape(cfg.stop_tokens[i]); + stop_arr += '"'; + } + stop_arr += ']'; + + std::string out = R"("options":{"num_predict":)" + std::to_string(cfg.num_predict) + + R"(,"temperature":)" + number(cfg.temperature) + + R"(,"top_p":)" + number(cfg.top_p) + + R"(,"stop":)" + stop_arr; + if (cfg.threads.has_value()) + out += R"(,"num_thread":)" + std::to_string(*cfg.threads); + out += '}'; + return out; +} + +} // namespace + +std::string buildChatPayload(const InferenceConfig& cfg, + const std::string& context, + const std::string& systemPrompt) { + // Assistant-prefill: context is placed in an incomplete assistant turn. + // The model continues its own text, bypassing the chat-response pattern that + // fires when conversational text appears in the user role. + return std::string(R"({"model":")") + json_escape(cfg.model) + + R"(","messages":[)" + + R"({"role":"system","content":")" + json_escape(systemPrompt) + R"("},)" + + R"({"role":"assistant","content":")" + json_escape(context) + R"("})" + + R"(],"stream":true,)" + options(cfg) + "}"; +} diff --git a/fcitx5-engine/src/ollama_request.h b/fcitx5-engine/src/ollama_request.h new file mode 100644 index 0000000..9946408 --- /dev/null +++ b/fcitx5-engine/src/ollama_request.h @@ -0,0 +1,14 @@ +#pragma once + +// Pure builder for the Ollama request body behind one suggestion: POST +// /api/chat with the system prompt and the typed text as an unfinished +// assistant turn (assistant-prefill), streamed back as +// {"message":{"content":"…"}} lines. + +#include + +#include "config.h" + +std::string buildChatPayload(const InferenceConfig& cfg, + const std::string& context, + const std::string& systemPrompt); diff --git a/fcitx5-engine/src/xtype.cpp b/fcitx5-engine/src/xtype.cpp index 3447540..fdf64a9 100644 --- a/fcitx5-engine/src/xtype.cpp +++ b/fcitx5-engine/src/xtype.cpp @@ -21,8 +21,10 @@ #include #include #include +#include #include +#include "caret_context.h" #include "config_loader.h" #include "path_utils.h" #include "prompt_builder.h" @@ -375,7 +377,25 @@ void XTypeEngine::requestInference( fcitx::TrackableObjectReference icRef, fcitx::InputContext *icPtr) { + if (!icRef.isValid()) return; + + // Prefer the app's own text before the caret: unlike the keystroke buffer + // it includes text that was there before focus, pastes, and caret moves. + // Fall back to the buffer when the app doesn't report surrounding text or + // its copy doesn't have the last keystroke yet. auto ctx = _ctx.contextText(); + const char *source = "typed"; + if (icPtr->capabilityFlags().test(fcitx::CapabilityFlag::SurroundingText) && + icPtr->surroundingText().isValid()) { + const auto &st = icPtr->surroundingText(); + if (auto before = caret_context::beforeCaret(st.text(), st.cursor(), + caret_context::lastChar(ctx))) { + ctx = std::move(*before); + source = "surrounding"; + } else { + source = "typed (surrounding stale)"; + } + } if (static_cast(ctx.size()) < _cfg.inference.min_context_chars) return; if (static_cast(ctx.size()) > _cfg.inference.context_window) @@ -386,34 +406,40 @@ void XTypeEngine::requestInference( ++_metrics.suggestions_generated; const uint64_t myGen = _gen; const auto startUs = fcitx::now(CLOCK_MONOTONIC); + const std::string sentCtx = ctx; // the text being continued; used to tidy the output + std::string prompt = caret_context::promptFor(ctx); InferenceConfig reqCfg = _cfg.inference; if (auto* ov = currentAppOverride(); ov && ov->num_predict.has_value()) reqCfg.num_predict = *ov->num_predict; - dbg("requestInference ctx='%.40s...'", ctx.c_str()); + dbg("requestInference src=%s len=%zu ctx='%.40s...'", source, ctx.size(), ctx.c_str()); + // Raw model output so far; only touched on the main thread. + auto raw = std::make_shared(); _inference.request( - std::move(ctx), + prompt, std::move(reqCfg), - [this, myGen, icRef, icPtr](std::string token) { + [this, myGen, icRef, icPtr, sentCtx, prompt, raw](std::string token) { dbg("token received: '%s' icValid=%d", token.c_str(), (int)icRef.isValid()); if (!icRef.isValid()) { dbg("on_token: icRef invalid — dropping"); return; } _instance->eventDispatcher().scheduleWithContext( icRef, - [this, myGen, tok = std::move(token), icPtr]() { + [this, myGen, tok = std::move(token), icPtr, sentCtx, prompt, raw]() { if (myGen != _gen) return; - std::string current = - _ctx.hasSuggestion() ? *_ctx.suggestion() : ""; - _ctx.setSuggestion(current + tok); - dbg("preedit updated: '%s'", _ctx.suggestion()->c_str()); + *raw += tok; + std::string s = caret_context::withoutPromptEcho(*raw, prompt); + caret_context::trimLeadingBlanksAfter(s, sentCtx); + _ctx.setSuggestion(std::move(s)); + if (_ctx.hasSuggestion()) + dbg("preedit updated: '%s'", _ctx.suggestion()->c_str()); updatePreedit(icPtr); }); }, - [this, myGen, icRef, icPtr, startUs]() { + [this, myGen, icRef, icPtr, startUs, sentCtx]() { dbg("inference done, myGen=%llu curGen=%llu", (unsigned long long)myGen, (unsigned long long)_gen); _instance->eventDispatcher().scheduleWithContext( icRef, - [this, myGen, icPtr, startUs]() { + [this, myGen, icPtr, startUs, sentCtx]() { if (myGen != _gen) return; if (_ctx.hasSuggestion()) { std::string s = *_ctx.suggestion(); @@ -421,7 +447,7 @@ void XTypeEngine::requestInference( while (!s.empty() && std::isspace(static_cast(s.back()))) s.pop_back(); - const std::string ctxText = _ctx.contextText(); + const std::string &ctxText = sentCtx; // Strip tail-of-context echo: model repeats recently typed text. constexpr size_t kMaxCheck = 80; size_t check = std::min({s.size(), ctxText.size(), kMaxCheck}); @@ -445,6 +471,7 @@ void XTypeEngine::requestInference( if (lower(s.substr(0, kHeadCheck)) == lower(ctxText.substr(0, kHeadCheck))) s.clear(); } + caret_context::trimLeadingBlanksAfter(s, ctxText); _ctx.setSuggestion(std::move(s)); } updatePreedit(icPtr); diff --git a/fcitx5-engine/tests/CMakeLists.txt b/fcitx5-engine/tests/CMakeLists.txt index 4d9d53f..ad7514c 100644 --- a/fcitx5-engine/tests/CMakeLists.txt +++ b/fcitx5-engine/tests/CMakeLists.txt @@ -59,3 +59,17 @@ target_link_libraries(test_xtype_helpers PRIVATE Catch2::Catch2WithMain ) catch_discover_tests(test_xtype_helpers) + +add_executable(test_caret_context test_caret_context.cpp) +target_include_directories(test_caret_context PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../src) +target_link_libraries(test_caret_context PRIVATE + Catch2::Catch2WithMain +) +catch_discover_tests(test_caret_context) + +add_executable(test_ollama_request test_ollama_request.cpp) +target_link_libraries(test_ollama_request PRIVATE + ollama_request_obj + Catch2::Catch2WithMain +) +catch_discover_tests(test_ollama_request) diff --git a/fcitx5-engine/tests/test_caret_context.cpp b/fcitx5-engine/tests/test_caret_context.cpp new file mode 100644 index 0000000..1cea299 --- /dev/null +++ b/fcitx5-engine/tests/test_caret_context.cpp @@ -0,0 +1,98 @@ +#include + +#include + +#include "caret_context.h" + +using caret_context::beforeCaret; +using caret_context::lastChar; +using caret_context::promptFor; +using caret_context::withoutPromptEcho; +using caret_context::trimLeadingBlanksAfter; + +TEST_CASE("beforeCaret returns the text up to the caret", "[caret_context]") { + auto r = beforeCaret("Hello world, again", 11, "d"); + REQUIRE(r.has_value()); + REQUIRE(*r == "Hello world"); +} + +TEST_CASE("beforeCaret counts the caret in code points, not bytes", "[caret_context]") { + // "ação ok": a ç ã o (4 chars, 6 bytes) then " ok". + auto r = beforeCaret("ação ok", 4, "o"); + REQUIRE(r.has_value()); + REQUIRE(*r == "ação"); +} + +TEST_CASE("beforeCaret accepts a caret at the very end", "[caret_context]") { + auto r = beforeCaret("abc", 3, "c"); + REQUIRE(r.has_value()); + REQUIRE(*r == "abc"); +} + +TEST_CASE("beforeCaret rejects a caret past the end", "[caret_context]") { + REQUIRE_FALSE(beforeCaret("abc", 4, "c").has_value()); +} + +TEST_CASE("beforeCaret rejects text missing the last keystroke", "[caret_context]") { + // App still reports the text from before the user typed "x". + REQUIRE_FALSE(beforeCaret("abc", 3, "x").has_value()); + REQUIRE_FALSE(beforeCaret("", 0, "x").has_value()); +} + +TEST_CASE("lastChar returns the final code point", "[caret_context]") { + REQUIRE(lastChar("") == ""); + REQUIRE(lastChar("ab") == "b"); + REQUIRE(lastChar("aç") == "ç"); + REQUIRE(lastChar("€") == "€"); +} + +TEST_CASE("trimLeadingBlanksAfter drops the space after a trailing space", "[caret_context]") { + std::string s = " relevant suggestions"; + trimLeadingBlanksAfter(s, "suggest "); + REQUIRE(s == "relevant suggestions"); + + std::string t = " \tnext"; + trimLeadingBlanksAfter(t, "line\n"); + REQUIRE(t == "next"); +} + +TEST_CASE("trimLeadingBlanksAfter keeps spacing when the context ends in a word", "[caret_context]") { + std::string s = " relevant"; + trimLeadingBlanksAfter(s, "are not"); + REQUIRE(s == " relevant"); + + std::string mid = "ustrated"; + trimLeadingBlanksAfter(mid, "i feel f"); + REQUIRE(mid == "ustrated"); +} + +TEST_CASE("trimLeadingBlanksAfter can empty an all-blank token", "[caret_context]") { + std::string s = " "; + trimLeadingBlanksAfter(s, "word "); + REQUIRE(s.empty()); +} + +TEST_CASE("promptFor drops trailing spaces and tabs only", "[caret_context]") { + REQUIRE(promptFor("out of the ") == "out of the"); + REQUIRE(promptFor("a \t ") == "a"); + REQUIRE(promptFor("Best regards,\n") == "Best regards,\n"); + REQUIRE(promptFor("environ") == "environ"); + REQUIRE(promptFor(" ") == ""); +} + +TEST_CASE("withoutPromptEcho cuts a repeated prompt", "[caret_context]") { + // Qwen3 in Ollama returns the prefill, then the continuation. + REQUIRE(withoutPromptEcho("Best regards,\nEwerton", "Best regards,\n") == "Ewerton"); + REQUIRE(withoutPromptEcho("when it expired", "when it") == " expired"); +} + +TEST_CASE("withoutPromptEcho hides a partial echo while it streams", "[caret_context]") { + REQUIRE(withoutPromptEcho("Best reg", "Best regards,\n") == ""); + REQUIRE(withoutPromptEcho("Best regards,\n", "Best regards,\n") == ""); +} + +TEST_CASE("withoutPromptEcho leaves a plain continuation alone", "[caret_context]") { + REQUIRE(withoutPromptEcho(" office", "I was out of the") == " office"); + REQUIRE(withoutPromptEcho("ment", "staging environ") == "ment"); + REQUIRE(withoutPromptEcho("anything", "") == "anything"); +} diff --git a/fcitx5-engine/tests/test_ollama_request.cpp b/fcitx5-engine/tests/test_ollama_request.cpp new file mode 100644 index 0000000..19f2101 --- /dev/null +++ b/fcitx5-engine/tests/test_ollama_request.cpp @@ -0,0 +1,38 @@ +#include + +#include + +#include "ollama_request.h" + +static bool contains(const std::string& s, const std::string& needle) { + return s.find(needle) != std::string::npos; +} + +TEST_CASE("payload has the system prompt and the context as assistant prefill", "[ollama_request]") { + InferenceConfig cfg; + cfg.model = "qwen2.5:3b"; + auto body = buildChatPayload(cfg, "I feel the", "Be an autocomplete engine."); + + CHECK(contains(body, R"("model":"qwen2.5:3b")")); + CHECK(contains(body, R"({"role":"system","content":"Be an autocomplete engine."})")); + CHECK(contains(body, R"({"role":"assistant","content":"I feel the"})")); + CHECK(contains(body, R"("stream":true)")); +} + +TEST_CASE("payload carries the sampling options", "[ollama_request]") { + InferenceConfig cfg; + cfg.num_predict = 12; + cfg.temperature = 0.3f; + cfg.top_p = 0.9f; + cfg.stop_tokens = {".", "\n"}; + cfg.threads = 4; + auto body = buildChatPayload(cfg, "x", ""); + + CHECK(contains(body, R"("options":{"num_predict":12,"temperature":0.3,"top_p":0.9,"stop":[".","\n"],"num_thread":4})")); +} + +TEST_CASE("context is JSON-escaped", "[ollama_request]") { + InferenceConfig cfg; + auto body = buildChatPayload(cfg, "say \"hi\"\nC:\\path\t", ""); + CHECK(contains(body, R"({"role":"assistant","content":"say \"hi\"\nC:\\path\t"})")); +} diff --git a/scripts/benchmark_models.py b/scripts/benchmark_models.py new file mode 100644 index 0000000..b6417b7 --- /dev/null +++ b/scripts/benchmark_models.py @@ -0,0 +1,299 @@ +#!/usr/bin/env python3 +"""Compare Ollama models as XType autocomplete backends. + +Sends each model the same request the fcitx5 engine sends — /api/chat with +the system prompt and the text as an unfinished assistant turn, trailing +blanks dropped — then cleans the output the way +the engine does (prompt-echo cut, leading-blank trim, echo strip, trailing +trim) and scores it against the known continuation. --legacy reproduces the +engine before those two fixes (prompt sent as typed, no prompt-echo cut). + +Cases: the built-in set below, plus --cases FILE.jsonl +({"category", "context", "expected"} per line; see build_bench_cases.py). + +Usage: + python3 scripts/benchmark_models.py qwen2.5:3b qwen3:4b-instruct-2507-q4_K_M + python3 scripts/benchmark_models.py --cases /tmp/cases.jsonl --out /tmp/bench.json MODEL ... + +Stdlib only. +""" + +import argparse +import json +import re +import statistics +import time +import urllib.request + +HOST = "http://localhost:11434" + +# Mirrors kBaseSystemPrompt + "About the user:" in the engine (prompt_builder.cpp). +BASE_PROMPT = ( + "You are an inline text autocomplete engine. Continue the text you are given with a " + "few natural words. Output ONLY the continuation. No explanations, no responses, " + "no punctuation at the start." +) +DEFAULT_DESCRIPTION = "Senior software engineer who writes in Brazilian portuguese and English" + +OPTIONS = {"num_predict": 30, "temperature": 0.3, "top_p": 0.9, "stop": [".", "!", "?", "\n"]} + +# (category, context, expected continuation). Expected text is what a human +# would most likely type next; scoring compares leading words. +CASES = [ + # English — chat / email + ("en-chat", "Thanks for the quick reply. I'll take a look at it and get back to you as soon as ", "possible"), + ("en-chat", "Sorry for the late reply, I was out of the ", "office"), + ("en-chat", "The meeting has been moved from Tuesday to Wednesday at 3 pm, so please update your ", "calendar"), + ("en-chat", "Hi Maria,\nI hope you are doing well. I'm writing to follow ", "up on"), + ("en-chat", "No worries, take your time. Let me know if you have any ", "questions"), + # English — engineering + ("en-dev", "Can you send me the link to the pull ", "request"), + ("en-dev", "Before merging, please make sure all the unit tests are ", "passing"), + ("en-dev", "The request fails intermittently, so we should add a retry with exponential ", "backoff"), + ("en-dev", "I fixed the bug in the login flow: the session token was not being refreshed when it ", "expired"), + ("en-dev", "I feel the suggestions this app gives are not that useful, it seems like the suggestions are not related to the text being ", "typed"), + # English — mid-word + ("en-midword", "I'll deploy the new version to the staging environ", "ment"), + ("en-midword", "Please review the documentation before the next relea", "se"), + # English — long context, needs something from earlier in the text + ("en-long", + "Hi team,\nQuick update on the payments migration. We are moving the settlement service from Azure " + "Functions to Azure Container Apps, because the cold starts on Functions were adding up to two seconds " + "to every payout request. The new deployment has been running in staging since Monday and the p95 " + "latency dropped from 2.4 seconds to 180 milliseconds. Next week we will start routing 10 percent of " + "production traffic to it. The main reason we chose Container ", "Apps"), + ("en-long", + "Hello Daniel,\nMy name is Ewerton and I'm a senior software engineer at Lumon Pay. I'm reaching out " + "because we are evaluating vendors for card tokenization and your platform came up several times in our " + "research. Would you have 30 minutes next week for a quick call to walk us through the API and pricing? " + "Any day after 2 pm works for me.\nBest regards,\n", "Ewerton"), + # Portuguese (BR) + ("pt-chat", "Oi, tudo bem? Você consegue me mandar o relatório até o final do ", "dia"), + ("pt-chat", "Obrigado pela ajuda, fico no ", "aguardo"), + ("pt-chat", "Desculpa a demora para responder, estava em uma ", "reunião"), + ("pt-chat", "Qualquer dúvida, estou à ", "disposição"), + ("pt-chat", "Vamos marcar uma call amanhã de manhã para alinhar os próximos ", "passos"), + ("pt-dev", "Esse problema ainda está acontecendo em produção, preciso que alguém olhe os ", "logs"), + ("pt-midword", "A gente precisa resolver esse problema o mais rápido possí", "vel"), + ("pt-long", + "Olá Carlos, tudo bem?\nAqui é o Ewerton, da Lumon Pay. Estou entrando em contato porque estamos " + "migrando o serviço de pagamentos para o Azure e gostaríamos de entender melhor como funciona o suporte " + "de vocês para clientes enterprise. Você teria disponibilidade para uma conversa rápida na quinta-feira " + "à tarde?\nUm abraço,\n", "Ewerton"), +] + + +# ── Engine-equivalent request + clean-up ───────────────────────────────────── + +def payload(model, context, system_prompt): + return {"model": model, "stream": True, "options": OPTIONS, + "messages": [{"role": "system", "content": system_prompt}, + {"role": "assistant", "content": context}]} + + +def ends_blank(ctx): + return bool(ctx) and ctx[-1] in " \t\n" + + +def strip_echo_prefix(s, sent): + """caret_context::withoutPromptEcho: drop a verbatim re-emission of the + prompt (some chat templates return the assistant prefill in front).""" + if not sent: + return s + if s.startswith(sent): + return s[len(sent):] + if sent.startswith(s): # still streaming the echo + return "" + return s + + +def finalize(s, ctx, sent=None): + """xtype.cpp done-handler: trailing trim, tail/head echo strip, leading trim. + With `sent`, a full echo of the prompt is stripped first.""" + if sent is not None: + s = strip_echo_prefix(s, sent) + s = s.rstrip() + for n in range(min(len(s), len(ctx), 80), 3, -1): + if ctx.endswith(s[:n]): + s = s[n:] + break + if len(s) >= 12 and len(ctx) >= 12 and s[:12].lower() == ctx[:12].lower(): + s = "" + if ends_blank(ctx): + s = s.lstrip(" \t") + return s + + +def run_one(model, context, system_prompt, fixes=True): + # caret_context::promptFor: don't end the prompt on a bare space — + # tokenizers attach the space to the next word, so a trailing one skews + # the model toward digits and word fragments. The real context still + # drives the clean-up. + sent = context.rstrip(" \t") if fixes else context + body = payload(model, sent, system_prompt) + req = urllib.request.Request(HOST + "/api/chat", data=json.dumps(body).encode(), + headers={"Content-Type": "application/json"}) + t0 = time.perf_counter() + ttft = None + text = "" + final = {} + with urllib.request.urlopen(req, timeout=120) as resp: + for line in resp: + line = line.strip() + if not line: + continue + d = json.loads(line) + tok = (d.get("message") or {}).get("content") + if tok: + # Ghost text appears once the (trimmed) suggestion is non-empty. + shown = text + tok + visible = strip_echo_prefix(shown, sent) if fixes else shown + if ttft is None and (visible.lstrip(" \t") if ends_blank(context) else visible): + ttft = (time.perf_counter() - t0) * 1000 + text = shown + if d.get("done"): + final = d + break + total = (time.perf_counter() - t0) * 1000 + return { + "raw": text, + "suggestion": finalize(text, context, sent if fixes else None), + "ttft_ms": ttft if ttft is not None else total, + "total_ms": total, + "prompt_tokens": final.get("prompt_eval_count", 0), + "prompt_ms": final.get("prompt_eval_duration", 0) / 1e6, + "gen_tokens": final.get("eval_count", 0), + "gen_ms": final.get("eval_duration", 0) / 1e6, + } + + +# ── Scoring ────────────────────────────────────────────────────────────────── + +def words(s): + return [w for w in re.split(r"\s+", re.sub(r"[^\w\s]", " ", s.lower())) if w] + + +def score(suggestion, expected): + # Mid-word cases expect the rest of the word ("environ" → "ment"), so the + # same leading-word comparison covers both. + sw, ew = words(suggestion), words(expected) + first_ok = bool(sw) and sw[0] == ew[0] + matched = 0 + for a, b in zip(sw, ew): + if a != b: + break + matched += 1 + return {"first_word": first_ok, "words_matched": matched, "empty": not suggestion.strip()} + + +def pct(xs, p): + xs = sorted(xs) + return xs[max(0, min(len(xs) - 1, round(p / 100 * (len(xs) - 1))))] + + +def loaded_info(model): + try: + with urllib.request.urlopen(HOST + "/api/ps", timeout=5) as r: + for m in json.load(r).get("models", []): + if m.get("name") == model or m.get("model") == model: + size, vram = m.get("size", 0), m.get("size_vram", 0) + return {"size_gb": size / 1e9, "gpu_pct": round(100 * vram / size) if size else 0} + except Exception: + pass + return {} + + +def unload(model): + body = json.dumps({"model": model, "keep_alive": 0}).encode() + req = urllib.request.Request(HOST + "/api/generate", data=body, + headers={"Content-Type": "application/json"}) + try: + urllib.request.urlopen(req, timeout=60).read() + except Exception: + pass + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("models", nargs="+", help="Ollama model names") + ap.add_argument("--runs", type=int, default=1, help="runs per case (default 1)") + ap.add_argument("--cases", help="extra cases, JSONL") + ap.add_argument("--no-builtin", action="store_true", help="skip the built-in cases") + ap.add_argument("--description", default=DEFAULT_DESCRIPTION) + ap.add_argument("--out", default="/tmp/xtype-bench.json") + ap.add_argument("--legacy", action="store_true", + help="engine before the fixes: prompt as typed, no prompt-echo cut") + args = ap.parse_args() + + cases = [] if args.no_builtin else [{"category": c, "context": x, "expected": e} + for c, x, e in CASES] + if args.cases: + with open(args.cases, encoding="utf-8") as f: + cases += [json.loads(line) for line in f if line.strip()] + print(f"{len(cases)} cases, {args.runs} run(s) each", flush=True) + + system_prompt = BASE_PROMPT + ("\n\nAbout the user: " + args.description if args.description else "") + results = {} + + for model in args.models: + print(f"\n=== {model} ===", flush=True) + + t0 = time.perf_counter() + run_one(model, "Hello, this is a warm-up request to load the ", system_prompt) + load_s = time.perf_counter() - t0 + info = loaded_info(model) + print(f" load+first request: {load_s:.1f}s {info}", flush=True) + + rows = [] + t_model = time.perf_counter() + for i, case in enumerate(cases, 1): + cat, ctx, exp = case["category"], case["context"], case["expected"] + for run in range(args.runs): + try: + r = run_one(model, ctx, system_prompt, not args.legacy) + except Exception as e: # keep going; count as an empty suggestion + r = {"raw": "", "suggestion": "", "ttft_ms": 0, "total_ms": 0, "error": str(e)} + r.update(score(r["suggestion"], exp), category=cat, run=run, + context_tail=ctx[-45:], expected=exp) + rows.append(r) + if i % 100 == 0 or i == len(cases): + acc = 100 * sum(x["first_word"] for x in rows) / len(rows) + print(f" {i}/{len(cases)} first-word {acc:.0f}% " + f"({time.perf_counter() - t_model:.0f}s)", flush=True) + + results[model] = {"model": model, "load_s": load_s, **info, "rows": rows} + unload(model) + with open(args.out, "w") as f: # after each model, so a crash keeps the rest + json.dump(results, f, ensure_ascii=False, indent=1) + + # ── Summary ── + print("\n" + "=" * 112) + print(f"{'model':46} {'1st-word':>8} {'words':>6} {'empty':>6} " + f"{'TTFT p50':>9} {'p90':>6} {'total p50':>10} {'p90':>6} {'GPU':>5}") + print("-" * 112) + for res in results.values(): + rows = res["rows"] + first = 100 * sum(r["first_word"] for r in rows) / len(rows) + wm = statistics.mean(r["words_matched"] for r in rows) + empty = 100 * sum(r["empty"] for r in rows) / len(rows) + # Latency from each case's first run only: repeats hit Ollama's prompt cache. + tt = [r["ttft_ms"] for r in rows if r["run"] == 0] + tot = [r["total_ms"] for r in rows if r["run"] == 0] + print(f"{res['model'][:46]:46} {first:7.0f}% {wm:6.2f} {empty:5.0f}% " + f"{pct(tt, 50):7.0f}ms {pct(tt, 90):5.0f} {pct(tot, 50):8.0f}ms {pct(tot, 90):5.0f} " + f"{str(res.get('gpu_pct', '?')) + '%':>5}") + + print("\nFirst-word accuracy by category:") + cats = sorted({c["category"] for c in cases}) + print(f"{'model':46} " + " ".join(f"{c[:16]:>16}" for c in cats)) + for res in results.values(): + line = f"{res['model'][:46]:46} " + for c in cats: + rs = [r for r in res["rows"] if r["category"] == c] + line += f"{100 * sum(r['first_word'] for r in rs) / len(rs):15.0f}% " + print(line) + print(f"\nFull results: {args.out}") + + +if __name__ == "__main__": + main() diff --git a/scripts/build_bench_cases.py b/scripts/build_bench_cases.py new file mode 100644 index 0000000..9f7f611 --- /dev/null +++ b/scripts/build_bench_cases.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python3 +"""Build autocomplete test cases for benchmark_models.py from public text. + +Sources (downloaded into --data-dir): + en-email AESLC (Enron e-mail bodies) via the Hugging Face datasets-server + en-short Tatoeba English sentences + pt-short Tatoeba Portuguese sentences + +Each case cuts a real sentence at a word boundary (or, for ~15%, inside a +word): the text before the cut is the context, the rest of the sentence up to +. ! ? or a newline is the expected continuation. E-mail cases keep the +preceding e-mail text as context (last --context-chars characters). + +Usage: + python3 -I scripts/build_bench_cases.py --data-dir /tmp/xtype-bench-data \\ + --out /tmp/xtype-bench-cases.jsonl +""" + +import argparse +import bz2 +import csv +import json +import os +import random +import re +import sys +import urllib.request + +TATOEBA = "https://downloads.tatoeba.org/exports/per_language/{lang}/{lang}_sentences.tsv.bz2" +AESLC = ("https://datasets-server.huggingface.co/rows?dataset=Yale-LILY/aeslc" + "&config=default&split=train&offset={offset}&length=100") + +csv.field_size_limit(sys.maxsize) + + +def fetch(url, path): + if not os.path.exists(path): + print(f" downloading {url}", file=sys.stderr) + with urllib.request.urlopen(url, timeout=120) as r, open(path, "wb") as f: + f.write(r.read()) + return path + + +def tatoeba(lang, data_dir): + path = fetch(TATOEBA.format(lang=lang), os.path.join(data_dir, f"{lang}_sentences.tsv.bz2")) + with bz2.open(path, "rt", encoding="utf-8") as f: + for row in csv.reader(f, delimiter="\t", quoting=csv.QUOTE_NONE): + if len(row) >= 3: + yield row[2] + + +def aeslc(data_dir, pages): + for p in range(pages): + path = fetch(AESLC.format(offset=p * 100), os.path.join(data_dir, f"aeslc_{p:03}.json")) + with open(path, encoding="utf-8") as f: + for r in json.load(f).get("rows", []): + yield r["row"]["email_body"] + + +SENT_END = re.compile(r"[.!?\n]") +BAD = re.compile(r"https?://|www\.|@|-----|\b(From|To|Cc|Subject|Sent):|[<>{}|=_*#]") + + +def make_case(rng, prefix_text, sentence, category, context_chars, mid_word_rate=0.15): + """Cut `sentence` (which follows prefix_text) and return a case or None.""" + words = list(re.finditer(r"\S+", sentence)) + if len(words) < 4: + return None + if rng.random() < mid_word_rate: + # Inside a word of >= 5 letters, keeping >= 2 letters on each side. + cands = [w for w in words[1:] if re.fullmatch(r"\w{5,}", w.group())] + if not cands: + return None + w = rng.choice(cands) + cut = w.start() + rng.randint(2, len(w.group()) - 2) + kind = "midword" + else: + # Before word k (k >= 2): the context ends with the space after word k-1. + k = rng.randint(2, len(words) - 1) + cut = words[k].start() + kind = "word" + context = prefix_text + sentence[:cut] + m = SENT_END.search(sentence, cut) + expected = sentence[cut:m.start() if m else len(sentence)].strip() + if not re.search(r"\w", expected) or len(context.strip()) < 10: + return None + return {"category": f"{category}-{kind}", "context": context[-context_chars:], "expected": expected} + + +def short_cases(rng, sentences, category, n, context_chars): + pool = [s for s in sentences if 25 <= len(s) <= 160 and not BAD.search(s)] + rng.shuffle(pool) + out = [] + for s in pool: + c = make_case(rng, "", s, category, context_chars) + if c: + out.append(c) + if len(out) >= n: + break + return out + + +def email_cases(rng, bodies, n, context_chars): + out = [] + bodies = [b for b in bodies if 200 <= len(b) <= 4000] + rng.shuffle(bodies) + for body in bodies: + body = re.sub(r"[ \t]+", " ", body) + # Sentences (or lines) that look like prose. + spans = [m for m in re.finditer(r"[^.!?\n]+[.!?]?", body) + if 30 <= len(m.group().strip()) <= 220 and not BAD.search(m.group())] + if not spans: + continue + m = rng.choice(spans) + start = m.start() + (len(m.group()) - len(m.group().lstrip())) + c = make_case(rng, body[:start], body[start:m.end()], "en-email", context_chars) + if c: + out.append(c) + if len(out) >= n: + break + return out + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--data-dir", required=True) + ap.add_argument("--out", required=True) + ap.add_argument("--seed", type=int, default=7) + ap.add_argument("--emails", type=int, default=450) + ap.add_argument("--en", type=int, default=300) + ap.add_argument("--pt", type=int, default=450) + ap.add_argument("--context-chars", type=int, default=1000, help="engine context_window") + args = ap.parse_args() + os.makedirs(args.data_dir, exist_ok=True) + rng = random.Random(args.seed) + + cases = email_cases(rng, list(aeslc(args.data_dir, pages=12)), args.emails, args.context_chars) + cases += short_cases(rng, tatoeba("eng", args.data_dir), "en-short", args.en, args.context_chars) + cases += short_cases(rng, tatoeba("por", args.data_dir), "pt-short", args.pt, args.context_chars) + + with open(args.out, "w", encoding="utf-8") as f: + for c in cases: + f.write(json.dumps(c, ensure_ascii=False) + "\n") + by = {} + for c in cases: + by[c["category"]] = by.get(c["category"], 0) + 1 + print(f"{len(cases)} cases → {args.out} {by}", file=sys.stderr) + + +if __name__ == "__main__": + main() diff --git a/settings/qt6-app/qml/pages/PageModel.qml b/settings/qt6-app/qml/pages/PageModel.qml index 549a0d2..067de86 100644 --- a/settings/qt6-app/qml/pages/PageModel.qml +++ b/settings/qt6-app/qml/pages/PageModel.qml @@ -105,10 +105,10 @@ Item { XRow { label: "Context window" - desc: "Tokens of context fed to the model" + desc: "Characters of text before the caret fed to the model" width: parent.width XSlider { - min: 512; max: 4096; step: 256 + min: 100; max: 4000; step: 100 value: Config.contextWindow onCommitted: (v) => Config.contextWindow = v width: 200