From 1c3bd1e3554b79c075834d04ebc0e20a44529fbc Mon Sep 17 00:00:00 2001 From: Ewerton Mattos Date: Thu, 15 Oct 2026 18:51:36 -0300 Subject: [PATCH 1/3] feat(engine): continue the app's real text, and fix three prompt bugs Context: when the app reports surrounding text, send the text before the caret instead of the engine's keystroke buffer, so text that was there before focus, pastes and caret moves are included. Falls back to the buffer when the app has no surrounding text or its copy doesn't have the last keystroke yet. Prompt fixes, found with scripts/benchmark_models.py: - Send the context without trailing spaces. Tokenizers attach a word's leading space to the word, so a prompt ending on a bare space pushed models toward digits and word fragments ("0800 726 0000", "ritten"). - Some chat templates (Qwen3 in Ollama) return the assistant prefill before the continuation, and the head-echo check then threw away correct suggestions. Cut the echoed prompt instead. - Drop a suggestion's leading blanks when the text already ends in whitespace, so accepting it no longer types a double space. On 1,222 real sentences this lifts first-word accuracy from 5.9% to 18.8% (qwen2.5:3b) and 4.6% to 24.5% (qwen3:4b-instruct). The request body builder moves to ollama_request.{h,cpp} so it is unit-tested. Co-Authored-By: Claude Opus 5.5 --- fcitx5-engine/CMakeLists.txt | 6 ++ fcitx5-engine/src/caret_context.h | 79 +++++++++++++++++ fcitx5-engine/src/inference_client.cpp | 71 +-------------- fcitx5-engine/src/ollama_request.cpp | 67 ++++++++++++++ fcitx5-engine/src/ollama_request.h | 14 +++ fcitx5-engine/src/xtype.cpp | 49 ++++++++--- fcitx5-engine/tests/CMakeLists.txt | 14 +++ fcitx5-engine/tests/test_caret_context.cpp | 98 +++++++++++++++++++++ fcitx5-engine/tests/test_ollama_request.cpp | 38 ++++++++ 9 files changed, 357 insertions(+), 79 deletions(-) create mode 100644 fcitx5-engine/src/caret_context.h create mode 100644 fcitx5-engine/src/ollama_request.cpp create mode 100644 fcitx5-engine/src/ollama_request.h create mode 100644 fcitx5-engine/tests/test_caret_context.cpp create mode 100644 fcitx5-engine/tests/test_ollama_request.cpp diff --git a/fcitx5-engine/CMakeLists.txt b/fcitx5-engine/CMakeLists.txt index 8326f18..c4f39b3 100644 --- a/fcitx5-engine/CMakeLists.txt +++ b/fcitx5-engine/CMakeLists.txt @@ -41,6 +41,11 @@ add_library(config_loader_obj OBJECT src/config_loader.cpp) target_include_directories(config_loader_obj PUBLIC src) set_target_properties(config_loader_obj PROPERTIES POSITION_INDEPENDENT_CODE ON) +# ollama_request — pure request-body builder, consumed by the addon and tests +add_library(ollama_request_obj OBJECT src/ollama_request.cpp) +target_include_directories(ollama_request_obj PUBLIC src) +set_target_properties(ollama_request_obj PROPERTIES POSITION_INDEPENDENT_CODE ON) + add_library(xtype-fcitx5 SHARED src/xtype.cpp src/inference_client.cpp @@ -53,6 +58,7 @@ target_link_libraries(xtype-fcitx5 PRIVATE style_profile_obj prompt_builder_obj config_loader_obj + ollama_request_obj Fcitx5::Core CURL::libcurl ) diff --git a/fcitx5-engine/src/caret_context.h b/fcitx5-engine/src/caret_context.h new file mode 100644 index 0000000..287af89 --- /dev/null +++ b/fcitx5-engine/src/caret_context.h @@ -0,0 +1,79 @@ +#pragma once + +// Pure helpers for the text a suggestion continues. Header-only and +// Fcitx5-free so tests can use them directly. + +#include +#include + +namespace caret_context { + +inline bool isContinuation(char c) { // UTF-8 continuation byte (10xxxxxx) + return (static_cast(c) & 0xC0) == 0x80; +} + +// The app's own text before the caret, from Fcitx5 surrounding text: `text` +// with the caret `cursorChars` code points in. nullopt when it can't be +// trusted — the caret is past the end, or the text before it doesn't end with +// `lastTyped` (the app hasn't reported the keystroke we just saw yet). +inline std::optional beforeCaret(const std::string &text, + unsigned cursorChars, + const std::string &lastTyped) { + size_t bytes = 0; + for (unsigned n = 0; n < cursorChars; ++n) { + if (bytes >= text.size()) return std::nullopt; + ++bytes; + while (bytes < text.size() && isContinuation(text[bytes])) ++bytes; + } + std::string before = text.substr(0, bytes); + if (before.size() < lastTyped.size() || + before.compare(before.size() - lastTyped.size(), lastTyped.size(), lastTyped) != 0) + return std::nullopt; + return before; +} + +// The last code point of s, or "" if s is empty. +inline std::string lastChar(const std::string &s) { + if (s.empty()) return {}; + size_t start = s.size() - 1; + while (start > 0 && isContinuation(s[start])) --start; + return s.substr(start); +} + +inline bool isBlank(char c) { return c == ' ' || c == '\t'; } + +// The prompt to send for `context`: without trailing spaces/tabs. Tokenizers +// attach a word's leading space to the word, so a prompt ending on a bare +// space pushes the model toward digits and word fragments ("ritten", "0800"). +// The model then emits " word"; trimLeadingBlanksAfter() drops that space +// against the real context. +inline std::string promptFor(const std::string &context) { + size_t end = context.size(); + while (end > 0 && isBlank(context[end - 1])) --end; + return context.substr(0, end); +} + +// The model's continuation from its raw streamed output. Some chat templates +// (e.g. Qwen3 in Ollama) return the assistant prefill in front of the +// continuation; that echo is cut off. While the stream is still inside the +// echo, nothing is shown yet. +inline std::string withoutPromptEcho(const std::string &raw, const std::string &prompt) { + if (prompt.empty()) return raw; + if (raw.size() >= prompt.size()) + return raw.compare(0, prompt.size(), prompt) == 0 ? raw.substr(prompt.size()) : raw; + return prompt.compare(0, raw.size(), raw) == 0 ? std::string{} : raw; +} + +// A suggestion's leading spaces/tabs when the context already ends in +// whitespace — accepting them would type a double space. Mid-word and +// after-word contexts keep the model's spacing as is. +inline void trimLeadingBlanksAfter(std::string &suggestion, const std::string &context) { + if (context.empty()) return; + const char last = context.back(); + if (!isBlank(last) && last != '\n') return; + size_t n = 0; + while (n < suggestion.size() && isBlank(suggestion[n])) ++n; + suggestion.erase(0, n); +} + +} // namespace caret_context diff --git a/fcitx5-engine/src/inference_client.cpp b/fcitx5-engine/src/inference_client.cpp index 4fe66a5..1c65a8a 100644 --- a/fcitx5-engine/src/inference_client.cpp +++ b/fcitx5-engine/src/inference_client.cpp @@ -2,7 +2,6 @@ #include #include -#include #include #include #include @@ -10,6 +9,8 @@ #include #include +#include "ollama_request.h" + static FILE *ic_logfile() { static FILE *f = []() -> FILE* { const char* home = std::getenv("HOME"); @@ -35,22 +36,6 @@ static constexpr char kBaseSystemPrompt[] = // ── JSON helpers ────────────────────────────────────────────────────────────── -static std::string json_escape(std::string_view s) { - std::string out; - out.reserve(s.size()); - for (char c : s) { - switch (c) { - case '"': out += "\\\""; break; - case '\\': out += "\\\\"; break; - case '\n': out += "\\n"; break; - case '\r': out += "\\r"; break; - case '\t': out += "\\t"; break; - default: out += c; - } - } - return out; -} - // Extract a JSON string value for `key` from a flat JSON object. // Returns empty string if not found. static std::string json_str(std::string_view json, std::string_view key) { @@ -81,56 +66,6 @@ static std::string json_str(std::string_view json, std::string_view key) { return result; } -// ── Payload builder ─────────────────────────────────────────────────────────── - -static std::string build_payload(const InferenceConfig& cfg, - const std::string& context, - const std::string& system_prompt) { - std::string stop_arr = "["; - for (size_t i = 0; i < cfg.stop_tokens.size(); ++i) { - if (i) stop_arr += ','; - stop_arr += '"'; - stop_arr += json_escape(cfg.stop_tokens[i]); - stop_arr += '"'; - } - stop_arr += ']'; - - // std::to_chars is locale-independent — snprintf respects LC_NUMERIC and would - // emit "0,3" instead of "0.3" under European locales, breaking Ollama's JSON parser. - char temp_buf[32] = {}, top_p_buf[32] = {}; - { - auto r = std::to_chars(temp_buf, temp_buf + sizeof(temp_buf) - 1, - static_cast(cfg.temperature), - std::chars_format::general, 6); - *r.ptr = '\0'; - } - { - auto r = std::to_chars(top_p_buf, top_p_buf + sizeof(top_p_buf) - 1, - static_cast(cfg.top_p), - std::chars_format::general, 6); - *r.ptr = '\0'; - } - - // Assistant-prefill: context is placed in an incomplete assistant turn. - // The model continues its own text, bypassing the chat-response pattern that - // fires when conversational text appears in the user role. - std::string payload = std::string(R"({"model":")") + json_escape(cfg.model) - + R"(","messages":[)" - + R"({"role":"system","content":")" + json_escape(system_prompt) + R"("},)" - + R"({"role":"assistant","content":")" + json_escape(context) + R"("})" - + R"(],"stream":true,"options":{"num_predict":)" - + std::to_string(cfg.num_predict) - + R"(,"temperature":)" + temp_buf - + R"(,"top_p":)" + top_p_buf - + R"(,"stop":)" + stop_arr; - - if (cfg.threads.has_value()) - payload += R"(,"num_thread":)" + std::to_string(*cfg.threads); - - payload += "}}"; - return payload; -} - // ── CURL write callbacks ────────────────────────────────────────────────────── struct WriteState { @@ -289,7 +224,7 @@ void InferenceClient::execute(Req& req) { prompt_snapshot = _system_prompt; // frozen for this request } - std::string payload = build_payload(req.cfg, req.context, prompt_snapshot); + std::string payload = buildChatPayload(req.cfg, req.context, prompt_snapshot); std::string url = req.cfg.ollama_host + "/api/chat"; WriteState ws(req.gen, _gen, req.on_token); diff --git a/fcitx5-engine/src/ollama_request.cpp b/fcitx5-engine/src/ollama_request.cpp new file mode 100644 index 0000000..3bb7af2 --- /dev/null +++ b/fcitx5-engine/src/ollama_request.cpp @@ -0,0 +1,67 @@ +#include "ollama_request.h" + +#include +#include + +namespace { + +std::string json_escape(std::string_view s) { + std::string out; + out.reserve(s.size()); + for (char c : s) { + switch (c) { + case '"': out += "\\\""; break; + case '\\': out += "\\\\"; break; + case '\n': out += "\\n"; break; + case '\r': out += "\\r"; break; + case '\t': out += "\\t"; break; + default: out += c; + } + } + return out; +} + +// std::to_chars is locale-independent — snprintf respects LC_NUMERIC and would +// emit "0,3" instead of "0.3" under European locales, breaking Ollama's JSON parser. +std::string number(float v) { + char buf[32] = {}; + auto r = std::to_chars(buf, buf + sizeof(buf) - 1, static_cast(v), + std::chars_format::general, 6); + *r.ptr = '\0'; + return buf; +} + +std::string options(const InferenceConfig& cfg) { + std::string stop_arr = "["; + for (size_t i = 0; i < cfg.stop_tokens.size(); ++i) { + if (i) stop_arr += ','; + stop_arr += '"'; + stop_arr += json_escape(cfg.stop_tokens[i]); + stop_arr += '"'; + } + stop_arr += ']'; + + std::string out = R"("options":{"num_predict":)" + std::to_string(cfg.num_predict) + + R"(,"temperature":)" + number(cfg.temperature) + + R"(,"top_p":)" + number(cfg.top_p) + + R"(,"stop":)" + stop_arr; + if (cfg.threads.has_value()) + out += R"(,"num_thread":)" + std::to_string(*cfg.threads); + out += '}'; + return out; +} + +} // namespace + +std::string buildChatPayload(const InferenceConfig& cfg, + const std::string& context, + const std::string& systemPrompt) { + // Assistant-prefill: context is placed in an incomplete assistant turn. + // The model continues its own text, bypassing the chat-response pattern that + // fires when conversational text appears in the user role. + return std::string(R"({"model":")") + json_escape(cfg.model) + + R"(","messages":[)" + + R"({"role":"system","content":")" + json_escape(systemPrompt) + R"("},)" + + R"({"role":"assistant","content":")" + json_escape(context) + R"("})" + + R"(],"stream":true,)" + options(cfg) + "}"; +} diff --git a/fcitx5-engine/src/ollama_request.h b/fcitx5-engine/src/ollama_request.h new file mode 100644 index 0000000..9946408 --- /dev/null +++ b/fcitx5-engine/src/ollama_request.h @@ -0,0 +1,14 @@ +#pragma once + +// Pure builder for the Ollama request body behind one suggestion: POST +// /api/chat with the system prompt and the typed text as an unfinished +// assistant turn (assistant-prefill), streamed back as +// {"message":{"content":"…"}} lines. + +#include + +#include "config.h" + +std::string buildChatPayload(const InferenceConfig& cfg, + const std::string& context, + const std::string& systemPrompt); diff --git a/fcitx5-engine/src/xtype.cpp b/fcitx5-engine/src/xtype.cpp index 3447540..fdf64a9 100644 --- a/fcitx5-engine/src/xtype.cpp +++ b/fcitx5-engine/src/xtype.cpp @@ -21,8 +21,10 @@ #include #include #include +#include #include +#include "caret_context.h" #include "config_loader.h" #include "path_utils.h" #include "prompt_builder.h" @@ -375,7 +377,25 @@ void XTypeEngine::requestInference( fcitx::TrackableObjectReference icRef, fcitx::InputContext *icPtr) { + if (!icRef.isValid()) return; + + // Prefer the app's own text before the caret: unlike the keystroke buffer + // it includes text that was there before focus, pastes, and caret moves. + // Fall back to the buffer when the app doesn't report surrounding text or + // its copy doesn't have the last keystroke yet. auto ctx = _ctx.contextText(); + const char *source = "typed"; + if (icPtr->capabilityFlags().test(fcitx::CapabilityFlag::SurroundingText) && + icPtr->surroundingText().isValid()) { + const auto &st = icPtr->surroundingText(); + if (auto before = caret_context::beforeCaret(st.text(), st.cursor(), + caret_context::lastChar(ctx))) { + ctx = std::move(*before); + source = "surrounding"; + } else { + source = "typed (surrounding stale)"; + } + } if (static_cast(ctx.size()) < _cfg.inference.min_context_chars) return; if (static_cast(ctx.size()) > _cfg.inference.context_window) @@ -386,34 +406,40 @@ void XTypeEngine::requestInference( ++_metrics.suggestions_generated; const uint64_t myGen = _gen; const auto startUs = fcitx::now(CLOCK_MONOTONIC); + const std::string sentCtx = ctx; // the text being continued; used to tidy the output + std::string prompt = caret_context::promptFor(ctx); InferenceConfig reqCfg = _cfg.inference; if (auto* ov = currentAppOverride(); ov && ov->num_predict.has_value()) reqCfg.num_predict = *ov->num_predict; - dbg("requestInference ctx='%.40s...'", ctx.c_str()); + dbg("requestInference src=%s len=%zu ctx='%.40s...'", source, ctx.size(), ctx.c_str()); + // Raw model output so far; only touched on the main thread. + auto raw = std::make_shared(); _inference.request( - std::move(ctx), + prompt, std::move(reqCfg), - [this, myGen, icRef, icPtr](std::string token) { + [this, myGen, icRef, icPtr, sentCtx, prompt, raw](std::string token) { dbg("token received: '%s' icValid=%d", token.c_str(), (int)icRef.isValid()); if (!icRef.isValid()) { dbg("on_token: icRef invalid — dropping"); return; } _instance->eventDispatcher().scheduleWithContext( icRef, - [this, myGen, tok = std::move(token), icPtr]() { + [this, myGen, tok = std::move(token), icPtr, sentCtx, prompt, raw]() { if (myGen != _gen) return; - std::string current = - _ctx.hasSuggestion() ? *_ctx.suggestion() : ""; - _ctx.setSuggestion(current + tok); - dbg("preedit updated: '%s'", _ctx.suggestion()->c_str()); + *raw += tok; + std::string s = caret_context::withoutPromptEcho(*raw, prompt); + caret_context::trimLeadingBlanksAfter(s, sentCtx); + _ctx.setSuggestion(std::move(s)); + if (_ctx.hasSuggestion()) + dbg("preedit updated: '%s'", _ctx.suggestion()->c_str()); updatePreedit(icPtr); }); }, - [this, myGen, icRef, icPtr, startUs]() { + [this, myGen, icRef, icPtr, startUs, sentCtx]() { dbg("inference done, myGen=%llu curGen=%llu", (unsigned long long)myGen, (unsigned long long)_gen); _instance->eventDispatcher().scheduleWithContext( icRef, - [this, myGen, icPtr, startUs]() { + [this, myGen, icPtr, startUs, sentCtx]() { if (myGen != _gen) return; if (_ctx.hasSuggestion()) { std::string s = *_ctx.suggestion(); @@ -421,7 +447,7 @@ void XTypeEngine::requestInference( while (!s.empty() && std::isspace(static_cast(s.back()))) s.pop_back(); - const std::string ctxText = _ctx.contextText(); + const std::string &ctxText = sentCtx; // Strip tail-of-context echo: model repeats recently typed text. constexpr size_t kMaxCheck = 80; size_t check = std::min({s.size(), ctxText.size(), kMaxCheck}); @@ -445,6 +471,7 @@ void XTypeEngine::requestInference( if (lower(s.substr(0, kHeadCheck)) == lower(ctxText.substr(0, kHeadCheck))) s.clear(); } + caret_context::trimLeadingBlanksAfter(s, ctxText); _ctx.setSuggestion(std::move(s)); } updatePreedit(icPtr); diff --git a/fcitx5-engine/tests/CMakeLists.txt b/fcitx5-engine/tests/CMakeLists.txt index 4d9d53f..ad7514c 100644 --- a/fcitx5-engine/tests/CMakeLists.txt +++ b/fcitx5-engine/tests/CMakeLists.txt @@ -59,3 +59,17 @@ target_link_libraries(test_xtype_helpers PRIVATE Catch2::Catch2WithMain ) catch_discover_tests(test_xtype_helpers) + +add_executable(test_caret_context test_caret_context.cpp) +target_include_directories(test_caret_context PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../src) +target_link_libraries(test_caret_context PRIVATE + Catch2::Catch2WithMain +) +catch_discover_tests(test_caret_context) + +add_executable(test_ollama_request test_ollama_request.cpp) +target_link_libraries(test_ollama_request PRIVATE + ollama_request_obj + Catch2::Catch2WithMain +) +catch_discover_tests(test_ollama_request) diff --git a/fcitx5-engine/tests/test_caret_context.cpp b/fcitx5-engine/tests/test_caret_context.cpp new file mode 100644 index 0000000..1cea299 --- /dev/null +++ b/fcitx5-engine/tests/test_caret_context.cpp @@ -0,0 +1,98 @@ +#include + +#include + +#include "caret_context.h" + +using caret_context::beforeCaret; +using caret_context::lastChar; +using caret_context::promptFor; +using caret_context::withoutPromptEcho; +using caret_context::trimLeadingBlanksAfter; + +TEST_CASE("beforeCaret returns the text up to the caret", "[caret_context]") { + auto r = beforeCaret("Hello world, again", 11, "d"); + REQUIRE(r.has_value()); + REQUIRE(*r == "Hello world"); +} + +TEST_CASE("beforeCaret counts the caret in code points, not bytes", "[caret_context]") { + // "ação ok": a ç ã o (4 chars, 6 bytes) then " ok". + auto r = beforeCaret("ação ok", 4, "o"); + REQUIRE(r.has_value()); + REQUIRE(*r == "ação"); +} + +TEST_CASE("beforeCaret accepts a caret at the very end", "[caret_context]") { + auto r = beforeCaret("abc", 3, "c"); + REQUIRE(r.has_value()); + REQUIRE(*r == "abc"); +} + +TEST_CASE("beforeCaret rejects a caret past the end", "[caret_context]") { + REQUIRE_FALSE(beforeCaret("abc", 4, "c").has_value()); +} + +TEST_CASE("beforeCaret rejects text missing the last keystroke", "[caret_context]") { + // App still reports the text from before the user typed "x". + REQUIRE_FALSE(beforeCaret("abc", 3, "x").has_value()); + REQUIRE_FALSE(beforeCaret("", 0, "x").has_value()); +} + +TEST_CASE("lastChar returns the final code point", "[caret_context]") { + REQUIRE(lastChar("") == ""); + REQUIRE(lastChar("ab") == "b"); + REQUIRE(lastChar("aç") == "ç"); + REQUIRE(lastChar("€") == "€"); +} + +TEST_CASE("trimLeadingBlanksAfter drops the space after a trailing space", "[caret_context]") { + std::string s = " relevant suggestions"; + trimLeadingBlanksAfter(s, "suggest "); + REQUIRE(s == "relevant suggestions"); + + std::string t = " \tnext"; + trimLeadingBlanksAfter(t, "line\n"); + REQUIRE(t == "next"); +} + +TEST_CASE("trimLeadingBlanksAfter keeps spacing when the context ends in a word", "[caret_context]") { + std::string s = " relevant"; + trimLeadingBlanksAfter(s, "are not"); + REQUIRE(s == " relevant"); + + std::string mid = "ustrated"; + trimLeadingBlanksAfter(mid, "i feel f"); + REQUIRE(mid == "ustrated"); +} + +TEST_CASE("trimLeadingBlanksAfter can empty an all-blank token", "[caret_context]") { + std::string s = " "; + trimLeadingBlanksAfter(s, "word "); + REQUIRE(s.empty()); +} + +TEST_CASE("promptFor drops trailing spaces and tabs only", "[caret_context]") { + REQUIRE(promptFor("out of the ") == "out of the"); + REQUIRE(promptFor("a \t ") == "a"); + REQUIRE(promptFor("Best regards,\n") == "Best regards,\n"); + REQUIRE(promptFor("environ") == "environ"); + REQUIRE(promptFor(" ") == ""); +} + +TEST_CASE("withoutPromptEcho cuts a repeated prompt", "[caret_context]") { + // Qwen3 in Ollama returns the prefill, then the continuation. + REQUIRE(withoutPromptEcho("Best regards,\nEwerton", "Best regards,\n") == "Ewerton"); + REQUIRE(withoutPromptEcho("when it expired", "when it") == " expired"); +} + +TEST_CASE("withoutPromptEcho hides a partial echo while it streams", "[caret_context]") { + REQUIRE(withoutPromptEcho("Best reg", "Best regards,\n") == ""); + REQUIRE(withoutPromptEcho("Best regards,\n", "Best regards,\n") == ""); +} + +TEST_CASE("withoutPromptEcho leaves a plain continuation alone", "[caret_context]") { + REQUIRE(withoutPromptEcho(" office", "I was out of the") == " office"); + REQUIRE(withoutPromptEcho("ment", "staging environ") == "ment"); + REQUIRE(withoutPromptEcho("anything", "") == "anything"); +} diff --git a/fcitx5-engine/tests/test_ollama_request.cpp b/fcitx5-engine/tests/test_ollama_request.cpp new file mode 100644 index 0000000..19f2101 --- /dev/null +++ b/fcitx5-engine/tests/test_ollama_request.cpp @@ -0,0 +1,38 @@ +#include + +#include + +#include "ollama_request.h" + +static bool contains(const std::string& s, const std::string& needle) { + return s.find(needle) != std::string::npos; +} + +TEST_CASE("payload has the system prompt and the context as assistant prefill", "[ollama_request]") { + InferenceConfig cfg; + cfg.model = "qwen2.5:3b"; + auto body = buildChatPayload(cfg, "I feel the", "Be an autocomplete engine."); + + CHECK(contains(body, R"("model":"qwen2.5:3b")")); + CHECK(contains(body, R"({"role":"system","content":"Be an autocomplete engine."})")); + CHECK(contains(body, R"({"role":"assistant","content":"I feel the"})")); + CHECK(contains(body, R"("stream":true)")); +} + +TEST_CASE("payload carries the sampling options", "[ollama_request]") { + InferenceConfig cfg; + cfg.num_predict = 12; + cfg.temperature = 0.3f; + cfg.top_p = 0.9f; + cfg.stop_tokens = {".", "\n"}; + cfg.threads = 4; + auto body = buildChatPayload(cfg, "x", ""); + + CHECK(contains(body, R"("options":{"num_predict":12,"temperature":0.3,"top_p":0.9,"stop":[".","\n"],"num_thread":4})")); +} + +TEST_CASE("context is JSON-escaped", "[ollama_request]") { + InferenceConfig cfg; + auto body = buildChatPayload(cfg, "say \"hi\"\nC:\\path\t", ""); + CHECK(contains(body, R"({"role":"assistant","content":"say \"hi\"\nC:\\path\t"})")); +} From 09e4386a1ca42c86c4328faf1400eaf314e94e2e Mon Sep 17 00:00:00 2001 From: Ewerton Mattos Date: Thu, 15 Oct 2026 18:51:27 -0300 Subject: [PATCH 2/3] fix(settings): label the context window in characters The engine counts inference.context_window in characters, but the Model page called it tokens and the slider started at 512. Say characters and allow 100-4000. Co-Authored-By: Claude Opus 5.5 --- settings/qt6-app/qml/pages/PageModel.qml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/settings/qt6-app/qml/pages/PageModel.qml b/settings/qt6-app/qml/pages/PageModel.qml index 549a0d2..067de86 100644 --- a/settings/qt6-app/qml/pages/PageModel.qml +++ b/settings/qt6-app/qml/pages/PageModel.qml @@ -105,10 +105,10 @@ Item { XRow { label: "Context window" - desc: "Tokens of context fed to the model" + desc: "Characters of text before the caret fed to the model" width: parent.width XSlider { - min: 512; max: 4096; step: 256 + min: 100; max: 4000; step: 100 value: Config.contextWindow onCommitted: (v) => Config.contextWindow = v width: 200 From e0ea4763f3f9cf89b763930ef987f5e04f4ec1a0 Mon Sep 17 00:00:00 2001 From: Ewerton Mattos Date: Thu, 15 Oct 2026 18:51:27 -0300 Subject: [PATCH 3/3] test(scripts): model benchmark that mirrors the engine's requests benchmark_models.py sends each model the engine's exact /api/chat request, applies its output clean-up and reports first-word accuracy and latency. build_bench_cases.py turns public text (AESLC e-mails, Tatoeba EN/PT) into cut-sentence test cases. Co-Authored-By: Claude Opus 5.5 --- scripts/benchmark_models.py | 299 +++++++++++++++++++++++++++++++++++ scripts/build_bench_cases.py | 151 ++++++++++++++++++ 2 files changed, 450 insertions(+) create mode 100644 scripts/benchmark_models.py create mode 100644 scripts/build_bench_cases.py diff --git a/scripts/benchmark_models.py b/scripts/benchmark_models.py new file mode 100644 index 0000000..b6417b7 --- /dev/null +++ b/scripts/benchmark_models.py @@ -0,0 +1,299 @@ +#!/usr/bin/env python3 +"""Compare Ollama models as XType autocomplete backends. + +Sends each model the same request the fcitx5 engine sends — /api/chat with +the system prompt and the text as an unfinished assistant turn, trailing +blanks dropped — then cleans the output the way +the engine does (prompt-echo cut, leading-blank trim, echo strip, trailing +trim) and scores it against the known continuation. --legacy reproduces the +engine before those two fixes (prompt sent as typed, no prompt-echo cut). + +Cases: the built-in set below, plus --cases FILE.jsonl +({"category", "context", "expected"} per line; see build_bench_cases.py). + +Usage: + python3 scripts/benchmark_models.py qwen2.5:3b qwen3:4b-instruct-2507-q4_K_M + python3 scripts/benchmark_models.py --cases /tmp/cases.jsonl --out /tmp/bench.json MODEL ... + +Stdlib only. +""" + +import argparse +import json +import re +import statistics +import time +import urllib.request + +HOST = "http://localhost:11434" + +# Mirrors kBaseSystemPrompt + "About the user:" in the engine (prompt_builder.cpp). +BASE_PROMPT = ( + "You are an inline text autocomplete engine. Continue the text you are given with a " + "few natural words. Output ONLY the continuation. No explanations, no responses, " + "no punctuation at the start." +) +DEFAULT_DESCRIPTION = "Senior software engineer who writes in Brazilian portuguese and English" + +OPTIONS = {"num_predict": 30, "temperature": 0.3, "top_p": 0.9, "stop": [".", "!", "?", "\n"]} + +# (category, context, expected continuation). Expected text is what a human +# would most likely type next; scoring compares leading words. +CASES = [ + # English — chat / email + ("en-chat", "Thanks for the quick reply. I'll take a look at it and get back to you as soon as ", "possible"), + ("en-chat", "Sorry for the late reply, I was out of the ", "office"), + ("en-chat", "The meeting has been moved from Tuesday to Wednesday at 3 pm, so please update your ", "calendar"), + ("en-chat", "Hi Maria,\nI hope you are doing well. I'm writing to follow ", "up on"), + ("en-chat", "No worries, take your time. Let me know if you have any ", "questions"), + # English — engineering + ("en-dev", "Can you send me the link to the pull ", "request"), + ("en-dev", "Before merging, please make sure all the unit tests are ", "passing"), + ("en-dev", "The request fails intermittently, so we should add a retry with exponential ", "backoff"), + ("en-dev", "I fixed the bug in the login flow: the session token was not being refreshed when it ", "expired"), + ("en-dev", "I feel the suggestions this app gives are not that useful, it seems like the suggestions are not related to the text being ", "typed"), + # English — mid-word + ("en-midword", "I'll deploy the new version to the staging environ", "ment"), + ("en-midword", "Please review the documentation before the next relea", "se"), + # English — long context, needs something from earlier in the text + ("en-long", + "Hi team,\nQuick update on the payments migration. We are moving the settlement service from Azure " + "Functions to Azure Container Apps, because the cold starts on Functions were adding up to two seconds " + "to every payout request. The new deployment has been running in staging since Monday and the p95 " + "latency dropped from 2.4 seconds to 180 milliseconds. Next week we will start routing 10 percent of " + "production traffic to it. The main reason we chose Container ", "Apps"), + ("en-long", + "Hello Daniel,\nMy name is Ewerton and I'm a senior software engineer at Lumon Pay. I'm reaching out " + "because we are evaluating vendors for card tokenization and your platform came up several times in our " + "research. Would you have 30 minutes next week for a quick call to walk us through the API and pricing? " + "Any day after 2 pm works for me.\nBest regards,\n", "Ewerton"), + # Portuguese (BR) + ("pt-chat", "Oi, tudo bem? Você consegue me mandar o relatório até o final do ", "dia"), + ("pt-chat", "Obrigado pela ajuda, fico no ", "aguardo"), + ("pt-chat", "Desculpa a demora para responder, estava em uma ", "reunião"), + ("pt-chat", "Qualquer dúvida, estou à ", "disposição"), + ("pt-chat", "Vamos marcar uma call amanhã de manhã para alinhar os próximos ", "passos"), + ("pt-dev", "Esse problema ainda está acontecendo em produção, preciso que alguém olhe os ", "logs"), + ("pt-midword", "A gente precisa resolver esse problema o mais rápido possí", "vel"), + ("pt-long", + "Olá Carlos, tudo bem?\nAqui é o Ewerton, da Lumon Pay. Estou entrando em contato porque estamos " + "migrando o serviço de pagamentos para o Azure e gostaríamos de entender melhor como funciona o suporte " + "de vocês para clientes enterprise. Você teria disponibilidade para uma conversa rápida na quinta-feira " + "à tarde?\nUm abraço,\n", "Ewerton"), +] + + +# ── Engine-equivalent request + clean-up ───────────────────────────────────── + +def payload(model, context, system_prompt): + return {"model": model, "stream": True, "options": OPTIONS, + "messages": [{"role": "system", "content": system_prompt}, + {"role": "assistant", "content": context}]} + + +def ends_blank(ctx): + return bool(ctx) and ctx[-1] in " \t\n" + + +def strip_echo_prefix(s, sent): + """caret_context::withoutPromptEcho: drop a verbatim re-emission of the + prompt (some chat templates return the assistant prefill in front).""" + if not sent: + return s + if s.startswith(sent): + return s[len(sent):] + if sent.startswith(s): # still streaming the echo + return "" + return s + + +def finalize(s, ctx, sent=None): + """xtype.cpp done-handler: trailing trim, tail/head echo strip, leading trim. + With `sent`, a full echo of the prompt is stripped first.""" + if sent is not None: + s = strip_echo_prefix(s, sent) + s = s.rstrip() + for n in range(min(len(s), len(ctx), 80), 3, -1): + if ctx.endswith(s[:n]): + s = s[n:] + break + if len(s) >= 12 and len(ctx) >= 12 and s[:12].lower() == ctx[:12].lower(): + s = "" + if ends_blank(ctx): + s = s.lstrip(" \t") + return s + + +def run_one(model, context, system_prompt, fixes=True): + # caret_context::promptFor: don't end the prompt on a bare space — + # tokenizers attach the space to the next word, so a trailing one skews + # the model toward digits and word fragments. The real context still + # drives the clean-up. + sent = context.rstrip(" \t") if fixes else context + body = payload(model, sent, system_prompt) + req = urllib.request.Request(HOST + "/api/chat", data=json.dumps(body).encode(), + headers={"Content-Type": "application/json"}) + t0 = time.perf_counter() + ttft = None + text = "" + final = {} + with urllib.request.urlopen(req, timeout=120) as resp: + for line in resp: + line = line.strip() + if not line: + continue + d = json.loads(line) + tok = (d.get("message") or {}).get("content") + if tok: + # Ghost text appears once the (trimmed) suggestion is non-empty. + shown = text + tok + visible = strip_echo_prefix(shown, sent) if fixes else shown + if ttft is None and (visible.lstrip(" \t") if ends_blank(context) else visible): + ttft = (time.perf_counter() - t0) * 1000 + text = shown + if d.get("done"): + final = d + break + total = (time.perf_counter() - t0) * 1000 + return { + "raw": text, + "suggestion": finalize(text, context, sent if fixes else None), + "ttft_ms": ttft if ttft is not None else total, + "total_ms": total, + "prompt_tokens": final.get("prompt_eval_count", 0), + "prompt_ms": final.get("prompt_eval_duration", 0) / 1e6, + "gen_tokens": final.get("eval_count", 0), + "gen_ms": final.get("eval_duration", 0) / 1e6, + } + + +# ── Scoring ────────────────────────────────────────────────────────────────── + +def words(s): + return [w for w in re.split(r"\s+", re.sub(r"[^\w\s]", " ", s.lower())) if w] + + +def score(suggestion, expected): + # Mid-word cases expect the rest of the word ("environ" → "ment"), so the + # same leading-word comparison covers both. + sw, ew = words(suggestion), words(expected) + first_ok = bool(sw) and sw[0] == ew[0] + matched = 0 + for a, b in zip(sw, ew): + if a != b: + break + matched += 1 + return {"first_word": first_ok, "words_matched": matched, "empty": not suggestion.strip()} + + +def pct(xs, p): + xs = sorted(xs) + return xs[max(0, min(len(xs) - 1, round(p / 100 * (len(xs) - 1))))] + + +def loaded_info(model): + try: + with urllib.request.urlopen(HOST + "/api/ps", timeout=5) as r: + for m in json.load(r).get("models", []): + if m.get("name") == model or m.get("model") == model: + size, vram = m.get("size", 0), m.get("size_vram", 0) + return {"size_gb": size / 1e9, "gpu_pct": round(100 * vram / size) if size else 0} + except Exception: + pass + return {} + + +def unload(model): + body = json.dumps({"model": model, "keep_alive": 0}).encode() + req = urllib.request.Request(HOST + "/api/generate", data=body, + headers={"Content-Type": "application/json"}) + try: + urllib.request.urlopen(req, timeout=60).read() + except Exception: + pass + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("models", nargs="+", help="Ollama model names") + ap.add_argument("--runs", type=int, default=1, help="runs per case (default 1)") + ap.add_argument("--cases", help="extra cases, JSONL") + ap.add_argument("--no-builtin", action="store_true", help="skip the built-in cases") + ap.add_argument("--description", default=DEFAULT_DESCRIPTION) + ap.add_argument("--out", default="/tmp/xtype-bench.json") + ap.add_argument("--legacy", action="store_true", + help="engine before the fixes: prompt as typed, no prompt-echo cut") + args = ap.parse_args() + + cases = [] if args.no_builtin else [{"category": c, "context": x, "expected": e} + for c, x, e in CASES] + if args.cases: + with open(args.cases, encoding="utf-8") as f: + cases += [json.loads(line) for line in f if line.strip()] + print(f"{len(cases)} cases, {args.runs} run(s) each", flush=True) + + system_prompt = BASE_PROMPT + ("\n\nAbout the user: " + args.description if args.description else "") + results = {} + + for model in args.models: + print(f"\n=== {model} ===", flush=True) + + t0 = time.perf_counter() + run_one(model, "Hello, this is a warm-up request to load the ", system_prompt) + load_s = time.perf_counter() - t0 + info = loaded_info(model) + print(f" load+first request: {load_s:.1f}s {info}", flush=True) + + rows = [] + t_model = time.perf_counter() + for i, case in enumerate(cases, 1): + cat, ctx, exp = case["category"], case["context"], case["expected"] + for run in range(args.runs): + try: + r = run_one(model, ctx, system_prompt, not args.legacy) + except Exception as e: # keep going; count as an empty suggestion + r = {"raw": "", "suggestion": "", "ttft_ms": 0, "total_ms": 0, "error": str(e)} + r.update(score(r["suggestion"], exp), category=cat, run=run, + context_tail=ctx[-45:], expected=exp) + rows.append(r) + if i % 100 == 0 or i == len(cases): + acc = 100 * sum(x["first_word"] for x in rows) / len(rows) + print(f" {i}/{len(cases)} first-word {acc:.0f}% " + f"({time.perf_counter() - t_model:.0f}s)", flush=True) + + results[model] = {"model": model, "load_s": load_s, **info, "rows": rows} + unload(model) + with open(args.out, "w") as f: # after each model, so a crash keeps the rest + json.dump(results, f, ensure_ascii=False, indent=1) + + # ── Summary ── + print("\n" + "=" * 112) + print(f"{'model':46} {'1st-word':>8} {'words':>6} {'empty':>6} " + f"{'TTFT p50':>9} {'p90':>6} {'total p50':>10} {'p90':>6} {'GPU':>5}") + print("-" * 112) + for res in results.values(): + rows = res["rows"] + first = 100 * sum(r["first_word"] for r in rows) / len(rows) + wm = statistics.mean(r["words_matched"] for r in rows) + empty = 100 * sum(r["empty"] for r in rows) / len(rows) + # Latency from each case's first run only: repeats hit Ollama's prompt cache. + tt = [r["ttft_ms"] for r in rows if r["run"] == 0] + tot = [r["total_ms"] for r in rows if r["run"] == 0] + print(f"{res['model'][:46]:46} {first:7.0f}% {wm:6.2f} {empty:5.0f}% " + f"{pct(tt, 50):7.0f}ms {pct(tt, 90):5.0f} {pct(tot, 50):8.0f}ms {pct(tot, 90):5.0f} " + f"{str(res.get('gpu_pct', '?')) + '%':>5}") + + print("\nFirst-word accuracy by category:") + cats = sorted({c["category"] for c in cases}) + print(f"{'model':46} " + " ".join(f"{c[:16]:>16}" for c in cats)) + for res in results.values(): + line = f"{res['model'][:46]:46} " + for c in cats: + rs = [r for r in res["rows"] if r["category"] == c] + line += f"{100 * sum(r['first_word'] for r in rs) / len(rs):15.0f}% " + print(line) + print(f"\nFull results: {args.out}") + + +if __name__ == "__main__": + main() diff --git a/scripts/build_bench_cases.py b/scripts/build_bench_cases.py new file mode 100644 index 0000000..9f7f611 --- /dev/null +++ b/scripts/build_bench_cases.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python3 +"""Build autocomplete test cases for benchmark_models.py from public text. + +Sources (downloaded into --data-dir): + en-email AESLC (Enron e-mail bodies) via the Hugging Face datasets-server + en-short Tatoeba English sentences + pt-short Tatoeba Portuguese sentences + +Each case cuts a real sentence at a word boundary (or, for ~15%, inside a +word): the text before the cut is the context, the rest of the sentence up to +. ! ? or a newline is the expected continuation. E-mail cases keep the +preceding e-mail text as context (last --context-chars characters). + +Usage: + python3 -I scripts/build_bench_cases.py --data-dir /tmp/xtype-bench-data \\ + --out /tmp/xtype-bench-cases.jsonl +""" + +import argparse +import bz2 +import csv +import json +import os +import random +import re +import sys +import urllib.request + +TATOEBA = "https://downloads.tatoeba.org/exports/per_language/{lang}/{lang}_sentences.tsv.bz2" +AESLC = ("https://datasets-server.huggingface.co/rows?dataset=Yale-LILY/aeslc" + "&config=default&split=train&offset={offset}&length=100") + +csv.field_size_limit(sys.maxsize) + + +def fetch(url, path): + if not os.path.exists(path): + print(f" downloading {url}", file=sys.stderr) + with urllib.request.urlopen(url, timeout=120) as r, open(path, "wb") as f: + f.write(r.read()) + return path + + +def tatoeba(lang, data_dir): + path = fetch(TATOEBA.format(lang=lang), os.path.join(data_dir, f"{lang}_sentences.tsv.bz2")) + with bz2.open(path, "rt", encoding="utf-8") as f: + for row in csv.reader(f, delimiter="\t", quoting=csv.QUOTE_NONE): + if len(row) >= 3: + yield row[2] + + +def aeslc(data_dir, pages): + for p in range(pages): + path = fetch(AESLC.format(offset=p * 100), os.path.join(data_dir, f"aeslc_{p:03}.json")) + with open(path, encoding="utf-8") as f: + for r in json.load(f).get("rows", []): + yield r["row"]["email_body"] + + +SENT_END = re.compile(r"[.!?\n]") +BAD = re.compile(r"https?://|www\.|@|-----|\b(From|To|Cc|Subject|Sent):|[<>{}|=_*#]") + + +def make_case(rng, prefix_text, sentence, category, context_chars, mid_word_rate=0.15): + """Cut `sentence` (which follows prefix_text) and return a case or None.""" + words = list(re.finditer(r"\S+", sentence)) + if len(words) < 4: + return None + if rng.random() < mid_word_rate: + # Inside a word of >= 5 letters, keeping >= 2 letters on each side. + cands = [w for w in words[1:] if re.fullmatch(r"\w{5,}", w.group())] + if not cands: + return None + w = rng.choice(cands) + cut = w.start() + rng.randint(2, len(w.group()) - 2) + kind = "midword" + else: + # Before word k (k >= 2): the context ends with the space after word k-1. + k = rng.randint(2, len(words) - 1) + cut = words[k].start() + kind = "word" + context = prefix_text + sentence[:cut] + m = SENT_END.search(sentence, cut) + expected = sentence[cut:m.start() if m else len(sentence)].strip() + if not re.search(r"\w", expected) or len(context.strip()) < 10: + return None + return {"category": f"{category}-{kind}", "context": context[-context_chars:], "expected": expected} + + +def short_cases(rng, sentences, category, n, context_chars): + pool = [s for s in sentences if 25 <= len(s) <= 160 and not BAD.search(s)] + rng.shuffle(pool) + out = [] + for s in pool: + c = make_case(rng, "", s, category, context_chars) + if c: + out.append(c) + if len(out) >= n: + break + return out + + +def email_cases(rng, bodies, n, context_chars): + out = [] + bodies = [b for b in bodies if 200 <= len(b) <= 4000] + rng.shuffle(bodies) + for body in bodies: + body = re.sub(r"[ \t]+", " ", body) + # Sentences (or lines) that look like prose. + spans = [m for m in re.finditer(r"[^.!?\n]+[.!?]?", body) + if 30 <= len(m.group().strip()) <= 220 and not BAD.search(m.group())] + if not spans: + continue + m = rng.choice(spans) + start = m.start() + (len(m.group()) - len(m.group().lstrip())) + c = make_case(rng, body[:start], body[start:m.end()], "en-email", context_chars) + if c: + out.append(c) + if len(out) >= n: + break + return out + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--data-dir", required=True) + ap.add_argument("--out", required=True) + ap.add_argument("--seed", type=int, default=7) + ap.add_argument("--emails", type=int, default=450) + ap.add_argument("--en", type=int, default=300) + ap.add_argument("--pt", type=int, default=450) + ap.add_argument("--context-chars", type=int, default=1000, help="engine context_window") + args = ap.parse_args() + os.makedirs(args.data_dir, exist_ok=True) + rng = random.Random(args.seed) + + cases = email_cases(rng, list(aeslc(args.data_dir, pages=12)), args.emails, args.context_chars) + cases += short_cases(rng, tatoeba("eng", args.data_dir), "en-short", args.en, args.context_chars) + cases += short_cases(rng, tatoeba("por", args.data_dir), "pt-short", args.pt, args.context_chars) + + with open(args.out, "w", encoding="utf-8") as f: + for c in cases: + f.write(json.dumps(c, ensure_ascii=False) + "\n") + by = {} + for c in cases: + by[c["category"]] = by.get(c["category"], 0) + 1 + print(f"{len(cases)} cases → {args.out} {by}", file=sys.stderr) + + +if __name__ == "__main__": + main()