diff --git a/fcitx5-engine/CMakeLists.txt b/fcitx5-engine/CMakeLists.txt index 8326f18..75d7888 100644 --- a/fcitx5-engine/CMakeLists.txt +++ b/fcitx5-engine/CMakeLists.txt @@ -5,7 +5,7 @@ set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_EXPORT_COMPILE_COMMANDS ON) -find_package(Fcitx5Core 5.0 REQUIRED) +find_package(Fcitx5Core 5.0.4 REQUIRED) # 5.0.4: Instance::processComposeString find_package(CURL REQUIRED) option(BUILD_TESTS "Build unit tests" OFF) diff --git a/fcitx5-engine/src/context_buffer.cpp b/fcitx5-engine/src/context_buffer.cpp index 3c87ed2..3bbbf51 100644 --- a/fcitx5-engine/src/context_buffer.cpp +++ b/fcitx5-engine/src/context_buffer.cpp @@ -1,11 +1,17 @@ #include "context_buffer.h" +#include "utf8_util.h" + // ── Private ─────────────────────────────────────────────────────────────────── void ContextBuffer::push(char ch) { - if (_buf.size() >= MAX_CHARS) - _buf.pop_front(); _buf.push_back(ch); + if (_buf.size() > MAX_CHARS) { + _buf.pop_front(); + // Never leave the tail bytes of a multi-byte character at the front. + while (!_buf.empty() && utf8_util::isContinuation(_buf.front())) + _buf.pop_front(); + } } // ── Queries ─────────────────────────────────────────────────────────────────── @@ -29,11 +35,20 @@ void ContextBuffer::appendChar(char ch) { push(ch); } +void ContextBuffer::appendText(std::string_view utf8) { + dismiss(); + for (char ch : utf8) + push(ch); +} + void ContextBuffer::backspace() { if (_suggestion) { dismiss(); - } else if (!_buf.empty()) { - _buf.pop_back(); + } else { + while (!_buf.empty() && utf8_util::isContinuation(_buf.back())) + _buf.pop_back(); + if (!_buf.empty()) + _buf.pop_back(); } } diff --git a/fcitx5-engine/src/context_buffer.h b/fcitx5-engine/src/context_buffer.h index 0e372be..f1813b2 100644 --- a/fcitx5-engine/src/context_buffer.h +++ b/fcitx5-engine/src/context_buffer.h @@ -3,11 +3,13 @@ #include #include #include +#include class ContextBuffer { public: void appendChar(char ch); - void backspace(); + void appendText(std::string_view utf8); // a typed character or composed sequence + void backspace(); // removes one code point void setSuggestion(std::string text); std::string acceptNextWord(); std::string acceptAll(); @@ -19,7 +21,7 @@ class ContextBuffer { bool hasSuggestion() const; private: - void push(char ch); // rolling-window insert: drops oldest when at MAX_CHARS + void push(char ch); // rolling-window insert: drops oldest code point past MAX_CHARS bytes std::deque _buf; std::optional _suggestion; diff --git a/fcitx5-engine/src/utf8_util.h b/fcitx5-engine/src/utf8_util.h new file mode 100644 index 0000000..6b534fd --- /dev/null +++ b/fcitx5-engine/src/utf8_util.h @@ -0,0 +1,29 @@ +#pragma once + +#include + +// Minimal UTF-8 helpers for the typed-text buffers. The engine stores what the +// user typed as UTF-8 bytes; anything that drops or slices bytes must stay on +// code-point boundaries, or the text sent to Ollama / written to JSON becomes +// invalid UTF-8. +namespace utf8_util { + +inline bool isContinuation(char c) { + return (static_cast(c) & 0xC0) == 0x80; +} + +// Removes the last code point (not just the last byte). +inline void popBack(std::string &s) { + while (!s.empty() && isContinuation(s.back())) s.pop_back(); + if (!s.empty()) s.pop_back(); +} + +// The last <= maxBytes bytes of s, starting on a code-point boundary. +inline std::string tail(const std::string &s, size_t maxBytes) { + if (s.size() <= maxBytes) return s; + size_t start = s.size() - maxBytes; + while (start < s.size() && isContinuation(s[start])) ++start; + return s.substr(start); +} + +} // namespace utf8_util diff --git a/fcitx5-engine/src/xtype.cpp b/fcitx5-engine/src/xtype.cpp index 3447540..4ab573d 100644 --- a/fcitx5-engine/src/xtype.cpp +++ b/fcitx5-engine/src/xtype.cpp @@ -26,6 +26,7 @@ #include "config_loader.h" #include "path_utils.h" #include "prompt_builder.h" +#include "utf8_util.h" // ── Helpers ─────────────────────────────────────────────────────────────────── @@ -48,6 +49,14 @@ static void dbg(const char *fmt, ...) { va_list ap; va_start(ap, fmt); std::vfprintf(f, fmt, ap); va_end(ap); std::fputc('\n', f); std::fflush(f); } +// UTF-8 text a key types on its own, or "" for non-text keys (arrows, F-keys, +// modifiers, dead keys) and control characters. +static std::string typedText(fcitx::KeySym sym) { + const uint32_t uc = fcitx::Key::keySymToUnicode(sym); + if (uc < 0x20 || uc == 0x7f) return {}; + return fcitx::Key::keySymToUTF8(sym); +} + static void log_warn(const char *msg) { std::fprintf(stderr, "[xtype] WARNING: %s\n", msg); dbg("WARN: %s", msg); @@ -246,6 +255,16 @@ void XTypeEngine::keyEvent(const fcitx::InputMethodEntry &, const bool hasSuggestion = _ctx.hasSuggestion(); const auto sym = event.key().sym(); + // These keys cancel a pending dead-key / Compose sequence in the app; keep + // our mirror of that state (see the text branch below) in step. A cancelled + // sequence leaves the text untouched, so Backspace deletes nothing. + if (_instance->isComposing(ic) && + (sym == FcitxKey_Escape || sym == FcitxKey_Return || + sym == FcitxKey_KP_Enter || sym == FcitxKey_BackSpace)) { + _instance->resetCompose(ic); + if (sym == FcitxKey_BackSpace) return; + } + // Tab — accept next word; Shift+Tab / ISO_Left_Tab — accept entire suggestion. if (sym == FcitxKey_Tab || sym == FcitxKey_ISO_Left_Tab) { if (!hasSuggestion) return; @@ -299,8 +318,7 @@ void XTypeEngine::keyEvent(const fcitx::InputMethodEntry &, event.filterAndAccept(); } else { _ctx.backspace(); - if (!_userTypedSinceLastTerminator.empty()) - _userTypedSinceLastTerminator.pop_back(); + utf8_util::popBack(_userTypedSinceLastTerminator); _debounceTimer.reset(); } return; @@ -323,15 +341,36 @@ void XTypeEngine::keyEvent(const fcitx::InputMethodEntry &, return; } - // Printable ASCII (space through ~). Pass key through to app, update buffer, - // arm debounce timer for inference. - if (sym >= FcitxKey_space && sym <= FcitxKey_asciitilde) { - dbg("key '%c' prog=%s ctx_len=%zu", static_cast(sym), + // Text keys. The key always passes through and the app inserts the text + // itself — including dead-key / Compose sequences, which the app composes + // on its side (US-International: ' then space -> ', ' then a -> á). Feed + // the same keys through Fcitx5's XCompose state so the buffer records the + // text the app shows rather than raw keysyms. + std::string text; + if (auto composed = _instance->processComposeString(ic, sym)) { + if (!composed->empty()) { + text = std::move(*composed); // sequence completed + } else if (_instance->isComposing(ic)) { + // Mid-sequence (e.g. a dead key): nothing inserted yet. Drop the + // ghost text so it can't sit on top of the app's compose preedit. + invalidate(); + updatePreedit(ic); + _debounceTimer.reset(); + return; + } else { + text = typedText(sym); // not part of a sequence + } + } else { + // Sequence cancelled (no rule for dead key + this key). Toolkits differ + // on what they insert; at least the key itself is the common case. + text = typedText(sym); + } + if (!text.empty()) { + dbg("key '%s' prog=%s ctx_len=%zu", text.c_str(), ic->program().c_str(), _ctx.contextText().size()); invalidate(); updatePreedit(ic); - const char ch = static_cast(sym); - _ctx.appendChar(ch); + _ctx.appendText(text); // User-typed-only buffer for corpus harvest (excludes AI accept paths). // Rolling strategy: on overflow, erase through the next sentence terminator @@ -344,8 +383,8 @@ void XTypeEngine::keyEvent(const fcitx::InputMethodEntry &, else buf.clear(); } - _userTypedSinceLastTerminator.push_back(ch); - if (ch == '.' || ch == '!' || ch == '?') + _userTypedSinceLastTerminator += text; + if (text == "." || text == "!" || text == "?") harvestSentence(ic->program()); _debounceTimer.reset(); @@ -379,8 +418,7 @@ void XTypeEngine::requestInference( if (static_cast(ctx.size()) < _cfg.inference.min_context_chars) return; if (static_cast(ctx.size()) > _cfg.inference.context_window) - ctx = ctx.substr(ctx.size() - - static_cast(_cfg.inference.context_window)); + ctx = utf8_util::tail(ctx, static_cast(_cfg.inference.context_window)); ++_gen; ++_metrics.suggestions_generated; @@ -453,8 +491,7 @@ void XTypeEngine::requestInference( if (_ctx.hasSuggestion()) { RecentEvent ev; const std::string ctxText = _ctx.contextText(); - ev.typed = ctxText.substr( - static_cast(std::max(0, (int)ctxText.size() - 40))); + ev.typed = utf8_util::tail(ctxText, 40); ev.ghost = _ctx.suggestion() ? *_ctx.suggestion() : ""; ev.app = icPtr->program(); ev.accepted = false; diff --git a/fcitx5-engine/tests/CMakeLists.txt b/fcitx5-engine/tests/CMakeLists.txt index 4d9d53f..65b7bff 100644 --- a/fcitx5-engine/tests/CMakeLists.txt +++ b/fcitx5-engine/tests/CMakeLists.txt @@ -54,6 +54,13 @@ target_link_libraries(test_phrase_blocklist PRIVATE ) catch_discover_tests(test_phrase_blocklist) +add_executable(test_utf8_util test_utf8_util.cpp) +target_include_directories(test_utf8_util PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../src) +target_link_libraries(test_utf8_util PRIVATE + Catch2::Catch2WithMain +) +catch_discover_tests(test_utf8_util) + add_executable(test_xtype_helpers test_xtype_helpers.cpp) target_link_libraries(test_xtype_helpers PRIVATE Catch2::Catch2WithMain diff --git a/fcitx5-engine/tests/test_context_buffer.cpp b/fcitx5-engine/tests/test_context_buffer.cpp index 39bdc7a..a4f35c1 100644 --- a/fcitx5-engine/tests/test_context_buffer.cpp +++ b/fcitx5-engine/tests/test_context_buffer.cpp @@ -273,3 +273,40 @@ TEST_CASE("reset allows fresh use", "[reset]") { REQUIRE(cb.contextText() == "x"); REQUIRE_FALSE(cb.hasSuggestion()); } + +// ── UTF-8 ───────────────────────────────────────────────────────────────────── + +TEST_CASE("appendText stores multi-byte characters", "[utf8]") { + ContextBuffer cb; + cb.appendText("não"); + cb.appendText("'"); + REQUIRE(cb.contextText() == "não'"); +} + +TEST_CASE("appendText dismisses active suggestion", "[utf8]") { + ContextBuffer cb; + cb.setSuggestion("hello"); + cb.appendText("ç"); + REQUIRE_FALSE(cb.hasSuggestion()); +} + +TEST_CASE("backspace removes a whole multi-byte character", "[utf8][backspace]") { + ContextBuffer cb; + cb.appendText("café"); + cb.backspace(); + REQUIRE(cb.contextText() == "caf"); + cb.appendText("€"); + cb.backspace(); + REQUIRE(cb.contextText() == "caf"); +} + +TEST_CASE("rolling window never starts with a partial character", "[utf8][window]") { + ContextBuffer cb; + cb.appendChar('x'); // shifts the 2-byte chars off even offsets + for (int i = 0; i < 400; ++i) cb.appendText("ã"); + const std::string t = cb.contextText(); + REQUIRE(t.size() <= 500); + REQUIRE(t.size() >= 499); + REQUIRE((static_cast(t.front()) & 0xC0) != 0x80); + REQUIRE(t.substr(0, 2) == "ã"); +} diff --git a/fcitx5-engine/tests/test_style_profile.cpp b/fcitx5-engine/tests/test_style_profile.cpp index f563788..403fd64 100644 --- a/fcitx5-engine/tests/test_style_profile.cpp +++ b/fcitx5-engine/tests/test_style_profile.cpp @@ -206,6 +206,26 @@ TEST_CASE("serialize/deserialize round-trip preserves all fields", "[style][json fs::remove_all(d); } +TEST_CASE("serialize/deserialize round-trip preserves UTF-8 text", "[style][json][utf8]") { + auto d = uniqueDir("utf8"); + auto cp = d / "corpus.txt"; + writeCorpus(cp, { + "não sei se a configuração está certa agora", + "você já viu a nova versão do aplicativo", + "I'm sure it works with “quotes” and ç too", + }); + StyleProfile a; + a.loadFromCorpus(cp.string(), 7); + REQUIRE_FALSE(a.exemplars().empty()); + + auto jp = d / "style.json"; + REQUIRE(a.serialize(jp.string())); + StyleProfile b = StyleProfile::deserialize(jp.string()); + REQUIRE(b.exemplars() == a.exemplars()); + REQUIRE(b.commonOpeners() == a.commonOpeners()); + fs::remove_all(d); +} + TEST_CASE("corrupt JSON deserialize is safe", "[style][json]") { auto d = uniqueDir("corrupt"); auto jp = d / "style.json"; diff --git a/fcitx5-engine/tests/test_utf8_util.cpp b/fcitx5-engine/tests/test_utf8_util.cpp new file mode 100644 index 0000000..0e7a786 --- /dev/null +++ b/fcitx5-engine/tests/test_utf8_util.cpp @@ -0,0 +1,31 @@ +#include + +#include + +#include "utf8_util.h" + +TEST_CASE("popBack removes one code point", "[utf8_util]") { + std::string s = "aç€"; + utf8_util::popBack(s); + REQUIRE(s == "aç"); + utf8_util::popBack(s); + REQUIRE(s == "a"); + utf8_util::popBack(s); + REQUIRE(s.empty()); + utf8_util::popBack(s); // empty: no-op + REQUIRE(s.empty()); +} + +TEST_CASE("tail returns whole string when short enough", "[utf8_util]") { + REQUIRE(utf8_util::tail("ção", 10) == "ção"); +} + +TEST_CASE("tail skips a split leading character", "[utf8_util]") { + // "x€x": x(1) €(3) x(1) = 5 bytes; the last 3 bytes start inside the €. + REQUIRE(utf8_util::tail("x€x", 3) == "x"); + REQUIRE(utf8_util::tail("x€x", 4) == "€x"); +} + +TEST_CASE("tail of ASCII is a plain suffix", "[utf8_util]") { + REQUIRE(utf8_util::tail("hello world", 5) == "world"); +}