Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion fcitx5-engine/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ set(CMAKE_CXX_STANDARD 20)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)

find_package(Fcitx5Core 5.0 REQUIRED)
find_package(Fcitx5Core 5.0.4 REQUIRED) # 5.0.4: Instance::processComposeString
find_package(CURL REQUIRED)

option(BUILD_TESTS "Build unit tests" OFF)
Expand Down
23 changes: 19 additions & 4 deletions fcitx5-engine/src/context_buffer.cpp
Original file line number Diff line number Diff line change
@@ -1,11 +1,17 @@
#include "context_buffer.h"

#include "utf8_util.h"

// ── Private ───────────────────────────────────────────────────────────────────

void ContextBuffer::push(char ch) {
if (_buf.size() >= MAX_CHARS)
_buf.pop_front();
_buf.push_back(ch);
if (_buf.size() > MAX_CHARS) {
_buf.pop_front();
// Never leave the tail bytes of a multi-byte character at the front.
while (!_buf.empty() && utf8_util::isContinuation(_buf.front()))
_buf.pop_front();
}
}

// ── Queries ───────────────────────────────────────────────────────────────────
Expand All @@ -29,11 +35,20 @@ void ContextBuffer::appendChar(char ch) {
push(ch);
}

void ContextBuffer::appendText(std::string_view utf8) {
dismiss();
for (char ch : utf8)
push(ch);
}

void ContextBuffer::backspace() {
if (_suggestion) {
dismiss();
} else if (!_buf.empty()) {
_buf.pop_back();
} else {
while (!_buf.empty() && utf8_util::isContinuation(_buf.back()))
_buf.pop_back();
if (!_buf.empty())
_buf.pop_back();
}
}

Expand Down
6 changes: 4 additions & 2 deletions fcitx5-engine/src/context_buffer.h
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,13 @@
#include <deque>
#include <optional>
#include <string>
#include <string_view>

class ContextBuffer {
public:
void appendChar(char ch);
void backspace();
void appendText(std::string_view utf8); // a typed character or composed sequence
void backspace(); // removes one code point
void setSuggestion(std::string text);
std::string acceptNextWord();
std::string acceptAll();
Expand All @@ -19,7 +21,7 @@ class ContextBuffer {
bool hasSuggestion() const;

private:
void push(char ch); // rolling-window insert: drops oldest when at MAX_CHARS
void push(char ch); // rolling-window insert: drops oldest code point past MAX_CHARS bytes

std::deque<char> _buf;
std::optional<std::string> _suggestion;
Expand Down
29 changes: 29 additions & 0 deletions fcitx5-engine/src/utf8_util.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
#pragma once

#include <string>

// Minimal UTF-8 helpers for the typed-text buffers. The engine stores what the
// user typed as UTF-8 bytes; anything that drops or slices bytes must stay on
// code-point boundaries, or the text sent to Ollama / written to JSON becomes
// invalid UTF-8.
namespace utf8_util {

inline bool isContinuation(char c) {
return (static_cast<unsigned char>(c) & 0xC0) == 0x80;
}

// Removes the last code point (not just the last byte).
inline void popBack(std::string &s) {
while (!s.empty() && isContinuation(s.back())) s.pop_back();
if (!s.empty()) s.pop_back();
}

// The last <= maxBytes bytes of s, starting on a code-point boundary.
inline std::string tail(const std::string &s, size_t maxBytes) {
if (s.size() <= maxBytes) return s;
size_t start = s.size() - maxBytes;
while (start < s.size() && isContinuation(s[start])) ++start;
return s.substr(start);
}

} // namespace utf8_util
65 changes: 51 additions & 14 deletions fcitx5-engine/src/xtype.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
#include "config_loader.h"
#include "path_utils.h"
#include "prompt_builder.h"
#include "utf8_util.h"

// ── Helpers ───────────────────────────────────────────────────────────────────

Expand All @@ -48,6 +49,14 @@ static void dbg(const char *fmt, ...) {
va_list ap; va_start(ap, fmt); std::vfprintf(f, fmt, ap); va_end(ap);
std::fputc('\n', f); std::fflush(f);
}
// UTF-8 text a key types on its own, or "" for non-text keys (arrows, F-keys,
// modifiers, dead keys) and control characters.
static std::string typedText(fcitx::KeySym sym) {
const uint32_t uc = fcitx::Key::keySymToUnicode(sym);
if (uc < 0x20 || uc == 0x7f) return {};
return fcitx::Key::keySymToUTF8(sym);
}

static void log_warn(const char *msg) {
std::fprintf(stderr, "[xtype] WARNING: %s\n", msg);
dbg("WARN: %s", msg);
Expand Down Expand Up @@ -246,6 +255,16 @@ void XTypeEngine::keyEvent(const fcitx::InputMethodEntry &,
const bool hasSuggestion = _ctx.hasSuggestion();
const auto sym = event.key().sym();

// These keys cancel a pending dead-key / Compose sequence in the app; keep
// our mirror of that state (see the text branch below) in step. A cancelled
// sequence leaves the text untouched, so Backspace deletes nothing.
if (_instance->isComposing(ic) &&
(sym == FcitxKey_Escape || sym == FcitxKey_Return ||
sym == FcitxKey_KP_Enter || sym == FcitxKey_BackSpace)) {
_instance->resetCompose(ic);
if (sym == FcitxKey_BackSpace) return;
}

// Tab — accept next word; Shift+Tab / ISO_Left_Tab — accept entire suggestion.
if (sym == FcitxKey_Tab || sym == FcitxKey_ISO_Left_Tab) {
if (!hasSuggestion) return;
Expand Down Expand Up @@ -299,8 +318,7 @@ void XTypeEngine::keyEvent(const fcitx::InputMethodEntry &,
event.filterAndAccept();
} else {
_ctx.backspace();
if (!_userTypedSinceLastTerminator.empty())
_userTypedSinceLastTerminator.pop_back();
utf8_util::popBack(_userTypedSinceLastTerminator);
_debounceTimer.reset();
}
return;
Expand All @@ -323,15 +341,36 @@ void XTypeEngine::keyEvent(const fcitx::InputMethodEntry &,
return;
}

// Printable ASCII (space through ~). Pass key through to app, update buffer,
// arm debounce timer for inference.
if (sym >= FcitxKey_space && sym <= FcitxKey_asciitilde) {
dbg("key '%c' prog=%s ctx_len=%zu", static_cast<char>(sym),
// Text keys. The key always passes through and the app inserts the text
// itself — including dead-key / Compose sequences, which the app composes
// on its side (US-International: ' then space -> ', ' then a -> á). Feed
// the same keys through Fcitx5's XCompose state so the buffer records the
// text the app shows rather than raw keysyms.
std::string text;
if (auto composed = _instance->processComposeString(ic, sym)) {
if (!composed->empty()) {
text = std::move(*composed); // sequence completed
} else if (_instance->isComposing(ic)) {
// Mid-sequence (e.g. a dead key): nothing inserted yet. Drop the
// ghost text so it can't sit on top of the app's compose preedit.
invalidate();
updatePreedit(ic);
_debounceTimer.reset();
return;
} else {
text = typedText(sym); // not part of a sequence
}
} else {
// Sequence cancelled (no rule for dead key + this key). Toolkits differ
// on what they insert; at least the key itself is the common case.
text = typedText(sym);
}
if (!text.empty()) {
dbg("key '%s' prog=%s ctx_len=%zu", text.c_str(),
ic->program().c_str(), _ctx.contextText().size());
invalidate();
updatePreedit(ic);
const char ch = static_cast<char>(sym);
_ctx.appendChar(ch);
_ctx.appendText(text);

// User-typed-only buffer for corpus harvest (excludes AI accept paths).
// Rolling strategy: on overflow, erase through the next sentence terminator
Expand All @@ -344,8 +383,8 @@ void XTypeEngine::keyEvent(const fcitx::InputMethodEntry &,
else
buf.clear();
}
_userTypedSinceLastTerminator.push_back(ch);
if (ch == '.' || ch == '!' || ch == '?')
_userTypedSinceLastTerminator += text;
if (text == "." || text == "!" || text == "?")
harvestSentence(ic->program());

_debounceTimer.reset();
Expand Down Expand Up @@ -379,8 +418,7 @@ void XTypeEngine::requestInference(
if (static_cast<int>(ctx.size()) < _cfg.inference.min_context_chars)
return;
if (static_cast<int>(ctx.size()) > _cfg.inference.context_window)
ctx = ctx.substr(ctx.size() -
static_cast<size_t>(_cfg.inference.context_window));
ctx = utf8_util::tail(ctx, static_cast<size_t>(_cfg.inference.context_window));

++_gen;
++_metrics.suggestions_generated;
Expand Down Expand Up @@ -453,8 +491,7 @@ void XTypeEngine::requestInference(
if (_ctx.hasSuggestion()) {
RecentEvent ev;
const std::string ctxText = _ctx.contextText();
ev.typed = ctxText.substr(
static_cast<size_t>(std::max(0, (int)ctxText.size() - 40)));
ev.typed = utf8_util::tail(ctxText, 40);
ev.ghost = _ctx.suggestion() ? *_ctx.suggestion() : "";
ev.app = icPtr->program();
ev.accepted = false;
Expand Down
7 changes: 7 additions & 0 deletions fcitx5-engine/tests/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,13 @@ target_link_libraries(test_phrase_blocklist PRIVATE
)
catch_discover_tests(test_phrase_blocklist)

add_executable(test_utf8_util test_utf8_util.cpp)
target_include_directories(test_utf8_util PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../src)
target_link_libraries(test_utf8_util PRIVATE
Catch2::Catch2WithMain
)
catch_discover_tests(test_utf8_util)

add_executable(test_xtype_helpers test_xtype_helpers.cpp)
target_link_libraries(test_xtype_helpers PRIVATE
Catch2::Catch2WithMain
Expand Down
37 changes: 37 additions & 0 deletions fcitx5-engine/tests/test_context_buffer.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -273,3 +273,40 @@ TEST_CASE("reset allows fresh use", "[reset]") {
REQUIRE(cb.contextText() == "x");
REQUIRE_FALSE(cb.hasSuggestion());
}

// ── UTF-8 ─────────────────────────────────────────────────────────────────────

TEST_CASE("appendText stores multi-byte characters", "[utf8]") {
ContextBuffer cb;
cb.appendText("não");
cb.appendText("'");
REQUIRE(cb.contextText() == "não'");
}

TEST_CASE("appendText dismisses active suggestion", "[utf8]") {
ContextBuffer cb;
cb.setSuggestion("hello");
cb.appendText("ç");
REQUIRE_FALSE(cb.hasSuggestion());
}

TEST_CASE("backspace removes a whole multi-byte character", "[utf8][backspace]") {
ContextBuffer cb;
cb.appendText("café");
cb.backspace();
REQUIRE(cb.contextText() == "caf");
cb.appendText("€");
cb.backspace();
REQUIRE(cb.contextText() == "caf");
}

TEST_CASE("rolling window never starts with a partial character", "[utf8][window]") {
ContextBuffer cb;
cb.appendChar('x'); // shifts the 2-byte chars off even offsets
for (int i = 0; i < 400; ++i) cb.appendText("ã");
const std::string t = cb.contextText();
REQUIRE(t.size() <= 500);
REQUIRE(t.size() >= 499);
REQUIRE((static_cast<unsigned char>(t.front()) & 0xC0) != 0x80);
REQUIRE(t.substr(0, 2) == "ã");
}
20 changes: 20 additions & 0 deletions fcitx5-engine/tests/test_style_profile.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -206,6 +206,26 @@ TEST_CASE("serialize/deserialize round-trip preserves all fields", "[style][json
fs::remove_all(d);
}

TEST_CASE("serialize/deserialize round-trip preserves UTF-8 text", "[style][json][utf8]") {
auto d = uniqueDir("utf8");
auto cp = d / "corpus.txt";
writeCorpus(cp, {
"não sei se a configuração está certa agora",
"você já viu a nova versão do aplicativo",
"I'm sure it works with “quotes” and ç too",
});
StyleProfile a;
a.loadFromCorpus(cp.string(), 7);
REQUIRE_FALSE(a.exemplars().empty());

auto jp = d / "style.json";
REQUIRE(a.serialize(jp.string()));
StyleProfile b = StyleProfile::deserialize(jp.string());
REQUIRE(b.exemplars() == a.exemplars());
REQUIRE(b.commonOpeners() == a.commonOpeners());
fs::remove_all(d);
}

TEST_CASE("corrupt JSON deserialize is safe", "[style][json]") {
auto d = uniqueDir("corrupt");
auto jp = d / "style.json";
Expand Down
31 changes: 31 additions & 0 deletions fcitx5-engine/tests/test_utf8_util.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
#include <catch2/catch_test_macros.hpp>

#include <string>

#include "utf8_util.h"

TEST_CASE("popBack removes one code point", "[utf8_util]") {
std::string s = "aç€";
utf8_util::popBack(s);
REQUIRE(s == "aç");
utf8_util::popBack(s);
REQUIRE(s == "a");
utf8_util::popBack(s);
REQUIRE(s.empty());
utf8_util::popBack(s); // empty: no-op
REQUIRE(s.empty());
}

TEST_CASE("tail returns whole string when short enough", "[utf8_util]") {
REQUIRE(utf8_util::tail("ção", 10) == "ção");
}

TEST_CASE("tail skips a split leading character", "[utf8_util]") {
// "x€x": x(1) €(3) x(1) = 5 bytes; the last 3 bytes start inside the €.
REQUIRE(utf8_util::tail("x€x", 3) == "x");
REQUIRE(utf8_util::tail("x€x", 4) == "€x");
}

TEST_CASE("tail of ASCII is a plain suffix", "[utf8_util]") {
REQUIRE(utf8_util::tail("hello world", 5) == "world");
}