From c82a15ced2835cad3057062e048cb114c388b661 Mon Sep 17 00:00:00 2001 From: albertonoys <3334359+albertonoys@users.noreply.github.com> Date: Tue, 17 Nov 2026 23:22:37 +0100 Subject: [PATCH] perf(backend): query dpkg once when listing the catalog - --summary ran dpkg-query twice per app (146 runs for 73 apps); each run parses the whole dpkg status file, ~0.2 s on the device, so opening the store spent ~29 s here - summary_output now takes one dpkg-query -W snapshot of every package and answers package_state() from it; other callers still query live, so install and repair checks see fresh state - a failed snapshot falls back to per-package queries; a package missing from the snapshot is reported as absent, as dpkg-query's exit 1 was - on a CardputerZero with 1900 packages, --summary drops from 28.8 s to 2.2 s with identical output - tests: dpkg-query fakes answer the batched form, and --summary must make exactly one call --- main/backend/appstore_native.cpp | 66 ++++++++++++++++++++++++++------ tests/test_native_backend.cpp | 10 ++++- tests/test_native_backend.sh | 4 ++ 3 files changed, 68 insertions(+), 12 deletions(-) diff --git a/main/backend/appstore_native.cpp b/main/backend/appstore_native.cpp index 8c6f3c0..d725301 100644 --- a/main/backend/appstore_native.cpp +++ b/main/backend/appstore_native.cpp @@ -1053,9 +1053,61 @@ struct PackageState { bool known = false; }; +PackageState parse_package_state(std::string status, std::string version) +{ + while (!status.empty() && std::isspace(static_cast(status.back()))) status.pop_back(); + while (!version.empty() && std::isspace(static_cast(version.back()))) version.pop_back(); + // A third status character denotes a dpkg error such as Reinst-required. + // Do not treat that state as a healthy installed package merely because the + // current-state character is `i`. + const bool installed = status.size() >= 2 && status[1] == 'i' && status.size() < 3; + const bool absent = (status.size() == 2 || status.size() == 3) && + std::string("uihpr").find(status[0]) != std::string::npos && (status[1] == 'n' || status[1] == 'c'); + return {installed, version, installed || absent}; +} + +// Each dpkg-query run parses the whole dpkg status file (~0.2 s on the device), +// so listing the catalog queries every package once instead of once per app. +bool g_package_snapshot_active = false; +std::map g_package_snapshot; + +struct PackageSnapshotScope { + PackageSnapshotScope() + { + auto [rc, output] = capture({"dpkg-query", "-W", "-f=${Package}\t${db:Status-Abbrev}\t${Version}\n"}); + if (rc != 0) return; // Fall back to querying each package. + std::istringstream lines(output); + std::string line; + while (std::getline(lines, line)) { + const size_t first = line.find('\t'); + const size_t second = first == std::string::npos ? first : line.find('\t', first + 1); + if (second == std::string::npos) continue; + const std::string name = line.substr(0, first); + PackageState state = parse_package_state(line.substr(first + 1, second - first - 1), + line.substr(second + 1)); + // With several architectures of one package, report the installed one. + auto [entry, inserted] = g_package_snapshot.emplace(name, state); + if (!inserted && state.installed) entry->second = state; + } + g_package_snapshot_active = true; + } + ~PackageSnapshotScope() + { + g_package_snapshot_active = false; + g_package_snapshot.clear(); + } + PackageSnapshotScope(const PackageSnapshotScope &) = delete; + PackageSnapshotScope &operator=(const PackageSnapshotScope &) = delete; +}; + PackageState package_state(const std::string &package) { if (package.empty()) return {false, "", true}; + if (g_package_snapshot_active) { + // dpkg-query -W lists every package dpkg knows; a missing one is absent. + const auto entry = g_package_snapshot.find(package); + return entry == g_package_snapshot.end() ? PackageState{false, "", true} : entry->second; + } auto [rc, output] = capture({"dpkg-query", "-W", "-f=${db:Status-Abbrev}\t${Version}", package}, 8192); if (rc != 0) { // dpkg-query exits 1 when the package is not known to dpkg. The @@ -1064,17 +1116,8 @@ PackageState package_state(const std::string &package) return {false, "", rc == 1}; } size_t tab = output.find('\t'); - std::string status = tab == std::string::npos ? output : output.substr(0, tab); - std::string version = tab == std::string::npos ? "" : output.substr(tab + 1); - while (!status.empty() && std::isspace(static_cast(status.back()))) status.pop_back(); - while (!version.empty() && std::isspace(static_cast(version.back()))) version.pop_back(); - // A third status character denotes a dpkg error such as Reinst-required. - // Do not treat that state as a healthy installed package merely because the - // current-state character is `i`. - const bool installed = status.size() >= 2 && status[1] == 'i' && status.size() < 3; - const bool absent = (status.size() == 2 || status.size() == 3) && - std::string("uihpr").find(status[0]) != std::string::npos && (status[1] == 'n' || status[1] == 'c'); - return {installed, version, installed || absent}; + return parse_package_state(tab == std::string::npos ? output : output.substr(0, tab), + tab == std::string::npos ? "" : output.substr(tab + 1)); } PackageState effective_package_state(const std::string &package) @@ -1140,6 +1183,7 @@ json find_app(const std::string &wanted) std::string summary_output() { + PackageSnapshotScope package_snapshot; std::ostringstream out; auto records = load_records(false); json apps = merge_apps(records); diff --git a/tests/test_native_backend.cpp b/tests/test_native_backend.cpp index 8990658..6277e2a 100644 --- a/tests/test_native_backend.cpp +++ b/tests/test_native_backend.cpp @@ -127,7 +127,10 @@ esac set -eu if [ -n "${FAKE_DPKG_LOCALE_LOG:-}" ]; then printf '%s\n' "${LC_ALL:-}" >> "$FAKE_DPKG_LOCALE_LOG"; fi if [ "$1" = -L ]; then [ "${FAKE_FILES_FAIL:-0}" = 0 ] || exit 2; printf '/usr/bin/demo-package\n/usr/share/APPLaunch/applications/demo.desktop\n'; exit; fi -case "${FAKE_DPKG_MODE:-absent}" in installed) printf '%s\t%s\n' "${FAKE_STATUS:-ii }" "${FAKE_VERSION:-1.2.0}";; un) printf 'un \t0.1.0\n';; localized-absent) printf 'dpkg-query: 没有找到与 demo-package 相匹配的软件包\n' >&2; exit 1;; error) echo 'database error' >&2;exit 2;; *) echo 'no packages found matching' >&2;exit 1;; esac +if [ -n "${FAKE_DPKG_CALL_LOG:-}" ]; then printf '%s\n' "$*" >> "$FAKE_DPKG_CALL_LOG"; fi +# The catalog lists every package in one call: -f starts with ${Package} and no name is given. +p=; case "$2" in *'${Package}'*) p=$(printf 'demo-package\t');; esac +case "${FAKE_DPKG_MODE:-absent}" in installed) printf '%s%s\t%s\n' "$p" "${FAKE_STATUS:-ii }" "${FAKE_VERSION:-1.2.0}";; un) printf '%sun \t0.1.0\n' "$p";; localized-absent) [ -z "$p" ] || exit 0; printf 'dpkg-query: 没有找到与 demo-package 相匹配的软件包\n' >&2; exit 1;; error) echo 'database error' >&2;exit 2;; *) [ -z "$p" ] || exit 0; echo 'no packages found matching' >&2;exit 1;; esac )",0755); text(root/"bin/dpkg-deb", R"(#!/bin/sh case "$3" in Package) echo "${FAKE_DEB_PACKAGE:-demo-package}";; Version) echo "${FAKE_DEB_VERSION:-1.2.0}";; Pre-Depends|Depends) echo -n "${FAKE_DEPENDS:-}";; *) exit 1;; esac @@ -144,7 +147,12 @@ static void registry_cases() { CHECK(call({"--set-region","moon"}).rc!=0); CHECK(call({"--set-region","CN"}).rc==0); CHECK(has(call({"--registry-config"}),"CONFIG\tCN\tCN\t1")); CHECK(call({"--set-region","default"}).rc==0); Result s=call({"--sync"}); CHECK(s.rc==0&&has(s,"SYNC\t1\t0\t0\t2")); + const fs::path dpkg_calls=root/"state/dpkg-calls.log"; fs::remove(dpkg_calls); + setenv("FAKE_DPKG_CALL_LOG",dpkg_calls.c_str(),1); Result sum=call({"--summary"}); CHECK(has(sum,"META\t1\t2 apps/1 registries")); + unsetenv("FAKE_DPKG_CALL_LOG"); + { std::ifstream calls(dpkg_calls); std::string line; int count=0; while (std::getline(calls,line)) if (line.rfind("-",0)==0) ++count; // -f ends in a newline + CHECK(count==1); } // One dpkg-query for the whole catalog, not one per app. CHECK(has(sum,"CAT\tAll\nCAT\tInstalled\nCAT\tUtilities\nCAT\tTools\n")); CHECK(!has(sum,"CAT\tRecommended\n")); CHECK(has(sum,"APP\tapp-id\tDemo Local\t1.2.0\tUtilities\t0\t1\t")); diff --git a/tests/test_native_backend.sh b/tests/test_native_backend.sh index 1e5bcb6..479d377 100755 --- a/tests/test_native_backend.sh +++ b/tests/test_native_backend.sh @@ -46,6 +46,10 @@ if [ "${FAKE_DPKG_LOCALE:-}" = zh ]; then printf 'dpkg-query: 没有找到与 demo-package 相匹配的软件包\n' >&2 exit 1 fi +# The catalog lists every package in one call: -f starts with ${Package} and no name is given. +case "$2" in + *'${Package}'*) if [ -f "$FAKE_PACKAGE_STATE" ]; then printf 'demo-package\tii \t1.2.0\n'; fi; exit 0 ;; +esac if [ -f "$FAKE_PACKAGE_STATE" ]; then printf 'ii \t1.2.0\n'; exit 0; fi printf 'dpkg-query: no packages found matching demo-package\n' >&2 exit 1