From 1725dedaf3c9ff8750e5c3667c8f3399c85da1cf Mon Sep 17 00:00:00 2001 From: Aaron Queen Date: Mon, 28 Sep 2026 01:53:23 -0600 Subject: [PATCH] perf(explore): cover name lookups with (kind, name); size budgets from cheap counts idx_nodes_kind becomes (kind, name) in schema v17, so explore's name LIKE scans read the index instead of every row of the kind. The substring query now states its old tie order (length, kind, rowid) so answers do not move with the index shape. Explore and the tool list size their budgets from getSizeStats() instead of getStats(), which grouped every edge by kind. trezor-suite: one lookup 7-10 ms -> 1-2.4 ms; 10-question explore round about 15% faster; output byte-identical. --- CHANGELOG.md | 1 + __tests__/db-perf.test.ts | 26 +++++++++++++++++ __tests__/explore-output-budget.test.ts | 4 +-- docs/design/metrics-ledger.md | 18 ++++++++++++ src/codegraph.ts | 8 ++++++ src/db/migrations.ts | 14 +++++++++- src/db/queries.ts | 37 ++++++++++++++++++------- src/db/schema.sql | 4 ++- src/mcp/tools.ts | 8 +++--- 9 files changed, 102 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 95f4a43806..9f3a240908 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -217,6 +217,7 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - In a JavaScript or TypeScript monorepo, imports between packages now reach the right file. Each package's own `tsconfig.json` or `jsconfig.json` `paths` apply to its files, so `@/logger` in one app no longer lands in another app's `src/`. A workspace package whose `main`, `module` or `exports` point at a `dist/` folder that isn't committed now resolves to the source file that builds it. Calls through those imports used to fall back to matching the name anywhere in the project. +- `codegraph_explore` answers about 15% faster on large projects. Its name lookups read a narrower index instead of every matching row, and it no longer counts every edge by kind on each call to size its answer. An existing index is upgraded the next time it is opened for writing, such as by `codegraph sync`. - A one-file sync is faster on large JavaScript and TypeScript projects: codegraph remembers which files each inference step and the framework check found nothing in and skips them until they change, and reads only the symbols each step can use. Checking the project's files against `.gitignore` and preparing source text for the inference steps also take less time. Syncs also use less memory. - Nuxt page routes now link to the page component they render, and `index` folders no longer leak into the URL (`pages/account/index/[page].vue` is `/account/:page`). diff --git a/__tests__/db-perf.test.ts b/__tests__/db-perf.test.ts index df83a4c4c0..0ae42df35d 100644 --- a/__tests__/db-perf.test.ts +++ b/__tests__/db-perf.test.ts @@ -364,6 +364,32 @@ describe('migration v6: dedup edges + add identity index on upgrade (#1034)', () }); }); +describe('migration v17: idx_nodes_kind covers name substring scans', () => { + it('widens the index and keeps the kind-then-insertion tie order', () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'db-mig17-')); + const db = DatabaseConnection.initialize(path.join(dir, 'test.db')); + const raw = db.getDb(); + const q = new QueryBuilder(raw); + raw.exec('DROP INDEX idx_nodes_kind; CREATE INDEX idx_nodes_kind ON nodes(kind)'); + raw.prepare('DELETE FROM schema_versions WHERE version >= 17').run(); + // Same length, inserted out of name order: the (kind, name) walk would + // reorder them if the query left ties to the plan. + q.insertNodes([makeNode('n1', 'zFooBar'), makeNode('n2', 'aFooBar'), makeNode('n3', 'FooBar')]); + const names = () => q.findNodesByNameSubstring('Foo', { kinds: ['function'] }).map((r) => r.node.name); + expect(names()).toEqual(['FooBar', 'zFooBar', 'aFooBar']); + + runMigrations(raw, 16); + + expect(getCurrentVersion(raw)).toBe(CURRENT_SCHEMA_VERSION); + const cols = (raw.prepare('PRAGMA index_info(idx_nodes_kind)').all() as Array<{ name: string }>).map((c) => c.name); + expect(cols).toEqual(['kind', 'name']); + expect(names()).toEqual(['FooBar', 'zFooBar', 'aFooBar']); + + db.close(); + fs.rmSync(dir, { recursive: true, force: true }); + }); +}); + describe('edge traversal order', () => { let dir: string; let db: DatabaseConnection; diff --git a/__tests__/explore-output-budget.test.ts b/__tests__/explore-output-budget.test.ts index e77b66f91d..80b48e4807 100644 --- a/__tests__/explore-output-budget.test.ts +++ b/__tests__/explore-output-budget.test.ts @@ -207,8 +207,8 @@ describe('codegraph_explore output respects the adaptive budget', () => { // WORDING is what this test pins. Regression guard: quota phrasing // ("remaining calls" / "Synthesize once") must never come back — agents // read it as a hard cap, stop exploring early, and fall back to grep+Read. - const spy = vi.spyOn(cg, 'getStats').mockReturnValue({ - ...cg.getStats(), + const spy = vi.spyOn(cg, 'getSizeStats').mockReturnValue({ + ...cg.getSizeStats(), fileCount: 1500, filesByLanguage: { typescript: 1000, markdown: 500 }, }); diff --git a/docs/design/metrics-ledger.md b/docs/design/metrics-ledger.md index ba1446c070..41a01ce94c 100644 --- a/docs/design/metrics-ledger.md +++ b/docs/design/metrics-ledger.md @@ -829,6 +829,24 @@ Now two programming languages that cannot name each other's symbols never bind b | `eval:precision` javalin | 1/1 absent, 1/1 present held (Kotlin → Java member import kept) | | Kernel/TS resolve parity, bridge suites (RN, Expo, Swift/ObjC, cross-tier) | pass | +### 5.99 Explore SQL: covering index for name lookups, cheap size counts (2026-09-28) + +Follow-up to §5.98's per-statement profile. Two changes: + +- `idx_nodes_kind` is now `(kind, name)` (schema v17). The `name LIKE '%x%'` lookups explore runs per query term and kind set already walked this index for `kind IN (...)`, then read every full row to test `name`. With `name` in the index, the scan never touches the table. The query now spells out the old tie order (`length(name), kind, rowid`); without it, ties followed the index's name order and one of ten trezor-suite answers swapped two RPC files for UI files. +- Explore and the tool list size their budgets from `getSizeStats()` (node and file counts, files per language) instead of `getStats()`, which also grouped every node and edge by kind on each call. + +| trezor-suite (151k nodes, 387k edges) | Before | After | +|---|---|---| +| One `LIKE` lookup, 200-row limit | 7–10 ms | 1–2.4 ms | +| 10-question explore round median, paired runs | 5,884 / 6,551 ms | 5,033 / 5,070 ms (about −15%) | +| Same, size counts only (index still v16) | | 5,456 / 5,601 ms | +| Explore output | | byte-identical on all 10 questions, v16 and v17 | +| Index size / build | `(kind)` 35–38 ms | `(kind, name)` 66–70 ms, 5.3 MB of a 513 MB database | +| v16 → v17 migration on open | | 86–98 ms | + +Read-only connections do not migrate, so an existing index gets the covering index the next time a writable process such as `codegraph index` or `codegraph sync` opens it. + ### 5.98 Explore read path: node:sqlite vs better-sqlite3 vs bun:sqlite (2026-09-28) §5.11 item 4 compared the three drivers on inserts only. This run covers reads: `codegraph_explore` through `ToolHandler.executeReadTool` (the query worker's path), 8 fixed questions per index, a read-only connection, 5 timed rounds per process after a warm-up, and each arm run twice in ABCD/DCBA order on an idle host. The two alternate drivers sat behind the `SqliteDatabase` adapter through a bench-only switch that was not landed. Every arm returned byte-identical output for every question. diff --git a/src/codegraph.ts b/src/codegraph.ts index 616791e9fc..70549f8a35 100644 --- a/src/codegraph.ts +++ b/src/codegraph.ts @@ -1631,6 +1631,14 @@ export class CodeGraph { // Graph Statistics // =========================================================================== + /** + * Node and file counts plus files per language, without getStats' per-kind + * grouping of every node and edge. Budget sizing reads only these. + */ + getSizeStats(): Pick { + return this.queries.getSizeStats(); + } + /** * Get statistics about the knowledge graph */ diff --git a/src/db/migrations.ts b/src/db/migrations.ts index 23e73f230c..83da0781f3 100644 --- a/src/db/migrations.ts +++ b/src/db/migrations.ts @@ -9,7 +9,7 @@ import { SqliteDatabase } from './sqlite-adapter'; /** * Current schema version */ -export const CURRENT_SCHEMA_VERSION = 16; +export const CURRENT_SCHEMA_VERSION = 17; /** * Migration definition @@ -315,6 +315,18 @@ CREATE TABLE synth_skips ( `); }, }, + { + version: 17, + description: 'Widen idx_nodes_kind to (kind, name) so name substring scans read only the index', + up: (db) => { + // Keep in lockstep with schema.sql. Partial legacy fixtures carry no nodes table. + if ((db.prepare('PRAGMA table_info(nodes)').all() as unknown[]).length === 0) return; + db.exec(` + DROP INDEX IF EXISTS idx_nodes_kind; + CREATE INDEX idx_nodes_kind ON nodes(kind, name); + `); + }, + }, ]; /** diff --git a/src/db/queries.ts b/src/db/queries.ts index a4a598409c..8b30f2ebff 100644 --- a/src/db/queries.ts +++ b/src/db/queries.ts @@ -1975,6 +1975,8 @@ export class QueryBuilder { * e.g. "TransportSearchAction" is one FTS token, not matchable by "Search"*. * * Results are ordered by name length (shorter = more likely to be the core type). + * Ties keep kind-then-insertion order, which the idx_nodes_kind walk used to + * give implicitly; spelling it out keeps results stable across index shapes. */ findNodesByNameSubstring( substring: string, @@ -2005,7 +2007,7 @@ export class QueryBuilder { params.push(...languages); } - sql += ' ORDER BY length(name) ASC LIMIT ?'; + sql += ' ORDER BY length(name) ASC, kind ASC, nodes.rowid ASC LIMIT ?'; params.push(limit); const rows = this.db.prepare(sql).all(...params) as (NodeRow & { score: number })[]; @@ -3894,6 +3896,29 @@ export class QueryBuilder { .get() as { nodes: number; edges: number }; } + /** + * The project-size figures explore and the tool list size their budgets + * from. getStats also groups every node and edge by kind, which costs tens + * of milliseconds per call on a large index; these counts stay cheap. + */ + getSizeStats(): Pick { + const counts = this.db + .prepare('SELECT (SELECT COUNT(*) FROM nodes) AS nodes, (SELECT COUNT(*) FROM files) AS files') + .get() as { nodes: number; files: number }; + return { nodeCount: counts.nodes, fileCount: counts.files, filesByLanguage: this.getFilesByLanguage() }; + } + + private getFilesByLanguage(): Record { + const filesByLanguage = {} as Record; + const languageRows = this.db + .prepare('SELECT language, COUNT(*) as count FROM files GROUP BY language') + .all() as Array<{ language: string; count: number }>; + for (const row of languageRows) { + filesByLanguage[row.language as Language] = row.count; + } + return filesByLanguage; + } + /** * Get graph statistics */ @@ -3922,21 +3947,13 @@ export class QueryBuilder { edgesByKind[row.kind as EdgeKind] = row.count; } - const filesByLanguage = {} as Record; - const languageRows = this.db - .prepare('SELECT language, COUNT(*) as count FROM files GROUP BY language') - .all() as Array<{ language: string; count: number }>; - for (const row of languageRows) { - filesByLanguage[row.language as Language] = row.count; - } - return { nodeCount: counts.node_count, edgeCount: counts.edge_count, fileCount: counts.file_count, nodesByKind, edgesByKind, - filesByLanguage, + filesByLanguage: this.getFilesByLanguage(), dbSizeBytes: 0, // Set by caller using DatabaseConnection.getSize() walSizeBytes: 0, // Set by caller using DatabaseConnection.getWalSizeBytes() lastUpdated: Date.now(), diff --git a/src/db/schema.sql b/src/db/schema.sql index 3fc6671154..16da0d961a 100644 --- a/src/db/schema.sql +++ b/src/db/schema.sql @@ -106,7 +106,9 @@ CREATE TABLE IF NOT EXISTS unresolved_refs ( -- ============================================================================= -- Node indexes -CREATE INDEX IF NOT EXISTS idx_nodes_kind ON nodes(kind); +-- (kind, name) covers the name LIKE scans explore runs per kind set: the scan +-- reads the index instead of each full row (5-7x faster on a 151k-node index). +CREATE INDEX IF NOT EXISTS idx_nodes_kind ON nodes(kind, name); CREATE INDEX IF NOT EXISTS idx_nodes_name ON nodes(name); CREATE INDEX IF NOT EXISTS idx_nodes_qualified_name ON nodes(qualified_name); CREATE INDEX IF NOT EXISTS idx_nodes_file_path ON nodes(file_path); diff --git a/src/mcp/tools.ts b/src/mcp/tools.ts index 73ead3ff98..839fe31670 100644 --- a/src/mcp/tools.ts +++ b/src/mcp/tools.ts @@ -172,7 +172,7 @@ export function normalizeQuerySpelling(query: string): string { * indexed for the doc tier, and the tiers were calibrated on code, so a * README-heavy repo must not cross a breakpoint and change its code answers. */ -export function codeFileCount(stats: GraphStats): number { +export function codeFileCount(stats: Pick): number { return stats.fileCount - (stats.filesByLanguage.markdown ?? 0); } @@ -2044,7 +2044,7 @@ export class ToolHandler { if (!this.cg) return withRequiredProjectPath(visible); try { - const fileCount = codeFileCount(this.cg.getStats()); + const fileCount = codeFileCount(this.cg.getSizeStats()); const budget = getExploreBudget(fileCount); // Tiny-repo tool gating: on projects under TINY_REPO_FILE_THRESHOLD @@ -4016,7 +4016,7 @@ export class ToolHandler { // Total files, not codeFileCount: excluding markdown demotes a doc-heavy // repo a tier, and the smaller tier drops the answer's lines and turns off // the completeness signal that would say so (measured, 465 vs 589 here). - const stats = cg.getStats(); + const stats = cg.getSizeStats(); indexedFileCount = stats.fileCount; indexedNodeCount = stats.nodeCount; budget = getExploreOutputBudget(indexedFileCount); @@ -7287,7 +7287,7 @@ export class ToolHandler { let budgetBlock: string[] = []; if (budget.includeBudgetNote) { try { - const fileCount = codeFileCount(cg.getStats()); + const fileCount = codeFileCount(cg.getSizeStats()); const callBudget = getExploreBudget(fileCount); budgetBlock = ['', `> **Exploration guidance — advisory only, NOT a quota: this project (~${fileCount.toLocaleString()} files indexed) is usually covered in ≈${callBudget} focused explore calls, and extra calls are never rejected or rate-limited.** If the response above does not fully cover your question, run another codegraph_explore on the uncovered symbols — it is cheaper and more complete than Read. Only stop exploring when the response actually covers the flow you asked about.`]; } catch {