diff --git a/apps/mobile/src/features/files/FileCodeRelationships.tsx b/apps/mobile/src/features/files/FileCodeRelationships.tsx index 9321fd2eb0e54..f83095627d030 100644 --- a/apps/mobile/src/features/files/FileCodeRelationships.tsx +++ b/apps/mobile/src/features/files/FileCodeRelationships.tsx @@ -50,7 +50,7 @@ function coverageLine(coverage: ProjectCodeCoverage): string { case "stale": return "Stale · some files changed since indexing. Verify in source."; case "unavailable": - return "The code index is unavailable for this checkout."; + return "The code index is not ready for this checkout."; } } @@ -220,7 +220,7 @@ function Body(props: { return ( {coverageLine(data.coverage)} - {data.nodes.length === 0 ? ( + {data.coverage.state === "unavailable" ? null : data.nodes.length === 0 ? ( No indexed declarations in this file. Its language may be unsupported. ) : ( data.nodes.map((node) => ( diff --git a/apps/mobile/src/features/settings/ThreadSettingsSections.tsx b/apps/mobile/src/features/settings/ThreadSettingsSections.tsx index 17a0067a7fa52..c398df416cb45 100644 --- a/apps/mobile/src/features/settings/ThreadSettingsSections.tsx +++ b/apps/mobile/src/features/settings/ThreadSettingsSections.tsx @@ -244,40 +244,81 @@ function BetaSettingsSection() { environment.connection.phase === "connected" && environment.serverConfig?.environment.capabilities.peerLinks === true, ); - const writable = useEnvironmentsWithScope(supported, AuthSettingsWriteScope); + const indexing = environments.filter( + (environment) => + environment.connection.phase === "connected" && + environment.serverConfig?.environment.capabilities.projectCode === true, + ); + const writable = useEnvironmentsWithScope(environments, AuthSettingsWriteScope); const updateSettings = useAtomCommand(serverEnvironment.updateSettings, { label: "linked environments beta update", reportFailure: true, }); - if (supported.length === 0) return null; + if (supported.length === 0 && indexing.length === 0) return null; return ( - - - {supported.map((environment) => { - const policy = environment.serverConfig?.managedPolicy; - const managed = isManagedSettingPath(policy, "linkedEnvironmentsEnabled"); - return ( - - void updateSettings({ - environmentId: environment.environmentId, - input: { patch: { linkedEnvironmentsEnabled: value } }, - }) - } - /> - ); - })} - - - Let agents on a machine work on your other machines, and theirs on it. Turn it on for each - machine. While it is off, nothing new starts over a link; tasks already running finish. - + + {supported.length === 0 ? null : ( + + + {supported.map((environment) => { + const policy = environment.serverConfig?.managedPolicy; + const managed = isManagedSettingPath(policy, "linkedEnvironmentsEnabled"); + return ( + + void updateSettings({ + environmentId: environment.environmentId, + input: { patch: { linkedEnvironmentsEnabled: value } }, + }) + } + /> + ); + })} + + + Let agents on a machine work on your other machines, and theirs on it. Turn it on for + each machine. While it is off, nothing new starts over a link; tasks already running + finish. + + + )} + {indexing.length === 0 ? null : ( + + + {indexing.map((environment) => { + const policy = environment.serverConfig?.managedPolicy; + const managed = isManagedSettingPath(policy, "autoCodeIndexEnabled"); + return ( + + void updateSettings({ + environmentId: environment.environmentId, + input: { patch: { autoCodeIndexEnabled: value } }, + }) + } + /> + ); + })} + + + Index a project when you add it, and a worktree while an agent works in it. A worktree's + index is removed when its thread settles. While this is off, a checkout only indexes + when something asks it a code question. + + + )} ); } diff --git a/apps/server/src/config.ts b/apps/server/src/config.ts index a411e31222df4..d1753f0f7c363 100644 --- a/apps/server/src/config.ts +++ b/apps/server/src/config.ts @@ -45,6 +45,7 @@ export interface ServerDerivedPaths { readonly providerStatusCacheDir: string; readonly worktreesDir: string; readonly attachmentsDir: string; + readonly codeIndexDir: string; readonly browserArtifactsDir: string; readonly logsDir: string; readonly serverTracePath: string; @@ -161,6 +162,7 @@ export const deriveServerPaths = Effect.fn(function* ( providerStatusCacheDir, worktreesDir: join(baseDir, "worktrees"), attachmentsDir, + codeIndexDir: join(stateDir, "code-index"), browserArtifactsDir: join(stateDir, "browser-artifacts"), logsDir, serverTracePath: join(logsDir, "server.trace.ndjson"), @@ -185,6 +187,7 @@ export const ensureServerDirectories = Effect.fn(function* (derivedPaths: Server fs.makeDirectory(derivedPaths.providerLogsDir, { recursive: true }), fs.makeDirectory(derivedPaths.terminalLogsDir, { recursive: true }), fs.makeDirectory(derivedPaths.attachmentsDir, { recursive: true }), + fs.makeDirectory(derivedPaths.codeIndexDir, { recursive: true }), fs.makeDirectory(derivedPaths.worktreesDir, { recursive: true }), fs.makeDirectory(path.dirname(derivedPaths.keybindingsConfigPath), { recursive: true }), fs.makeDirectory(path.dirname(derivedPaths.settingsPath), { recursive: true }), diff --git a/apps/server/src/orchestration-v2/RunFinalizationService.test.ts b/apps/server/src/orchestration-v2/RunFinalizationService.test.ts index 244276486f35b..383176540a71c 100644 --- a/apps/server/src/orchestration-v2/RunFinalizationService.test.ts +++ b/apps/server/src/orchestration-v2/RunFinalizationService.test.ts @@ -15,6 +15,7 @@ import { ProviderInstanceId, RunId, ThreadId, + VcsProcessExitError, type OrchestrationV2ThreadShell, } from "@t3tools/contracts"; import * as Deferred from "effect/Deferred"; @@ -29,9 +30,12 @@ import * as TestClock from "effect/testing/TestClock"; import * as ServerConfig from "../config.ts"; import * as CodeIntelligence from "../project/knowledge/CodeIntelligence.ts"; +import { indexRootFor } from "../project/knowledge/codeIndexLayout.ts"; import * as ModelEvaluations from "../modelEvaluations/ModelEvaluations.ts"; import * as PullRequestService from "../pullRequest/PullRequestService.ts"; +import * as ServerSettings from "../serverSettings.ts"; import * as GitVcsDriver from "../vcs/GitVcsDriver.ts"; +import * as VcsDriver from "../vcs/VcsDriver.ts"; import * as VcsStatusBroadcaster from "../vcs/VcsStatusBroadcaster.ts"; import * as WorkspaceEntries from "../workspace/WorkspaceEntries.ts"; import * as WorkspacePaths from "../workspace/WorkspacePaths.ts"; @@ -425,6 +429,13 @@ it.effect("finalizes a run without waiting for the code index refresh its checkp const threadId = ThreadId.make("thread_index_refresh"); const runId = RunId.make("run_index_refresh"); const scopeId = CheckpointScopeId.make("scope_index_refresh"); + const configContext = yield* Layer.build( + ServerConfig.layerTest(process.cwd(), { prefix: "t3-finalize-index-config-" }).pipe( + Layer.provide(NodeServices.layer), + ), + ); + const config = yield* ServerConfig.ServerConfig.pipe(Effect.provide(configContext)); + const indexRoot = indexRootFor(config.codeIndexDir, root); const layer = RunFinalization.layer.pipe( Layer.provideMerge(RunFinalization.layerObserver), Layer.provideMerge(CodeIntelligence.layer), @@ -465,6 +476,19 @@ it.effect("finalizes a run without waiting for the code index refresh its checkp truncated: false, }), }), + Layer.mock(VcsDriver.VcsDriver)({ + checkpoints: {} as VcsDriver.VcsDriver["Service"]["checkpoints"], + listWorkspaceFiles: () => + Effect.fail( + new VcsProcessExitError({ + operation: "test", + command: "git ls-files", + cwd: root, + exitCode: 128, + detail: "not a git repository", + }), + ), + }), Layer.mock(PullRequestService.PullRequestService)({ refreshAfterTurn: () => Effect.void, }), @@ -479,6 +503,8 @@ it.effect("finalizes a run without waiting for the code index refresh its checkp workingTree: { files: [], insertions: 0, deletions: 0 }, }), }), + ServerSettings.layerTest(), + Layer.succeed(ServerConfig.ServerConfig, config), ), ), ); @@ -492,12 +518,12 @@ it.effect("finalizes a run without waiting for the code index refresh its checkp const finalize = finalization.finalize({ threadId, runId, scopeId }); const published = () => Effect.promise(() => - NodeFSP.readFile(NodePath.join(root, ".codegraph-t3/published"), "utf8"), + NodeFSP.readFile(NodePath.join(indexRoot, ".codegraph-t3/published"), "utf8"), ); // A checkout nobody has queried is not indexed behind the user's back. yield* finalize; - assert.isFalse(NodeFS.existsSync(NodePath.join(root, ".codegraph-t3"))); + assert.isFalse(NodeFS.existsSync(indexRoot)); assert.deepEqual(yield* intelligence.refresh({ projectId }), { _tag: "Published", @@ -510,7 +536,7 @@ it.effect("finalizes a run without waiting for the code index refresh its checkp const buildHeld = yield* Deferred.make(); const releaseBuild = yield* Deferred.make(); const lease = yield* withWorkspaceLease( - NodePath.join(root, ".codegraph-t3"), + indexRoot, Deferred.succeed(buildHeld, undefined).pipe( Effect.andThen(Deferred.await(releaseBuild)), ), diff --git a/apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts b/apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts index 3e954cf9d694a..8723fabda85e2 100644 --- a/apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts +++ b/apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts @@ -99,6 +99,7 @@ export function makeReplayServerConfig( const providerLogsDir = path.join(logsDir, "provider"); const terminalLogsDir = path.join(logsDir, "terminals"); const attachmentsDir = path.join(stateDir, "attachments"); + const codeIndexDir = path.join(stateDir, "code-index"); const environmentThemesDir = path.join(stateDir, "themes"); const worktreesDir = path.join(baseDir, "worktrees"); const providerStatusCacheDir = path.join(baseDir, "caches"); @@ -109,6 +110,7 @@ export function makeReplayServerConfig( providerLogsDir, terminalLogsDir, attachmentsDir, + codeIndexDir, environmentThemesDir, worktreesDir, providerStatusCacheDir, @@ -152,6 +154,7 @@ export function makeReplayServerConfig( providerStatusCacheDir, worktreesDir, attachmentsDir, + codeIndexDir: path.join(stateDir, "code-index"), browserArtifactsDir: path.join(stateDir, "browser-artifacts"), environmentThemesDir, logsDir, diff --git a/apps/server/src/project/knowledge/CodeIndexReactor.test.ts b/apps/server/src/project/knowledge/CodeIndexReactor.test.ts new file mode 100644 index 0000000000000..88f13e3a52a31 --- /dev/null +++ b/apps/server/src/project/knowledge/CodeIndexReactor.test.ts @@ -0,0 +1,639 @@ +import { + DEFAULT_SERVER_SETTINGS, + EventId, + ProjectId, + ProviderInstanceId, + RunId, + ThreadId, + type ApplicationStoredEvent, + type OrchestrationV2Run, + type OrchestrationV2ThreadShell, + type ServerSettings, +} from "@t3tools/contracts"; +import * as HostProcess from "@t3tools/shared/HostProcess"; +import * as DateTime from "effect/DateTime"; +import * as Deferred from "effect/Deferred"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Path from "effect/Path"; +import * as PubSub from "effect/PubSub"; +import * as Queue from "effect/Queue"; +import * as Stream from "effect/Stream"; +import { assert, describe, it } from "@effect/vitest"; + +import * as ProjectionStore from "../../orchestration-v2/ProjectionStore.ts"; +import * as ProjectStore from "../../orchestration-v2/ProjectStore.ts"; +import * as OrchestrationEventStore from "../../persistence/OrchestrationEventStore.ts"; +import * as ServerSettingsService from "../../serverSettings.ts"; +import { StorageCleanup } from "../../storageCleanup.ts"; +import * as CodeIndexReactor from "./CodeIndexReactor.ts"; +import * as CodeIntelligence from "./CodeIntelligence.ts"; + +const NOW_MS = Date.parse("2026-06-10T12:00:00.000Z"); +const DAY_MS = 86_400_000; +const PROJECT_ID = ProjectId.make("project-index-reactor"); +const WORKTREE = "/worktrees/feature"; +const now = DateTime.makeUnsafe(NOW_MS); + +function at(offsetMs: number): DateTime.Utc { + return DateTime.makeUnsafe(NOW_MS + offsetMs); +} + +function shell(overrides: Partial = {}): OrchestrationV2ThreadShell { + return { + id: ThreadId.make("thread-1"), + projectId: PROJECT_ID, + title: "Thread", + providerInstanceId: ProviderInstanceId.make("codex"), + modelSelection: { instanceId: ProviderInstanceId.make("codex"), model: "gpt-5.4" }, + runtimeMode: "full-access", + interactionMode: "default", + worktreePath: WORKTREE, + branch: "feature", + activeProviderThreadId: null, + lineage: { + rootThreadId: ThreadId.make("thread-1"), + parentThreadId: null, + relationshipToParent: null, + }, + forkedFrom: null, + createdBy: "user", + creationSource: "web", + activeRunId: null, + latestVisibleMessage: null, + hasActionableProposedPlan: false, + itemCount: 0, + visibleItemCount: 0, + lastVisitedAt: null, + deletedAt: null, + linkedPullRequest: null, + status: "idle", + activityRunStatus: null, + pendingRuntimeRequest: null, + pendingBackgroundTasks: [], + latestRunId: null, + latestRunRequestedAt: null, + latestRunStartedAt: null, + latestRunCompletedAt: null, + latestUserMessageAt: null, + createdAt: at(-DAY_MS), + updatedAt: at(-DAY_MS), + archivedAt: null, + settledOverride: null, + settledAt: null, + snoozedUntil: null, + snoozedAt: null, + pinnedAt: null, + ...overrides, + }; +} + +describe("worktreeIndexReleasable", () => { + const settled = (overrides: Partial = {}) => + shell({ settledOverride: "settled", settledAt: at(-60_000), ...overrides }); + const subagent = (overrides: Partial = {}) => + shell({ + id: ThreadId.make("thread-subagent"), + lineage: { + rootThreadId: ThreadId.make("thread-1"), + parentThreadId: ThreadId.make("thread-1"), + relationshipToParent: "subagent", + }, + ...overrides, + }); + const unrelated = (overrides: Partial = {}) => + shell({ + id: ThreadId.make("thread-2"), + lineage: { + rootThreadId: ThreadId.make("thread-2"), + parentThreadId: null, + relationshipToParent: null, + }, + ...overrides, + }); + + it.each([ + { name: "one settled thread", group: () => [settled()], expected: true }, + { name: "one unsettled thread", group: () => [shell()], expected: false }, + { + name: "a settled parent with an idle subagent", + group: () => [settled(), subagent({ settledOverride: "settled", settledAt: at(-30_000) })], + expected: true, + }, + { + name: "a settled parent with an idle subagent nobody settled", + group: () => [settled(), subagent()], + expected: true, + }, + { + name: "a settled parent with a pinned subagent", + group: () => [settled(), subagent({ pinnedAt: now })], + expected: false, + }, + { + name: "a settled parent with a side chat kept active", + group: () => [ + settled(), + subagent({ + settledOverride: "active", + lineage: { + rootThreadId: ThreadId.make("thread-1"), + parentThreadId: ThreadId.make("thread-1"), + relationshipToParent: null, + sideChat: true, + }, + }), + ], + expected: false, + }, + { + name: "a settled parent with a running subagent", + group: () => [settled(), subagent({ status: "running", activeRunId: RunId.make("run-1") })], + expected: false, + }, + { + name: "two unrelated threads, both settled", + group: () => [settled(), unrelated({ settledOverride: "settled", settledAt: at(-90_000) })], + expected: true, + }, + { + name: "two unrelated threads, one unsettled", + group: () => [settled(), unrelated()], + expected: false, + }, + { + name: "an unsettled thread unused for 7 days", + group: () => [shell({ createdAt: at(-30 * DAY_MS), latestRunCompletedAt: at(-7 * DAY_MS) })], + expected: true, + }, + { + name: "an unsettled thread unused for 6 days", + group: () => [shell({ createdAt: at(-30 * DAY_MS), latestRunCompletedAt: at(-6 * DAY_MS) })], + expected: false, + }, + { + name: "a long unused thread that is pinned", + group: () => [shell({ createdAt: at(-30 * DAY_MS), pinnedAt: now })], + expected: false, + }, + { + name: "a long unused thread kept active", + group: () => [shell({ createdAt: at(-30 * DAY_MS), settledOverride: "active" })], + expected: false, + }, + { + name: "a long unused thread with a pending request", + group: () => [ + shell({ + createdAt: at(-30 * DAY_MS), + pendingBackgroundTasks: [{ taskId: "task", kind: "command" }], + }), + ], + expected: false, + }, + { + name: "a long unused thread beside a recently used one", + group: () => [shell({ createdAt: at(-30 * DAY_MS) }), unrelated()], + expected: false, + }, + { name: "no threads", group: () => [], expected: false }, + ])("$name", ({ group, expected }) => { + assert.equal( + CodeIndexReactor.worktreeIndexReleasable(group(), NOW_MS, DEFAULT_SERVER_SETTINGS), + expected, + ); + }); + + it.each([ + { name: "settled parent", parent: {}, child: {}, expected: true }, + { name: "active parent", parent: { settledAt: null }, child: {}, expected: false }, + { name: "running parent", parent: { status: "running" }, child: {}, expected: false }, + { name: "pinned child", parent: {}, child: { pinnedAt: now }, expected: false }, + { + name: "explicitly active child", + parent: {}, + child: { settledOverride: "active" }, + expected: false, + }, + { + name: "other project", + parent: { projectId: ProjectId.make("other") }, + child: {}, + expected: false, + }, + ] satisfies ReadonlyArray<{ + name: string; + parent: Partial; + child: Partial; + expected: boolean; + }>)("handles a subagent in its own worktree with a $name", ({ parent, child, expected }) => { + const owner = settled({ worktreePath: null, ...parent }); + const agent = subagent(child); + assert.equal( + CodeIndexReactor.worktreeIndexReleasable( + [agent], + NOW_MS, + DEFAULT_SERVER_SETTINGS, + new Map([owner, agent].map((thread) => [thread.id, thread])), + ), + expected, + ); + }); +}); + +function runEvent(runId: RunId, status: OrchestrationV2Run["status"]): ApplicationStoredEvent { + return { + sequence: 1, + commandId: null, + event: { + type: "run.updated", + occurredAt: now, + payload: { + id: runId, + threadId: ThreadId.make("thread-1"), + ordinal: 1, + providerInstanceId: ProviderInstanceId.make("codex"), + modelSelection: { instanceId: ProviderInstanceId.make("codex"), model: "gpt-5.4" }, + providerThreadId: null, + userMessageId: "message-1", + rootNodeId: null, + activeAttemptId: null, + status, + requestedAt: now, + startedAt: null, + completedAt: null, + checkpointId: null, + contextHandoffId: null, + }, + }, + } as unknown as ApplicationStoredEvent; +} + +function settledEvent(thread: OrchestrationV2ThreadShell): ApplicationStoredEvent { + return { + sequence: 2, + commandId: null, + event: { type: "thread.settled", occurredAt: now, payload: thread }, + } as unknown as ApplicationStoredEvent; +} + +/** A project event whose effect marks the end of the queue, proving earlier ones were skipped. */ +function sentinelEvent(projectId: ProjectId): ApplicationStoredEvent { + return { + sequence: 3, + eventId: EventId.make("event-sentinel"), + aggregateKind: "project", + aggregateId: projectId, + occurredAt: now, + commandId: null, + causationEventId: null, + correlationId: null, + metadata: {}, + type: "project.created", + payload: { projectId }, + } as unknown as ApplicationStoredEvent; +} + +interface Harness { + readonly send: (event: ApplicationStoredEvent) => Effect.Effect; + /** Effects the reactor asked for, in order. */ + readonly effects: Queue.Queue; + /** The owners the activation sweep's orphan pass rejected. */ + readonly orphansRejected: Effect.Effect>; + /** The checkouts the activation sweep asked to clear of an older version's folders. */ + readonly legacyCheckouts: Effect.Effect>; + /** Resolves when the next sweep starts. */ + readonly nextSweep: Effect.Effect; + readonly finishMaintenanceRun: Effect.Effect; + /** Resolves once the sweep asks whether a worktree has an index. */ + readonly indexChecked: Effect.Effect; + readonly settings: PubSub.PubSub; + /** The next effect the reactor asks for, waiting for it if needed. */ + readonly next: Effect.Effect; + /** Sends a sentinel and returns every effect recorded before it. */ + readonly effectsBeforeSentinel: Effect.Effect>; +} + +const SENTINEL = ProjectId.make("project-sentinel"); +const SENTINEL_EFFECT = `ensure:${SENTINEL}`; + +function reactorHarness(options: { + readonly threads: ReadonlyArray; + readonly projectRoots?: ReadonlyArray; + readonly autoCodeIndexEnabled?: boolean; + /** Worktrees the mocked service reports an index for. */ + readonly indexed?: ReadonlyArray; + /** Checkouts the mocked orphan sweep would find an index for. */ + readonly orphanOwners?: ReadonlyArray; +}): Effect.Effect { + return Effect.gen(function* () { + const events = yield* Queue.unbounded(); + const effects = yield* Queue.unbounded(); + const orphansRejected = yield* Deferred.make>(); + const legacyCheckouts = yield* Deferred.make>(); + const sweeps = yield* Queue.unbounded(); + const reports = yield* Queue.unbounded(); + yield* Queue.offer(reports, null); + const indexChecked = yield* Deferred.make(); + const settings = yield* PubSub.unbounded(); + const threadsById = new Map(options.threads.map((thread) => [thread.id, thread])); + const initial = { + ...DEFAULT_SERVER_SETTINGS, + autoCodeIndexEnabled: options.autoCodeIndexEnabled ?? true, + }; + + const layer = CodeIndexReactor.layer.pipe( + Layer.provide( + Layer.mergeAll( + Layer.mock(CodeIntelligence.CodeIntelligence)({ + ensureProjectIndexed: (projectId) => + Queue.offer(effects, `ensure:${projectId}`).pipe(Effect.asVoid), + refreshForActiveAgent: (target) => + Queue.offer(effects, `refresh:${target.threadId}`).pipe(Effect.asVoid), + removeCheckoutIndex: (checkoutPath) => + Queue.offer(effects, `remove:${checkoutPath}`).pipe(Effect.asVoid), + hasCheckoutIndex: (checkoutPath) => + Deferred.succeed(indexChecked, undefined).pipe( + Effect.as((options.indexed ?? []).includes(checkoutPath)), + ), + removeLegacyIndexes: (checkoutPaths) => + Deferred.succeed(legacyCheckouts, checkoutPaths).pipe( + Effect.andThen(Queue.offer(sweeps, undefined)), + Effect.asVoid, + ), + removeOrphanedIndexes: (isKnown) => + Deferred.succeed( + orphansRejected, + (options.orphanOwners ?? []).filter((owner) => !isKnown(owner)), + ).pipe(Effect.asVoid), + stopBuilds: Queue.offer(effects, "stop").pipe(Effect.asVoid), + }), + Layer.mock(StorageCleanup)({ reports: Stream.fromQueue(reports) }), + Layer.mock(OrchestrationEventStore.OrchestrationEventStore)({ + latestApplicationSequence: Effect.succeed(0), + streamApplicationEvents: () => Stream.fromQueue(events), + }), + Layer.mock(ProjectionStore.ProjectionStoreV2)({ + getThreadWorktreePaths: () => + Effect.succeed( + options.threads.flatMap((thread) => + thread.worktreePath === null + ? [] + : [{ threadId: thread.id, worktreePath: thread.worktreePath }], + ), + ), + getThreadShell: (threadId) => Effect.succeed(threadsById.get(threadId) ?? null), + }), + Layer.mock(ProjectStore.ProjectStoreV2)({ + listShells: () => + Effect.succeed( + (options.projectRoots ?? []).map((workspaceRoot) => ({ workspaceRoot })), + ) as unknown as ReturnType, + }), + Layer.mock(ServerSettingsService.ServerSettingsService)({ + getSettings: Effect.succeed(initial), + subscribeChanges: PubSub.subscribe(settings).pipe( + Effect.map((subscription) => Stream.fromSubscription(subscription)), + ), + }), + Path.layer, + ), + ), + ); + yield* Layer.build(layer).pipe(Effect.orDie); + + const untilSentinel = ( + collected: ReadonlyArray, + ): Effect.Effect> => + Queue.take(effects).pipe( + Effect.flatMap((effect) => + effect === SENTINEL_EFFECT + ? Effect.succeed(collected) + : untilSentinel([...collected, effect]), + ), + ); + return { + send: (event) => Queue.offer(events, event).pipe(Effect.asVoid), + effects, + orphansRejected: Deferred.await(orphansRejected), + legacyCheckouts: Deferred.await(legacyCheckouts), + nextSweep: Queue.take(sweeps), + finishMaintenanceRun: Queue.offer(reports, null).pipe(Effect.asVoid), + indexChecked: Deferred.await(indexChecked), + settings, + next: Queue.take(effects), + // Events process in order, so the sentinel's effect proves every earlier one finished. + effectsBeforeSentinel: Queue.offer(events, sentinelEvent(SENTINEL)).pipe( + Effect.andThen(untilSentinel([])), + ), + }; + }); +} + +/** Records the checkouts a sweep's `known` predicate rejected, for its orphan pass. */ + +describe("CodeIndexReactor", () => { + it.effect("requests an index once per run, and again for a new run", () => + Effect.gen(function* () { + const harness = yield* reactorHarness({ threads: [shell()] }); + const run = RunId.make("run-1"); + yield* harness.send(runEvent(RunId.make("run-0"), "waiting")); + // A run's status flips through `running` during approvals; each request can spawn a process. + yield* harness.send(runEvent(run, "running")); + yield* harness.send(runEvent(run, "waiting")); + yield* harness.send(runEvent(run, "running")); + yield* harness.send(runEvent(RunId.make("run-2"), "running")); + yield* harness.send(runEvent(RunId.make("run-2"), "running")); + yield* harness.send(runEvent(RunId.make("run-2"), "completed")); + assert.deepEqual(yield* harness.effectsBeforeSentinel, [ + `refresh:${ThreadId.make("thread-1")}`, + `refresh:${ThreadId.make("thread-1")}`, + ]); + }), + ); + + it.effect("removes a settled worktree's index", () => + Effect.gen(function* () { + const thread = shell({ settledOverride: "settled", settledAt: at(-60_000) }); + const harness = yield* reactorHarness({ threads: [thread] }); + yield* harness.send(settledEvent(thread)); + assert.deepEqual(yield* harness.effectsBeforeSentinel, [`remove:${WORKTREE}`]); + }), + ); + + it.effect("keeps a worktree's index while another thread in it is unsettled", () => + Effect.gen(function* () { + const settled = shell({ settledOverride: "settled", settledAt: at(-60_000) }); + const unrelated = shell({ + id: ThreadId.make("thread-2"), + lineage: { + rootThreadId: ThreadId.make("thread-2"), + parentThreadId: null, + relationshipToParent: null, + }, + }); + const harness = yield* reactorHarness({ threads: [settled, unrelated] }); + yield* harness.send(settledEvent(settled)); + assert.deepEqual(yield* harness.effectsBeforeSentinel, []); + }), + ); + + it.effect("keeps the index of a worktree that is a project root", () => + Effect.gen(function* () { + const thread = shell({ settledOverride: "settled", settledAt: at(-60_000) }); + const harness = yield* reactorHarness({ threads: [thread], projectRoots: [WORKTREE] }); + yield* harness.send(settledEvent(thread)); + assert.deepEqual(yield* harness.effectsBeforeSentinel, []); + }), + ); + + it.effect("removes nothing when automatic indexing is off", () => + Effect.gen(function* () { + const worktreePath = "/worktrees/idle"; + const thread = shell({ worktreePath, settledOverride: "settled", settledAt: at(-60_000) }); + const harness = yield* reactorHarness({ + threads: [thread], + indexed: [worktreePath], + autoCodeIndexEnabled: false, + }); + yield* harness.send(settledEvent(thread)); + assert.deepEqual(yield* harness.effectsBeforeSentinel, []); + // The activation sweep is its own fiber, scheduled by the time a second round trip ends. + assert.deepEqual(yield* harness.effectsBeforeSentinel, []); + }), + ); + + it.effect("removes orphaned indexes and old folders even while automatic indexing is off", () => + Effect.gen(function* () { + const harness = yield* reactorHarness({ + threads: [shell()], + autoCodeIndexEnabled: false, + orphanOwners: ["/worktrees/gone", WORKTREE], + }); + // A known worktree is kept; the deleted checkout is the only rejection. + assert.deepEqual(yield* harness.orphansRejected, ["/worktrees/gone"]); + assert.deepEqual(yield* harness.legacyCheckouts, [WORKTREE]); + }), + ); + + it.effect("sweeps again whenever a Maintenance run finishes", () => + Effect.gen(function* () { + const harness = yield* reactorHarness({ threads: [shell()] }); + yield* harness.nextSweep; + yield* harness.finishMaintenanceRun; + yield* harness.nextSweep; + }), + ); + + it.effect("treats project roots and thread worktrees as known checkouts", () => + Effect.gen(function* () { + const harness = yield* reactorHarness({ + threads: [shell()], + projectRoots: ["/projects/one"], + orphanOwners: ["/projects/one", WORKTREE], + }); + assert.deepEqual(yield* harness.orphansRejected, []); + assert.deepEqual(yield* harness.legacyCheckouts, ["/projects/one", WORKTREE]); + }), + ); + + it.effect("does not count a deleted project as a known checkout", () => + Effect.gen(function* () { + // `listShells` omits deleted projects, so its roots are absent from the known set. + const harness = yield* reactorHarness({ + threads: [shell()], + projectRoots: [], + orphanOwners: ["/projects/deleted"], + }); + assert.deepEqual(yield* harness.orphansRejected, ["/projects/deleted"]); + }), + ); + + it.effect("keeps known Windows checkouts through drive-letter and case differences", (context) => + Effect.gen(function* () { + if (!(yield* HostProcess.isWindows)) return context.skip(); + const harness = yield* reactorHarness({ + threads: [shell({ worktreePath: "c:\\worktrees\\Feature" })], + projectRoots: ["c:\\projects\\App"], + autoCodeIndexEnabled: false, + orphanOwners: ["C:\\PROJECTS\\APP", "C:\\WORKTREES\\FEATURE"], + }); + assert.deepEqual(yield* harness.orphansRejected, []); + }), + ); + + it.effect( + "keeps a settled Windows worktree while a differently cased thread is unsettled", + (context) => + Effect.gen(function* () { + if (!(yield* HostProcess.isWindows)) return context.skip(); + const settled = shell({ + worktreePath: "c:\\worktrees\\Feature", + settledOverride: "settled", + settledAt: at(-60_000), + }); + const harness = yield* reactorHarness({ + threads: [ + settled, + shell({ id: ThreadId.make("thread-2"), worktreePath: "C:\\WORKTREES\\FEATURE" }), + ], + }); + yield* harness.send(settledEvent(settled)); + assert.deepEqual(yield* harness.effectsBeforeSentinel, []); + }), + ); + + it.effect("stops running builds when automatic indexing is turned off", () => + Effect.gen(function* () { + const harness = yield* reactorHarness({ threads: [shell()] }); + yield* PubSub.publish(harness.settings, { + ...DEFAULT_SERVER_SETTINGS, + autoCodeIndexEnabled: false, + }); + assert.equal(yield* harness.next, "stop"); + }), + ); + + it.effect.each(["thread", "subagent's parent"] as const)( + "sweeps the index of a worktree whose %s settled before this feature", + (settledOwner) => + Effect.gen(function* () { + const worktreePath = "/worktrees/settled"; + const thread = shell({ + worktreePath, + settledOverride: "settled", + settledAt: at(-60_000), + }); + const threads = + settledOwner === "thread" + ? [thread] + : [ + { ...thread, worktreePath: null }, + shell({ + id: ThreadId.make("thread-subagent"), + worktreePath, + lineage: { + rootThreadId: thread.id, + parentThreadId: thread.id, + relationshipToParent: "subagent", + }, + }), + ]; + // Activation sweeps, so the removal arrives without any event. + const harness = yield* reactorHarness({ threads, indexed: [worktreePath] }); + assert.equal(yield* harness.next, `remove:${worktreePath}`); + }), + ); + + it.effect("skips a settled worktree that has no index", () => + Effect.gen(function* () { + const thread = shell({ settledOverride: "settled", settledAt: at(-60_000) }); + // The sweep asks the service whether an index exists, rather than checking the checkout. + const harness = yield* reactorHarness({ threads: [thread], indexed: [] }); + yield* harness.indexChecked; + assert.deepEqual(yield* harness.effectsBeforeSentinel, []); + // The sweep is its own fiber, scheduled by the time a second round trip ends. + assert.deepEqual(yield* harness.effectsBeforeSentinel, []); + }), + ); +}); diff --git a/apps/server/src/project/knowledge/CodeIndexReactor.ts b/apps/server/src/project/knowledge/CodeIndexReactor.ts new file mode 100644 index 0000000000000..20981c4ad1003 --- /dev/null +++ b/apps/server/src/project/knowledge/CodeIndexReactor.ts @@ -0,0 +1,236 @@ +import type { + ApplicationStoredEvent, + OrchestrationV2ThreadShell, + RunId, + ServerSettingsError, + ThreadId, +} from "@t3tools/contracts"; +import * as Clock from "effect/Clock"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Path from "effect/Path"; +import * as Stream from "effect/Stream"; + +import * as ProjectionStore from "../../orchestration-v2/ProjectionStore.ts"; +import * as ProjectStore from "../../orchestration-v2/ProjectStore.ts"; +import * as OrchestrationEventStore from "../../persistence/OrchestrationEventStore.ts"; +import { forkParked } from "../../serverActivation.ts"; +import * as ServerSettings from "../../serverSettings.ts"; +import { + StorageCleanup, + storageCleanupActivityAt, + storageCleanupThreadIdle, + worktreeOwner, + worktreeSettlementChain, +} from "../../storageCleanup.ts"; +import * as CodeIntelligence from "./CodeIntelligence.ts"; +import { checkoutPathKey } from "./codeIndexLayout.ts"; + +// An index is a cache the next agent run rebuilds, so an unused one is not worth its disk. +const UNUSED_INDEX_MS = 7 * 24 * 60 * 60 * 1000; + +/** + * Whether a worktree's index can go: its threads are idle, and either settled, with a subagent or + * side chat following its owner rather than holding the worktree open, or long unused. + */ +export function worktreeIndexReleasable( + group: ReadonlyArray, + now: number, + settings: Parameters[2], + threadsById: ReadonlyMap = new Map( + group.map((thread) => [thread.id, thread]), + ), +): boolean { + if (group.length === 0) return false; + const owner = worktreeOwner(group); + const keptGoing = group.some( + (t) => t.settledAt === null && (t.settledOverride === "active" || t.pinnedAt != null), + ); + // A subagent or side-chat chain follows its owner; unrelated threads must all be settled. + const settled = + owner !== undefined + ? worktreeSettlementChain(owner, threadsById, now, settings).length > 0 && !keptGoing + : group.every((t) => t.settledAt !== null); + const unused = + !keptGoing && group.every((t) => now - storageCleanupActivityAt(t) >= UNUSED_INDEX_MS); + return (settled || unused) && group.every((t) => storageCleanupThreadIdle(t, now, settings)); +} + +/** + * Keeps the code index in step with the work happening in a checkout: a project indexes when it + * is added, a worktree while an agent runs in it, and a worktree's index goes when its thread + * settles or goes unused. Each Maintenance run also sweeps, dropping indexes for checkouts that no + * longer exist even while automatic indexing is off. + */ +const make = Effect.gen(function* () { + const storageCleanup = yield* StorageCleanup; + const codeIntelligence = yield* CodeIntelligence.CodeIntelligence; + const events = yield* OrchestrationEventStore.OrchestrationEventStore; + const projections = yield* ProjectionStore.ProjectionStoreV2; + const projectStore = yield* ProjectStore.ProjectStoreV2; + const settingsService = yield* ServerSettings.ServerSettingsService; + const path = yield* Path.Path; + + const autoIndexEnabled = settingsService.getSettings.pipe( + Effect.map((settings) => settings.autoCodeIndexEnabled), + Effect.orElseSucceed(() => false), + ); + + const releaseWorktreeIndex = Effect.fn("CodeIndexReactor.releaseWorktreeIndex")(function* ( + worktreePath: string, + ) { + if (!(yield* autoIndexEnabled)) return; + const target = path.resolve(worktreePath); + const entries = yield* projections.getThreadWorktreePaths({ includeArchived: true }); + const group = (yield* Effect.forEach( + entries.filter( + (entry) => checkoutPathKey(path.resolve(entry.worktreePath)) === checkoutPathKey(target), + ), + (entry) => projections.getThreadShell(entry.threadId), + )).filter((thread) => thread !== null); + const threadsById = new Map(group.map((thread) => [thread.id, thread])); + for (const thread of group) { + let current = thread; + while (current.settledAt === null && current.lineage.relationshipToParent === "subagent") { + const parentId = current.lineage.parentThreadId; + if (parentId === null || threadsById.has(parentId)) break; + const parent = yield* projections.getThreadShell(parentId); + if (parent === null) break; + threadsById.set(parent.id, parent); + current = parent; + } + } + const settings = yield* settingsService.getSettings; + const now = yield* Clock.currentTimeMillis; + if (!worktreeIndexReleasable(group, now, settings, threadsById)) return; + // A worktree registered as its own project keeps its index. + const projects = yield* projectStore.listShells(); + if ( + projects.some( + (project) => + checkoutPathKey(path.resolve(project.workspaceRoot)) === checkoutPathKey(target), + ) + ) + return; + yield* codeIntelligence.removeCheckoutIndex(target); + }); + + const sweep = Effect.fn("CodeIndexReactor.sweep")(function* () { + const entries = yield* projections.getThreadWorktreePaths({ includeArchived: true }); + // `listShells` omits deleted projects, so a deleted project is not a known checkout. + const projects = yield* projectStore.listShells(); + const known = new Map( + [ + ...projects.map((project) => project.workspaceRoot), + ...entries.map((e) => e.worktreePath), + ].map((knownPath) => { + const resolved = path.resolve(knownPath); + return [checkoutPathKey(resolved), resolved]; + }), + ); + // Even with automatic indexing off: these only clean up T3's own data. + yield* codeIntelligence.removeLegacyIndexes([...known.values()]); + yield* codeIntelligence.removeOrphanedIndexes((checkoutPath) => + known.has(checkoutPathKey(path.resolve(checkoutPath))), + ); + if (!(yield* autoIndexEnabled)) return; + // De-duplicated first: hundreds of threads share a path, and each check hits the disk. + const candidates = new Map( + entries.map((entry) => { + const resolved = path.resolve(entry.worktreePath); + return [checkoutPathKey(resolved), resolved]; + }), + ); + // Sequentially: each release waits on the checkout's build lock. + for (const worktreePath of candidates.values()) { + if (!(yield* codeIntelligence.hasCheckoutIndex(worktreePath))) continue; + yield* releaseWorktreeIndex(worktreePath).pipe( + Effect.catchCause((cause) => + Effect.logWarning("code index sweep failed for a worktree", { worktreePath, cause }), + ), + ); + } + }); + + // Only a run's first `running` counts: its status flips during approvals, and each request can + // spawn a process. + const activeRuns = new Map(); + + const refreshRun = Effect.fn("CodeIndexReactor.refreshRun")(function* (threadId: ThreadId) { + const thread = yield* projections.getThreadShell(threadId); + if (thread === null) return; + yield* codeIntelligence.refreshForActiveAgent({ projectId: thread.projectId, threadId }); + }); + + const releaseSettled = Effect.fn("CodeIndexReactor.releaseSettled")(function* ( + threadId: ThreadId, + settledWorktreePath: string | null, + ) { + const shell = yield* projections.getThreadShell(threadId); + const worktreePath = shell?.worktreePath ?? settledWorktreePath; + if (worktreePath === null) return; + yield* releaseWorktreeIndex(worktreePath); + }); + + const handle = ( + stored: ApplicationStoredEvent, + ): Effect.Effect< + void, + ProjectionStore.ProjectionStoreV2Error | ProjectStore.ProjectStoreV2Error | ServerSettingsError + > => { + // Most of the stream is turn items; a plain switch keeps those free. + if (!("event" in stored)) { + return stored.type === "project.created" + ? codeIntelligence.ensureProjectIndexed(stored.payload.projectId) + : Effect.void; + } + const event = stored.event; + if (event.type === "thread.settled") { + activeRuns.delete(event.payload.id); + return releaseSettled(event.payload.id, event.payload.worktreePath); + } + if (event.type !== "run.created" && event.type !== "run.updated") return Effect.void; + const run = event.payload; + if (run.status !== "running" || activeRuns.get(run.threadId) === run.id) return Effect.void; + activeRuns.set(run.threadId, run.id); + return refreshRun(run.threadId); + }; + const guarded = (stored: ApplicationStoredEvent) => + // One unreadable thread must not end indexing for every later event. + handle(stored).pipe( + Effect.catchCause((cause) => + Effect.logWarning("failed to handle a code indexing event", { cause }), + ), + ); + + const afterSequence = yield* events.latestApplicationSequence; + yield* forkParked( + Stream.runForEach(events.streamApplicationEvents({ afterSequence }), guarded).pipe( + Effect.catchCause((cause) => Effect.logWarning("code index event stream failed", { cause })), + ), + ); + + // Every Maintenance run, "Delete now" included. The stream starts with the latest report, so + // this also sweeps at activation. + yield* forkParked( + Stream.runForEach(storageCleanup.reports, () => + sweep().pipe( + Effect.catchCause((cause) => Effect.logWarning("code index sweep failed", { cause })), + ), + ), + ); + + let lastAutoIndex = yield* autoIndexEnabled; + const changes = yield* settingsService.subscribeChanges; + yield* forkParked( + Stream.runForEach(changes, (settings) => { + const enabled = settings.autoCodeIndexEnabled; + const wasEnabled = lastAutoIndex; + lastAutoIndex = enabled; + // Turning it off stops builds already running; turning it on waits for the next turn. + return wasEnabled && !enabled ? codeIntelligence.stopBuilds : Effect.void; + }), + ); +}); + +export const layer = Layer.effectDiscard(make); diff --git a/apps/server/src/project/knowledge/CodeIntelligence.test.ts b/apps/server/src/project/knowledge/CodeIntelligence.test.ts index 9340d0697384a..aab525c5575db 100644 --- a/apps/server/src/project/knowledge/CodeIntelligence.test.ts +++ b/apps/server/src/project/knowledge/CodeIntelligence.test.ts @@ -10,20 +10,24 @@ import * as NodeURL from "node:url"; import * as NodeWorkerThreads from "node:worker_threads"; import * as NodeCryptoEffect from "@effect/platform-node/NodeCrypto"; +import * as NodeServices from "@effect/platform-node/NodeServices"; import { assert, it } from "@effect/vitest"; -import * as HostProcess from "@t3tools/shared/HostProcess"; import { EnvironmentId, OrchestrationV2AppThread, ProjectId, ThreadId, + VcsProcessExitError, type ProjectCodeIndexEvent, + type VcsListWorkspaceFilesResult, } from "@t3tools/contracts"; +import * as HostProcess from "@t3tools/shared/HostProcess"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as Exit from "effect/Exit"; import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; +import type { PlatformError } from "effect/PlatformError"; import * as Schema from "effect/Schema"; import * as Scope from "effect/Scope"; import * as Stream from "effect/Stream"; @@ -32,15 +36,29 @@ import { McpSchema, McpServer } from "effect/ai"; import * as ProjectionStore from "../../orchestration-v2/ProjectionStore.ts"; import * as ProjectStore from "../../orchestration-v2/ProjectStore.ts"; import * as ThreadManagement from "../../orchestration-v2/ThreadManagementService.ts"; +import * as ServerConfig from "../../config.ts"; import * as McpHttpServer from "../../mcp/McpHttpServer.ts"; import * as McpInvocationContext from "../../mcp/McpInvocationContext.ts"; import * as McpToolAccessTestkit from "../../mcp/McpToolAccess.testkit.ts"; +import * as ServerSettings from "../../serverSettings.ts"; +import * as VcsDriver from "../../vcs/VcsDriver.ts"; import * as WorkspaceEntries from "../../workspace/WorkspaceEntries.ts"; import * as WorkspacePaths from "../../workspace/WorkspacePaths.ts"; import { withWorkspaceLease } from "../../workspace/workspaceLease.ts"; import * as CodeIntelligence from "./CodeIntelligence.ts"; +import { indexRootFor, linkCheckoutSources } from "./codeIndexLayout.ts"; +import { pinPublishedGeneration } from "./codeIndexLocks.ts"; +import { rankListing } from "./codeIntelligenceWorker.ts"; import * as FileHistory from "./FileHistory.ts"; +const notARepository = new VcsProcessExitError({ + operation: "test", + command: "git ls-files", + cwd: "/", + exitCode: 128, + detail: "not a git repository", +}); + const projectId = ProjectId.make("project:code-intelligence"); const threadId = ThreadId.make("thread:code-intelligence"); const worktreeThreadId = ThreadId.make("thread:code-intelligence-worktree"); @@ -110,19 +128,53 @@ function withFixture( type Query = Parameters[0]["query"]; /** The published generation's directory name, read the way a fresh reader would. */ -async function publishedDirectory(root: string) { +async function publishedDirectory(indexRoot: string) { const generation = ( - await NodeFSP.readFile(NodePath.join(root, ".codegraph-t3/published"), "utf8") + await NodeFSP.readFile(NodePath.join(indexRoot, ".codegraph-t3/published"), "utf8") ).trim(); return `.codegraph-t3-g${generation}`; } -async function generationDirectories(root: string) { - return (await NodeFSP.readdir(root)) +async function generationDirectories(indexRoot: string) { + return (await NodeFSP.readdir(indexRoot).catch(() => [] as string[])) .filter((name) => /^\.codegraph-t3-g\d+$/.test(name)) .toSorted(); } +// Windows has no SIGABRT; SIGKILL ends the process just as abruptly there. +const abortProcess = (pid: number) => + Effect.map(HostProcess.isWindows, (windows) => + process.kill(pid, windows ? "SIGKILL" : "SIGABRT"), + ); + +function assertClearedIndex(indexRoot: string): void { + assert.isFalse(NodeFS.existsSync(indexRoot)); +} + +/** + * A server config whose data directory is outside the fixture, and the index root it implies for + * a checkout. + */ +interface CheckoutConfig { + readonly layer: Layer.Layer; + readonly codeIndexDir: string; + readonly indexRoot: (checkoutPath: string) => string; +} + +function checkoutConfig(): Effect.Effect { + return Effect.gen(function* () { + const context = yield* Layer.build( + ServerConfig.layerTest(process.cwd(), { prefix: "t3-code-index-" }), + ); + const config = yield* ServerConfig.ServerConfig.pipe(Effect.provide(context)); + return { + layer: Layer.succeed(ServerConfig.ServerConfig, config), + codeIndexDir: config.codeIndexDir, + indexRoot: (checkoutPath: string) => indexRootFor(config.codeIndexDir, checkoutPath), + }; + }).pipe(Effect.provide(NodeServices.layer)); +} + it.effect( "serves bounded multilingual graph queries from the authorized project and selected worktree", () => @@ -152,6 +204,8 @@ it.effect( worktreePath: worktree, } as unknown as OrchestrationV2AppThread; let includeGeneratedInventoryFiles = false; + const { layer: configLayer, indexRoot } = yield* checkoutConfig(); + const indexRootPath = indexRoot(root); const dependencies = Layer.mergeAll( Layer.mock(ProjectStore.ProjectStoreV2)({ get: () => Effect.succeedSome(project), @@ -178,13 +232,13 @@ it.effect( if ( cwd === root && includeGeneratedInventoryFiles && - (await NodeFSP.stat(NodePath.join(root, ".codegraph-t3/.gitignore")) + (await NodeFSP.stat(NodePath.join(indexRootPath, ".codegraph-t3/published")) .then(() => true) .catch(() => false)) ) { entries.push( ".codegraph-t3/.gitignore", - `${await publishedDirectory(root)}/.gitignore`, + `${await publishedDirectory(indexRootPath)}/.gitignore`, ".codegraph-t3-copy/user.txt", ); } @@ -194,6 +248,12 @@ it.effect( }; }), }), + Layer.mock(VcsDriver.VcsDriver)({ + checkpoints: {} as VcsDriver.VcsDriver["Service"]["checkpoints"], + listWorkspaceFiles: () => Effect.fail(notARepository), + }), + ServerSettings.layerTest(), + configLayer, ); const provided = Layer.provide(CodeIntelligence.layer, dependencies); const intelligence = yield* CodeIntelligence.CodeIntelligence.pipe( @@ -220,7 +280,7 @@ it.effect( })().catch((error) => parentPort.postMessage({ error: String(error) }));`, { eval: true, - workerData: { requirePath: codeGraphRequirePath, root }, + workerData: { requirePath: codeGraphRequirePath, root: indexRootPath }, env: { ...process.env, CODEGRAPH_DIR: ".codegraph-t3-g1" }, }, ); @@ -237,8 +297,8 @@ it.effect( if (code !== 0) reject(new Error(`Seed index worker exited with code ${code}.`)); }); }); - await NodeFSP.mkdir(NodePath.join(root, ".codegraph-t3")); - await NodeFSP.writeFile(NodePath.join(root, ".codegraph-t3/published"), "1"); + await NodeFSP.mkdir(NodePath.join(indexRootPath, ".codegraph-t3")); + await NodeFSP.writeFile(NodePath.join(indexRootPath, ".codegraph-t3/published"), "1"); }); const staleLinkedSymbols = yield* runQuery({ @@ -249,7 +309,7 @@ it.effect( assert.equal(staleLinkedSymbols.index?.generation, 0); // The successor is rebuilt rather than copied, so the links the seed indexed are gone. yield* refresh(); - const persistedDirectory = yield* Effect.promise(() => publishedDirectory(root)); + const persistedDirectory = yield* Effect.promise(() => publishedDirectory(indexRootPath)); assert.notEqual(persistedDirectory, ".codegraph-t3-g1"); const persistedLinkedSymbols = yield* Effect.promise( () => @@ -265,7 +325,7 @@ it.effect( })().catch((error) => parentPort.postMessage({ error: String(error) }));`, { eval: true, - workerData: { requirePath: codeGraphRequirePath, root }, + workerData: { requirePath: codeGraphRequirePath, root: indexRootPath }, env: { ...process.env, CODEGRAPH_DIR: persistedDirectory }, }, ); @@ -281,8 +341,11 @@ it.effect( ); assert.equal(persistedLinkedSymbols, 0); yield* Effect.promise(async () => { - for (const name of [".codegraph-t3", ...(await generationDirectories(root))]) { - await NodeFSP.rm(NodePath.join(root, name), { recursive: true, force: true }); + for (const name of [".codegraph-t3", ...(await generationDirectories(indexRootPath))]) { + await NodeFSP.rm(NodePath.join(indexRootPath, name), { + recursive: true, + force: true, + }); } }); @@ -296,7 +359,7 @@ it.effect( ["unindexed", "unindexed"], ); assert.deepEqual(yield* refresh(), { _tag: "Unchanged" }); - assert.deepEqual(yield* Effect.promise(() => generationDirectories(root)), [ + assert.deepEqual(yield* Effect.promise(() => generationDirectories(indexRootPath)), [ ".codegraph-t3-g1", ]); const [concurrentStatus, concurrentSearch] = yield* Effect.all( @@ -534,7 +597,13 @@ it.effect( assert.equal(repeatedStatus.coverage.filesListed, 5); assert.equal(repeatedStatus.coverage.filesUnsupported, 1); const graphDirectoryExists = yield* Effect.promise(() => - NodeFSP.stat(NodePath.join(root, ".codegraph-t3")).then(() => true), + NodeFSP.stat(NodePath.join(indexRootPath, ".codegraph-t3")).then(() => true), + ); + const checkoutControlExists = yield* Effect.promise(() => + NodeFSP.stat(NodePath.join(root, ".codegraph-t3")).then( + () => true, + () => false, + ), ); const standardIndexExists = yield* Effect.promise(() => NodeFSP.stat(NodePath.join(root, ".codegraph")).then( @@ -549,6 +618,7 @@ it.effect( ), ); assert.equal(graphDirectoryExists, true); + assert.equal(checkoutControlExists, false); assert.equal(standardIndexExists, false); assert.equal(rootGitignoreExists, false); @@ -614,31 +684,8 @@ it.effect( false, ); assert.equal(invalidUtf8.coverage.state, "stale"); - // Settles the build that query started before the index directory is replaced. + // Settles the build that query started. yield* refresh(); - - yield* Effect.promise(async () => { - await NodeFSP.rm(NodePath.join(root, ".codegraph-t3"), { - recursive: true, - force: true, - }); - await NodeFSP.rm(NodePath.join(worktree, ".codegraph-t3"), { - recursive: true, - force: true, - }); - await NodeFSP.symlink(worktree, NodePath.join(root, ".codegraph-t3")); - }); - const indexPathError = yield* Effect.flip(runQuery({ operation: "status" })); - assert.equal(indexPathError._tag, "CodeIntelligencePathOutsideCheckout"); - assert.equal( - yield* Effect.promise(() => - NodeFSP.stat(NodePath.join(worktree, ".codegraph-t3")).then( - () => true, - () => false, - ), - ), - false, - ); }), ), ); @@ -690,7 +737,7 @@ it.effect("reports index freshness against the checkout and walks usages to reac return git("rev-parse", "HEAD"); }); const gate = buildGate(); - const { run, refresh } = yield* checkoutService( + const { run, refresh, indexRoot } = yield* checkoutService( fixture.root, () => ["src/defs.ts", "src/use.ts", "src/defs.test.ts"], gate, @@ -698,7 +745,7 @@ it.effect("reports index freshness against the checkout and walks usages to reac // Freshness never indexes: an unindexed checkout stays unindexed. assert.equal((yield* run({ operation: "freshness" })).index?.state, "unindexed"); - assert.isFalse(NodeFS.existsSync(NodePath.join(fixture.root, ".codegraph-t3"))); + assert.isFalse(NodeFS.existsSync(indexRoot(fixture.root))); assert.deepEqual(yield* refresh(), { _tag: "Published", generation: 1 }); const indexed = yield* run({ operation: "status" }); @@ -779,12 +826,25 @@ it.effect("reports index freshness against the checkout and walks usages to reac function checkoutService( root: string, - listed: () => ReadonlyArray, + listed: (cwd?: string) => ReadonlyArray, { beforeListing = () => Effect.void, + repository = false, + truncated = false, + autoCodeIndexEnabled = true, + worktree, + config, ...serviceOptions }: CodeIntelligence.CodeIntelligenceOptions & { readonly beforeListing?: () => Effect.Effect; + /** Lists through git, as a real repository does, instead of the file finder. */ + readonly repository?: boolean; + readonly truncated?: boolean; + readonly autoCodeIndexEnabled?: boolean; + /** A thread whose checkout is `path`, as a worktree thread resolves. */ + readonly worktree?: { readonly threadId: ThreadId; readonly path: string }; + /** Shares one data directory across services, as one server would. */ + readonly config?: CheckoutConfig; } = {}, ) { const project = { @@ -792,45 +852,110 @@ function checkoutService( workspaceRoot: root, deletedAt: null, } as unknown as ProjectStore.ProjectRow; - const provided = Layer.provide( - CodeIntelligence.layerWith(serviceOptions), - Layer.mergeAll( - Layer.mock(ProjectStore.ProjectStoreV2)({ get: () => Effect.succeedSome(project) }), - Layer.mock(ProjectionStore.ProjectionStoreV2)({ getThread: () => Effect.die("unused") }), - Layer.mock(WorkspacePaths.WorkspacePaths)({ - normalizeWorkspaceRoot: () => Effect.succeed(root), - resolveRelativePathWithinRoot: ({ relativePath }) => - Effect.succeed({ absolutePath: relativePath, relativePath }), - }), - Layer.mock(WorkspaceEntries.WorkspaceEntries)({ - refresh: () => Effect.void, - list: () => - Effect.suspend(beforeListing).pipe( - Effect.andThen( - Effect.sync(() => ({ - entries: listed().map((path) => ({ path, kind: "file" as const })), - truncated: false, - })), - ), - ), - }), - ), - ); + const worktreeThread = { + id: worktree?.threadId ?? worktreeThreadId, + projectId, + worktreePath: worktree?.path ?? root, + } as unknown as OrchestrationV2AppThread; return Effect.gen(function* () { + const shared = config ?? (yield* checkoutConfig()); + const { layer: configLayer, indexRoot, codeIndexDir } = shared; + const provided = Layer.provide( + CodeIntelligence.layerWith(serviceOptions), + Layer.mergeAll( + Layer.mock(ProjectStore.ProjectStoreV2)({ get: () => Effect.succeedSome(project) }), + Layer.mock(ProjectionStore.ProjectionStoreV2)({ + getThread: (requestedThreadId) => + worktree !== undefined && requestedThreadId === worktree.threadId + ? Effect.succeed(worktreeThread) + : Effect.die("unused"), + }), + Layer.mock(WorkspacePaths.WorkspacePaths)({ + normalizeWorkspaceRoot: (workspaceRoot) => + NodeFS.existsSync(workspaceRoot) + ? Effect.succeed(workspaceRoot) + : Effect.fail( + new WorkspacePaths.WorkspaceRootNotExistsError({ + workspaceRoot, + normalizedWorkspaceRoot: workspaceRoot, + }), + ), + resolveRelativePathWithinRoot: ({ relativePath }) => + Effect.succeed({ absolutePath: relativePath, relativePath }), + }), + Layer.mock(WorkspaceEntries.WorkspaceEntries)({ + refresh: () => Effect.void, + list: ({ cwd }) => + Effect.suspend(beforeListing).pipe( + Effect.andThen( + Effect.sync(() => ({ + entries: listed(cwd).map((path) => ({ path, kind: "file" as const })), + truncated, + })), + ), + ), + }), + Layer.mock(VcsDriver.VcsDriver)({ + checkpoints: {} as VcsDriver.VcsDriver["Service"]["checkpoints"], + isInsideWorkTree: () => Effect.succeed(repository), + listWorkspaceFiles: (cwd) => + repository + ? Effect.sync( + () => + ({ + paths: listed(cwd), + truncated, + }) as unknown as VcsListWorkspaceFilesResult, + ) + : Effect.fail(notARepository), + }), + ServerSettings.layerTest({ autoCodeIndexEnabled }), + configLayer, + ), + ); const intelligence = yield* CodeIntelligence.CodeIntelligence.pipe( Effect.provide(yield* Layer.build(provided)), ); return { - run: (query: Query) => intelligence.query({ projectId, query }), - refresh: () => intelligence.refresh({ projectId }), - subscribe: () => intelligence.subscribeIndex({ projectId }), + indexRoot, + codeIndexDir, + hasCheckoutIndex: (checkoutPath: string) => intelligence.hasCheckoutIndex(checkoutPath), + removeOrphanedIndexes: (isKnown: (checkoutPath: string) => boolean) => + intelligence.removeOrphanedIndexes(isKnown), + run: (query: Query, selectedThreadId?: ThreadId) => + intelligence.query({ + projectId, + ...(selectedThreadId ? { threadId: selectedThreadId } : {}), + query, + }), + refresh: (selectedThreadId?: ThreadId) => + intelligence.refresh({ + projectId, + ...(selectedThreadId ? { threadId: selectedThreadId } : {}), + }), + subscribe: (selectedThreadId?: ThreadId) => + intelligence.subscribeIndex({ + projectId, + ...(selectedThreadId ? { threadId: selectedThreadId } : {}), + }), + ensureProjectIndexed: () => intelligence.ensureProjectIndexed(projectId), + refreshForActiveAgent: (selectedThreadId?: ThreadId) => + intelligence.refreshForActiveAgent({ + projectId, + ...(selectedThreadId ? { threadId: selectedThreadId } : {}), + }), + refreshIfIndexed: (cwd: string) => intelligence.refreshIfIndexed(cwd), + removeCheckoutIndex: (checkoutPath: string) => intelligence.removeCheckoutIndex(checkoutPath), + removeLegacyIndexes: (checkoutPaths: ReadonlyArray) => + intelligence.removeLegacyIndexes(checkoutPaths), + stopBuilds: intelligence.stopBuilds, }; }); } -function indexedSymbolCount(root: string, name: string) { +function indexedSymbolCount(indexRoot: string, name: string) { return Effect.promise(async () => { - const indexDirectory = await publishedDirectory(root); + const indexDirectory = await publishedDirectory(indexRoot); return await new Promise((resolve, reject) => { const inspect = new NodeWorkerThreads.Worker( `const { parentPort, workerData } = require("node:worker_threads"); @@ -843,7 +968,7 @@ function indexedSymbolCount(root: string, name: string) { })().catch((error) => parentPort.postMessage({ error: String(error) }));`, { eval: true, - workerData: { requirePath: codeGraphRequirePath, root, name }, + workerData: { requirePath: codeGraphRequirePath, root: indexRoot, name }, env: { ...process.env, CODEGRAPH_DIR: indexDirectory }, }, ); @@ -856,77 +981,6 @@ function indexedSymbolCount(root: string, name: string) { }); } -it.effect( - "refuses a linked index directory or child instead of reading or writing through it", - () => - withFixture((fixture) => - Effect.gen(function* () { - const victim = NodePath.join(fixture.worktree, "victim"); - const control = NodePath.join(fixture.root, ".codegraph-t3"); - const generation = NodePath.join(fixture.root, ".codegraph-t3-g1"); - const { run, refresh } = yield* checkoutService(fixture.root, () => [ - "src/defs.ts", - "src/use.ts", - ]); - const expectRefused = (label: string) => - Effect.gen(function* () { - const result = yield* Effect.result(run({ operation: "status" })); - assert.equal(result._tag, "Failure", label); - if (result._tag === "Failure") { - assert.equal(result.failure._tag, "CodeIntelligencePathOutsideCheckout", label); - } - assert.deepEqual( - yield* refresh(), - { _tag: "Failed", reason: "index-outside-checkout" }, - label, - ); - assert.equal( - yield* Effect.promise(() => NodeFSP.readFile(victim, "utf8")), - "untouched", - label, - ); - }); - yield* Effect.promise(async () => { - await NodeFSP.writeFile(victim, "untouched"); - await NodeFSP.mkdir(control); - }); - for (const child of ["published", ".gitignore"]) { - const link = NodePath.join(control, child); - yield* Effect.promise(() => NodeFSP.symlink(victim, link)); - yield* expectRefused(`control ${child}`); - yield* Effect.promise(() => NodeFSP.rm(link)); - } - yield* Effect.promise(async () => { - await NodeFSP.rm(control, { recursive: true }); - await NodeFSP.symlink(fixture.worktree, control); - }); - yield* expectRefused("control directory"); - yield* Effect.promise(() => NodeFSP.rm(control)); - - assert.deepEqual(yield* refresh(), { _tag: "Published", generation: 1 }); - for (const child of ["t3-index.json", "codegraph.db-wal"]) { - const link = NodePath.join(generation, child); - const kept = NodePath.join(fixture.worktree, `${child}.kept`); - yield* Effect.promise(async () => { - await NodeFSP.rename(link, kept).catch(() => undefined); - await NodeFSP.symlink(victim, link); - }); - yield* expectRefused(`generation ${child}`); - yield* Effect.promise(async () => { - await NodeFSP.rm(link); - await NodeFSP.rename(kept, link).catch(() => undefined); - }); - } - const keptGeneration = NodePath.join(fixture.worktree, "generation.kept"); - yield* Effect.promise(async () => { - await NodeFSP.rename(generation, keptGeneration); - await NodeFSP.symlink(keptGeneration, generation); - }); - yield* expectRefused("generation directory"); - }), - ), -); - it.effect("never widens an empty canonical allowlist into a full checkout scan", () => withFixture((fixture) => Effect.gen(function* () { @@ -936,12 +990,14 @@ it.effect("never widens an empty canonical allowlist into a full checkout scan", NodePath.join(fixture.root, "src/linked-work.ts"), ), ); - const { run, refresh } = yield* checkoutService(fixture.root, () => ["src/linked-work.ts"]); + const { run, refresh, indexRoot } = yield* checkoutService(fixture.root, () => [ + "src/linked-work.ts", + ]); assert.deepInclude(yield* refresh(), { _tag: "Published" }); const status = yield* run({ operation: "status" }); assert.equal(status.coverage.filesIndexed, 0); - assert.equal(yield* indexedSymbolCount(fixture.root, "workOnly"), 0); - assert.equal(yield* indexedSymbolCount(fixture.root, "greet"), 0); + assert.equal(yield* indexedSymbolCount(indexRoot(fixture.root), "workOnly"), 0); + assert.equal(yield* indexedSymbolCount(indexRoot(fixture.root), "greet"), 0); }), ), ); @@ -1022,13 +1078,19 @@ it.effect("keeps skipped build output partial while detecting newly added source it.effect("reports freshness coverage with the same classification as a query", () => withFixture((fixture) => Effect.gen(function* () { - yield* Effect.promise(() => - NodeFSP.writeFile(NodePath.join(fixture.root, "README.md"), "# Fixture\n"), - ); + yield* Effect.promise(async () => { + await NodeFSP.writeFile(NodePath.join(fixture.root, "README.md"), "# Fixture\n"); + await NodeFSP.writeFile(NodePath.join(fixture.root, "ci.yaml"), "steps: []\n"); + await NodeFSP.writeFile(NodePath.join(fixture.worktree, "README.md"), "# Fixture\n"); + }); const { run, refresh } = yield* checkoutService(fixture.root, () => [ "src/defs.ts", "src/use.ts", "README.md", + "ci.yaml", + // Git lists a tracked file deleted from the working tree, and a submodule as a path. + "src/deleted.ts", + "src", ]); yield* refresh(); const status = yield* run({ operation: "status" }); @@ -1039,14 +1101,417 @@ it.effect("reports freshness coverage with the same classification as a query", filesSkipped: result.coverage.filesSkipped, filesFailed: result.coverage.filesFailed, }); + // Docs and assets fill any repository, so they are counted but leave no gap. A file the SDK + // indexed anyway, such as YAML, is not also unsupported. assert.deepEqual(counts(status), { - state: "partial", + state: "complete", filesUnsupported: 1, filesSkipped: 0, filesFailed: 0, }); assert.deepEqual(counts(freshness), counts(status)); - assert.equal(freshness.index?.state, "partial"); + assert.equal(freshness.index?.state, "ready"); + + // With nothing else to index, the same files do leave the index empty and incomplete. + const docsOnly = yield* checkoutService(fixture.worktree, () => ["README.md"]); + yield* docsOnly.refresh(); + const docsOnlyFreshness = yield* docsOnly.run({ operation: "freshness" }); + assert.equal(docsOnlyFreshness.coverage.state, "partial"); + assert.equal(docsOnlyFreshness.index?.state, "partial"); + + // Source in a language the SDK cannot read is a gap, however much else is indexed. + yield* Effect.promise(() => + NodeFSP.writeFile(NodePath.join(fixture.root, "worker.ex"), "defmodule Worker do\nend\n"), + ); + const mixed = yield* checkoutService(fixture.root, () => ["src/defs.ts", "worker.ex"]); + yield* mixed.refresh(); + const mixedFreshness = yield* mixed.run({ operation: "freshness" }); + assert.equal(mixedFreshness.coverage.state, "partial"); + assert.equal(mixedFreshness.index?.state, "partial"); + + // One stray script among many sources is not. + const sources = yield* Effect.promise(() => writeGeneratedSources(fixture.root, 20)); + const stray = yield* checkoutService(fixture.root, () => [...sources, "worker.ex"]); + yield* stray.refresh(); + assert.equal((yield* stray.run({ operation: "freshness" })).index?.state, "ready"); + }), + ), +); + +it("spends an over-cap listing on first-party sources before anything else", () => { + assert.deepEqual( + rankListing( + [ + ".repos/lib/notes.md", + ".repos/lib/vendored.ts", + "README.md", + "apps/.local/generated.ts", + "apps/server.ts", + "packages/shared.ts", + ], + (path) => path.endsWith(".ts"), + ), + [ + "apps/server.ts", + "packages/shared.ts", + ".repos/lib/vendored.ts", + "apps/.local/generated.ts", + "README.md", + ".repos/lib/notes.md", + ], + ); +}); + +it.effect("starts over from an index that spent a truncated listing on what sorted first", () => + withFixture((fixture) => + Effect.gen(function* () { + yield* Effect.promise(() => + NodeFSP.writeFile( + NodePath.join(fixture.root, "src/vendored.ts"), + "export function vendoredOnly() {}\n", + ), + ); + let listed = ["src/vendored.ts", "src/defs.ts"]; + const { refresh, indexRoot } = yield* checkoutService(fixture.root, () => listed, { + truncated: true, + }); + assert.deepEqual(yield* refresh(), { _tag: "Published", generation: 1 }); + assert.equal(yield* indexedSymbolCount(indexRoot(fixture.root), "vendoredOnly"), 1); + + yield* Effect.promise(async () => { + const root = indexRoot(fixture.root); + const metaPath = NodePath.join(await publishedDirectory(root), "t3-index.json"); + const absolute = NodePath.join(root, metaPath); + const { ranked: _ranked, ...legacy } = JSON.parse(await NodeFSP.readFile(absolute, "utf8")); + await NodeFSP.writeFile(absolute, JSON.stringify(legacy)); + }); + listed = ["src/defs.ts"]; + assert.deepEqual(yield* refresh(), { _tag: "Published", generation: 2 }); + assert.equal(yield* indexedSymbolCount(indexRoot(fixture.root), "vendoredOnly"), 0); + assert.equal(yield* indexedSymbolCount(indexRoot(fixture.root), "greet"), 1); + }), + ), +); + +it.effect("starts a project's first index when a client watches it, and only its first", () => + withFixture((fixture) => + Effect.gen(function* () { + const gate = buildGate(); + const { run, subscribe } = yield* checkoutService(fixture.root, () => ["src/defs.ts"], { + ...gate, + repository: true, + }); + const next = (events: Stream.Stream) => + events.pipe(Stream.take(1), Stream.runCollect); + + // A watcher does not retry a failed first build, or a persistent failure would repeat for + // every client that looks. + const held = yield* gate.holdNext; + const events = yield* subscribe(); + assert.deepInclude((yield* run({ operation: "freshness" })).index, { + state: "unindexed", + refresh: "building", + }); + const buildPid = yield* Deferred.await(held.entered); + // Nobody asked for this build, so it yields to work someone is waiting on. + assert.equal(NodeOS.getPriority(buildPid), NodeOS.constants.priority.PRIORITY_BELOW_NORMAL); + // The first build announces itself before it starts, so a watching client reads the badge + // again and shows Indexing while there is no generation to answer from. + assert.deepEqual(yield* next(events), [{ outcome: "published", generation: null }]); + process.kill(buildPid, "SIGKILL"); + assert.deepEqual(yield* next(events), [{ outcome: "failed", generation: null }]); + const firstBuildFailed = { state: "unindexed", refresh: "failed" } as const; + // The event precedes the build's bookkeeping; this answer proves that bookkeeping finished. + assert.deepInclude((yield* run({ operation: "freshness" })).index, firstBuildFailed); + yield* subscribe().pipe(Effect.asVoid); + assert.deepInclude((yield* run({ operation: "freshness" })).index, firstBuildFailed); + + // A code query still asks for one, and the retry publishes without announcing again. + yield* run({ operation: "symbols", name: "greet" }); + assert.deepEqual(yield* next(events), [{ outcome: "published", generation: 1 }]); + assert.equal((yield* run({ operation: "freshness" })).index?.state, "ready"); + + // Once indexed, watching observes; refreshes stay with queries and finished turns. + yield* editDefinition(fixture.root, "edited"); + yield* subscribe().pipe(Effect.asVoid); + const stale = (yield* run({ operation: "freshness" })).index; + assert.equal(stale?.state, "stale"); + assert.notProperty(stale, "refresh"); + + // A folder outside a repository is not indexed unasked. + const plainFolder = yield* checkoutService(fixture.worktree, () => ["src/work.ts"]); + yield* plainFolder.subscribe().pipe(Effect.asVoid); + const unindexed = (yield* plainFolder.run({ operation: "freshness" })).index; + assert.equal(unindexed?.state, "unindexed"); + assert.notProperty(unindexed, "refresh"); + // Nor for an agent: a build its request started would leave this one nothing to publish. + yield* plainFolder.refreshForActiveAgent(); + assert.deepEqual(yield* plainFolder.refresh(), { _tag: "Published", generation: 1 }); + }), + ), +); + +it.effect("indexes a worktree for its active agent and never merely because it is viewed", () => + withFixture((fixture) => + Effect.gen(function* () { + const worktree = { threadId: worktreeThreadId, path: fixture.worktree }; + const { run, refresh, subscribe, refreshForActiveAgent, indexRoot } = yield* checkoutService( + fixture.root, + (cwd) => (cwd === fixture.worktree ? ["src/work.ts"] : ["src/defs.ts"]), + { worktree, repository: true }, + ); + const worktreeFreshness = () => + run({ operation: "freshness" }, worktreeThreadId).pipe( + Effect.map((result) => result.index), + ); + + // Viewing a worktree thread subscribes to its events but builds nothing. + yield* subscribe(worktreeThreadId).pipe(Effect.asVoid); + assert.equal((yield* worktreeFreshness())?.state, "unindexed"); + assert.isFalse(NodeFS.existsSync(indexRoot(fixture.worktree))); + + // An agent starting in it indexes it. + yield* refreshForActiveAgent(worktreeThreadId); + // The request returns at once. Its build publishes, leaving this one nothing to do. + assert.deepEqual(yield* refresh(worktreeThreadId), { _tag: "Unchanged" }); + const indexed = yield* worktreeFreshness(); + assert.equal(indexed?.state, "ready"); + assert.equal(indexed?.generation, 1); + const symbols = yield* run({ operation: "symbols", name: "workOnly" }, worktreeThreadId); + assert.ok(symbols.nodes.some((node) => node.path === "src/work.ts")); + }), + ), +); + +it.effect("leaves a worktree index stale until its agent acts again", () => + withFixture((fixture) => + Effect.gen(function* () { + const worktree = { threadId: worktreeThreadId, path: fixture.worktree }; + const { run, refresh, refreshForActiveAgent } = yield* checkoutService( + fixture.root, + (cwd) => (cwd === fixture.worktree ? ["src/work.ts"] : ["src/defs.ts"]), + { worktree }, + ); + const freshness = () => + run({ operation: "freshness" }, worktreeThreadId).pipe( + Effect.map((result) => result.index), + ); + + yield* refreshForActiveAgent(worktreeThreadId); + yield* refresh(worktreeThreadId); + assert.equal((yield* freshness())?.generation, 1); + + // Nothing refreshes an idle worktree. + yield* Effect.promise(() => + NodeFSP.writeFile( + NodePath.join(fixture.worktree, "src/work.ts"), + "export function workOnly() { return 3; }\n", + ), + ); + const idle = yield* freshness(); + assert.equal(idle?.state, "stale"); + assert.notProperty(idle, "refresh"); + + // The next turn's request publishes the successor. + yield* refreshForActiveAgent(worktreeThreadId); + yield* refresh(worktreeThreadId); + const refreshed = yield* freshness(); + assert.equal(refreshed?.state, "ready"); + assert.equal(refreshed?.generation, 2); + }), + ), +); + +it.effect("indexes only when something asks while automatic indexing is off", () => + withFixture((fixture) => + Effect.gen(function* () { + const worktree = { threadId: worktreeThreadId, path: fixture.worktree }; + const { + run, + refresh, + subscribe, + ensureProjectIndexed, + refreshForActiveAgent, + refreshIfIndexed, + indexRoot, + } = yield* checkoutService( + fixture.root, + (cwd) => (cwd === fixture.worktree ? ["src/work.ts"] : ["src/defs.ts"]), + { worktree, repository: true, autoCodeIndexEnabled: false }, + ); + const freshness = (threadId?: ThreadId) => + run({ operation: "freshness" }, threadId).pipe(Effect.map((result) => result.index)); + const noIndex = () => + Effect.sync( + () => + !NodeFS.existsSync(indexRoot(fixture.root)) && + !NodeFS.existsSync(indexRoot(fixture.worktree)), + ); + + yield* ensureProjectIndexed(); + yield* refreshForActiveAgent(); + yield* refreshForActiveAgent(worktreeThreadId); + yield* subscribe().pipe(Effect.asVoid); + yield* refreshIfIndexed(fixture.root); + assert.isTrue(yield* noIndex()); + assert.equal((yield* freshness())?.state, "unindexed"); + assert.equal((yield* freshness(worktreeThreadId))?.state, "unindexed"); + + // A build any of those started would run first, leaving these nothing to publish. + assert.deepEqual(yield* refresh(), { _tag: "Published", generation: 1 }); + assert.deepEqual(yield* refresh(worktreeThreadId), { _tag: "Published", generation: 1 }); + + // Nor does an existing index refresh on its own. + yield* editDefinition(fixture.root, "edited"); + yield* refreshForActiveAgent(); + yield* refreshIfIndexed(fixture.root); + assert.deepEqual(yield* refresh(), { _tag: "Published", generation: 2 }); + + // A code question always works. + const answer = yield* run({ operation: "symbols", name: "greet" }); + assert.ok(answer.nodes.some((node) => node.path === "src/defs.ts")); + }), + ), +); + +it.effect("rejects Windows network checkouts before creating partial source links", (context) => + withFixture((fixture) => + Effect.gen(function* () { + if (!(yield* HostProcess.isWindows)) return context.skip(); + for (const checkout of [ + "\\\\server\\share\\repo", + "//server/share/repo", + "\\\\?\\UNC\\server\\share\\repo", + ]) { + assert.throws( + () => linkCheckoutSources(fixture.worktree, checkout, ["src/defs.ts"]), + /network checkout/, + ); + } + assert.deepEqual(NodeFS.readdirSync(fixture.worktree), ["src"]); + }), + ), +); + +it.effect("shares one index between Windows project and thread path case variants", (context) => + withFixture((fixture) => + Effect.gen(function* () { + if (!(yield* HostProcess.isWindows)) return context.skip(); + const { refresh, indexRoot, codeIndexDir } = yield* checkoutService( + fixture.root.toLowerCase(), + () => ["src/defs.ts"], + { + worktree: { + threadId: worktreeThreadId, + path: fixture.root.toUpperCase().replaceAll("\\", "/"), + }, + }, + ); + assert.deepEqual(yield* refresh(), { _tag: "Published", generation: 1 }); + assert.deepEqual(yield* refresh(worktreeThreadId), { _tag: "Unchanged" }); + assert.equal(indexRoot(fixture.root.toLowerCase()), indexRoot(fixture.root.toUpperCase())); + assert.lengthOf(NodeFS.readdirSync(codeIndexDir), 1); + }), + ), +); + +it.effect("stops a Windows build removed through a different path case", (context) => + withFixture((fixture) => + Effect.gen(function* () { + if (!(yield* HostProcess.isWindows)) return context.skip(); + const gate = buildGate(); + const { refresh, removeCheckoutIndex, indexRoot } = yield* checkoutService( + fixture.root.toLowerCase(), + () => ["src/defs.ts"], + gate, + ); + const held = yield* gate.holdNext; + const building = yield* refresh().pipe(Effect.forkScoped); + const pid = yield* Deferred.await(held.entered); + yield* removeCheckoutIndex(fixture.root.toUpperCase()); + assert.isFalse(isProcessAlive(pid)); + yield* Fiber.await(building); + assertClearedIndex(indexRoot(fixture.root.toLowerCase())); + assert.isTrue(NodeFS.existsSync(NodePath.join(fixture.root, "src/defs.ts"))); + }), + ), +); + +it.effect("removes a checkout's index while a build is in flight", () => + withFixture((fixture) => + Effect.gen(function* () { + const gate = buildGate(); + const { removeCheckoutIndex, refresh, indexRoot } = yield* checkoutService( + fixture.root, + () => ["src/defs.ts"], + gate, + ); + const held = yield* gate.holdNext; + const building = yield* refresh().pipe(Effect.forkScoped); + const pid = yield* Deferred.await(held.entered); + yield* removeCheckoutIndex(fixture.root); + assert.isFalse(isProcessAlive(pid), "the build process is gone once removal returns"); + yield* Fiber.await(building); + assertClearedIndex(indexRoot(fixture.root)); + assert.deepEqual( + yield* Effect.promise(() => generationDirectories(indexRoot(fixture.root))), + [], + ); + }), + ), +); + +it.effect("keeps a generation a reader pins while removing the rest of the index", () => + withFixture((fixture) => + Effect.gen(function* () { + const { run, refresh, removeCheckoutIndex, indexRoot } = yield* checkoutService( + fixture.root, + () => ["src/defs.ts"], + ); + const root = indexRoot(fixture.root); + assert.deepEqual(yield* refresh(), { _tag: "Published", generation: 1 }); + const pin = yield* Effect.sync(() => pinPublishedGeneration(root)); + assert.equal(pin.generation, 1); + + yield* removeCheckoutIndex(fixture.root); + // The pointer goes first, so queries read unindexed even though the generation survives. + assert.isFalse(NodeFS.existsSync(NodePath.join(root, ".codegraph-t3", "published"))); + assert.deepEqual(yield* Effect.promise(() => generationDirectories(root)), [ + ".codegraph-t3-g1", + ]); + assert.equal((yield* run({ operation: "freshness" })).index?.state, "unindexed"); + + // The next removal, with nobody reading, clears it. + yield* Effect.sync(pin.release); + yield* removeCheckoutIndex(fixture.root); + assert.deepEqual(yield* Effect.promise(() => generationDirectories(root)), []); + assertClearedIndex(root); + }), + ), +); + +it.effect("leaves a checkout another server process is building alone", () => + withFixture((fixture) => + Effect.gen(function* () { + const { refresh, removeCheckoutIndex, indexRoot } = yield* checkoutService( + fixture.root, + () => ["src/defs.ts"], + ); + const root = indexRoot(fixture.root); + const other = yield* otherServerProcess(root); + assert.deepEqual(yield* refresh(), { _tag: "Published", generation: 1 }); + assert.equal(yield* other.send("lock"), "locked"); + + yield* removeCheckoutIndex(fixture.root); + assert.deepEqual(yield* Effect.promise(() => generationDirectories(root)), [ + ".codegraph-t3-g1", + ]); + assert.equal( + yield* Effect.promise(() => + NodeFSP.readFile(NodePath.join(root, ".codegraph-t3/published"), "utf8"), + ), + "1", + ); }), ), ); @@ -1159,8 +1624,9 @@ it.effect("stops a build when the service shuts down and never publishes the cut const afterShutdown = yield* restarted.run({ operation: "freshness" }); assert.equal(afterShutdown.index?.state, "unindexed"); assert.deepInclude(yield* restarted.refresh(), { _tag: "Published" }); - assert.deepEqual(yield* Effect.promise(() => generationDirectories(fixture.root)), [ - yield* Effect.promise(() => publishedDirectory(fixture.root)), + const root = restarted.indexRoot(fixture.root); + assert.deepEqual(yield* Effect.promise(() => generationDirectories(root)), [ + yield* Effect.promise(() => publishedDirectory(root)), ]); const recovered = yield* restarted.run({ operation: "status" }); assert.equal(recovered.coverage.filesIndexed, listed.length); @@ -1175,13 +1641,16 @@ it.effect("fails the build, not the server, when the analysis process aborts", ( const files = yield* Effect.promise(() => writeGeneratedSources(fixture.root, 4)); const gate = buildGate(); const held = yield* gate.holdNext; - const { refresh } = yield* checkoutService(fixture.root, () => files, gate); + const { refresh, indexRoot } = yield* checkoutService(fixture.root, () => files, gate); const building = yield* refresh().pipe(Effect.forkScoped); const pid = yield* Deferred.await(held.entered); assert.notEqual(pid, process.pid, "analysis runs outside the server process"); - process.kill(pid, "SIGABRT"); + yield* abortProcess(pid); assert.deepEqual(yield* Fiber.join(building), { _tag: "Failed", reason: "worker-failed" }); - assert.deepEqual(yield* Effect.promise(() => generationDirectories(fixture.root)), []); + assert.deepEqual( + yield* Effect.promise(() => generationDirectories(indexRoot(fixture.root))), + [], + ); }), ), ); @@ -1213,13 +1682,36 @@ it.effect("keeps the index out of the checkout's git status", () => withFixture((fixture) => Effect.gen(function* () { const git = (...args: ReadonlyArray) => - NodeChildProcess.execFileSync("git", args, { cwd: fixture.root, encoding: "utf8" }); + NodeChildProcess.execFileSync( + "git", + ["-c", "user.email=t3@example.com", "-c", "user.name=T3", ...args], + { cwd: fixture.root, encoding: "utf8" }, + ); git("init", "--quiet"); + git("add", "--all"); + git("commit", "--quiet", "-m", "fixture"); const gate = listingGate(); - const { run, refresh } = yield* checkoutService(fixture.root, () => ["src/defs.ts"], gate); - assert.deepEqual(yield* refresh(), { _tag: "Published", generation: 1 }); - // A reader keeps generation 1 while generation 2 is published, so both are on disk. - const held = yield* gate.holdNext; + const { run, refresh, indexRoot } = yield* checkoutService( + fixture.root, + () => ["src/defs.ts"], + gate, + ); + const root = indexRoot(fixture.root); + const before = (yield* Effect.promise(() => NodeFSP.readdir(fixture.root))).toSorted(); + assert.deepEqual(yield* refresh(), { _tag: "Published", generation: 1 }); + assert.ok((yield* run({ operation: "symbols", name: "greet" })).nodes.length > 0); + // Drain the query's background refresh before holding a reader and editing its source. + yield* refresh(); + + // Nothing the index owns is in the checkout: no new entries, and git sees nothing. + assert.deepEqual( + (yield* Effect.promise(() => NodeFSP.readdir(fixture.root))).toSorted(), + before, + ); + assert.equal(git("status", "--ignored", "--short"), ""); + + // A reader keeps generation 1 while generation 2 is published, so both are on disk. + const held = yield* gate.holdNext; const reader = yield* run({ operation: "freshness" }).pipe(Effect.forkScoped); yield* Deferred.await(held.entered); yield* Effect.promise(() => @@ -1229,16 +1721,14 @@ it.effect("keeps the index out of the checkout's git status", () => ), ); assert.deepEqual(yield* refresh(), { _tag: "Published", generation: 2 }); - assert.deepEqual(yield* Effect.promise(() => generationDirectories(fixture.root)), [ + assert.deepEqual(yield* Effect.promise(() => generationDirectories(root)), [ ".codegraph-t3-g1", ".codegraph-t3-g2", ]); - const untracked = git("status", "--porcelain", "--untracked-files=all") - .split("\n") - .filter((line) => /\.codegraph-t3(-g\d+)?\//.test(line)); - assert.deepEqual(untracked, []); yield* Deferred.succeed(held.release, undefined); yield* Fiber.join(reader); + // Even with two generations on disk, only the edited source shows. + assert.equal(git("status", "--ignored", "--short"), " M src/defs.ts\n"); }), ), ); @@ -1276,27 +1766,388 @@ it.effect("returns index not ready for a cold checkout without synchronizing sou ), ); -it.effect("removes the index an older version kept in the control directory", () => +it.effect("removes the index folders an older version left in the checkout", () => withFixture((fixture) => Effect.gen(function* () { const control = NodePath.join(fixture.root, ".codegraph-t3"); - yield* Effect.promise(async () => { + const legacyGeneration = NodePath.join(fixture.root, ".codegraph-t3-g3"); + const userGeneration = NodePath.join(fixture.root, ".codegraph-t3-g4"); + const before = yield* Effect.promise(async () => { + await NodeFSP.writeFile(NodePath.join(fixture.root, "top.ts"), "export const top = 1;\n"); + await NodeFSP.symlink("src", NodePath.join(fixture.root, "alias")); + // A folder that merely shares the name is the user's. + await NodeFSP.mkdir(userGeneration); + await NodeFSP.writeFile(NodePath.join(userGeneration, "notes.txt"), "mine"); + const entries = (await NodeFSP.readdir(fixture.root)).toSorted(); await NodeFSP.mkdir(control); await NodeFSP.writeFile(NodePath.join(control, "codegraph.db"), "legacy index"); await NodeFSP.writeFile(NodePath.join(control, "t3-index.json"), "{}"); + await NodeFSP.mkdir(legacyGeneration); + await NodeFSP.writeFile(NodePath.join(legacyGeneration, "codegraph.db"), "legacy"); + return entries; }); const { refresh } = yield* checkoutService(fixture.root, () => ["src/defs.ts"]); assert.deepEqual(yield* refresh(), { _tag: "Published", generation: 1 }); - assert.deepEqual((yield* Effect.promise(() => NodeFSP.readdir(control))).toSorted(), [ - ".gitignore", - "build.lock", - "building.lock", - "published", - ]); + // Every source entry is still there, and nothing of T3's is. + assert.deepEqual( + (yield* Effect.promise(() => NodeFSP.readdir(fixture.root))).toSorted(), + before, + ); + assert.isTrue(NodeFS.lstatSync(NodePath.join(fixture.root, "alias")).isSymbolicLink()); + assert.equal( + yield* Effect.promise(() => + NodeFSP.readFile(NodePath.join(userGeneration, "notes.txt"), "utf8"), + ), + "mine", + ); + }), + ), +); + +it.effect("finds and removes an older version's folders in a checkout with no index yet", () => + withFixture((fixture) => + Effect.gen(function* () { + const control = NodePath.join(fixture.root, ".codegraph-t3"); + const { hasCheckoutIndex, removeCheckoutIndex, indexRoot } = yield* checkoutService( + fixture.root, + () => ["src/defs.ts"], + ); + assert.isFalse(yield* hasCheckoutIndex(fixture.root)); + yield* Effect.promise(async () => { + await NodeFSP.mkdir(control); + await NodeFSP.writeFile(NodePath.join(control, "published"), "1"); + }); + assert.isFalse(NodeFS.existsSync(indexRoot(fixture.root))); + assert.isTrue(yield* hasCheckoutIndex(fixture.root)); + + // Settling a thread takes this path, so a worktree nothing rebuilds is still cleaned up. + yield* removeCheckoutIndex(fixture.root); + assert.isFalse(NodeFS.existsSync(control)); + assert.isFalse(yield* hasCheckoutIndex(fixture.root)); + assert.isTrue(NodeFS.existsSync(NodePath.join(fixture.root, "src/defs.ts"))); + }), + ), +); + +it.effect("clears an older version's folders from checkouts and leaves their indexes", () => + withFixture((fixture) => + Effect.gen(function* () { + const control = NodePath.join(fixture.root, ".codegraph-t3"); + const broken = NodePath.join(fixture.root, "broken"); + const { refresh, removeLegacyIndexes, hasCheckoutIndex } = yield* checkoutService( + fixture.root, + () => ["src/defs.ts"], + ); + assert.deepEqual(yield* refresh(), { _tag: "Published", generation: 1 }); + yield* Effect.promise(async () => { + await NodeFSP.mkdir(control); + await NodeFSP.writeFile(NodePath.join(control, "published"), "1"); + // Removing this one fails, which must not stop the sweep reaching the next checkout. + await NodeFSP.mkdir(NodePath.join(broken, ".codegraph-t3/codegraph.db"), { + recursive: true, + }); + }); + + yield* removeLegacyIndexes([broken, fixture.root]); + + assert.isFalse(NodeFS.existsSync(control)); + assert.isTrue(yield* hasCheckoutIndex(fixture.root)); + }), + ), +); + +it.effect("leaves a checkout's own .codegraph-t3 directory alone", () => + withFixture((fixture) => + Effect.gen(function* () { + const control = NodePath.join(fixture.root, ".codegraph-t3"); + yield* Effect.promise(async () => { + await NodeFSP.mkdir(control); + await NodeFSP.writeFile(NodePath.join(control, "notes.txt"), "mine"); + }); + const { refresh } = yield* checkoutService(fixture.root, () => ["src/defs.ts"]); + assert.deepEqual(yield* refresh(), { _tag: "Published", generation: 1 }); + assert.equal( + yield* Effect.promise(() => NodeFSP.readFile(NodePath.join(control, "notes.txt"), "utf8")), + "mine", + ); + }), + ), +); + +it.effect("leaves a linked .codegraph-t3 and its target alone", () => + withFixture((fixture) => + Effect.gen(function* () { + // Named like T3's own marker, so only the link check can protect it. + const victim = NodePath.join(fixture.worktree, "published"); + const control = NodePath.join(fixture.root, ".codegraph-t3"); + yield* Effect.promise(async () => { + await NodeFSP.writeFile(victim, "untouched"); + await NodeFSP.symlink(fixture.worktree, control); + }); + const { refresh } = yield* checkoutService(fixture.root, () => ["src/defs.ts"]); + assert.deepEqual(yield* refresh(), { _tag: "Published", generation: 1 }); + assert.isTrue(NodeFS.lstatSync(control).isSymbolicLink()); + assert.equal(yield* Effect.promise(() => NodeFSP.readFile(victim, "utf8")), "untouched"); + assert.isTrue(NodeFS.existsSync(NodePath.join(fixture.worktree, "src/work.ts"))); + }), + ), +); + +it.effect("leaves legacy folders for later while another server builds", () => + withFixture((fixture) => + Effect.gen(function* () { + const control = NodePath.join(fixture.root, ".codegraph-t3"); + yield* Effect.promise(async () => { + await NodeFSP.mkdir(control); + await NodeFSP.writeFile(NodePath.join(control, "codegraph.db"), "legacy index"); + }); + // An older server still building in the checkout holds the lock this removal would need. + const other = yield* otherServerProcess(fixture.root); + assert.equal(yield* other.send("lock"), "locked"); + const { refresh } = yield* checkoutService(fixture.root, () => ["src/defs.ts"]); + assert.deepEqual(yield* refresh(), { _tag: "Published", generation: 1 }); + assert.isTrue(NodeFS.existsSync(control)); + + // Once that server is gone, the next build removes them. + assert.equal(yield* other.send("release"), "released"); + yield* editDefinition(fixture.root, "second"); + assert.deepEqual(yield* refresh(), { _tag: "Published", generation: 2 }); + assert.isFalse(NodeFS.existsSync(control)); + }), + ), +); + +it.effect("removes an index without touching the checkout's sources", () => + withFixture((fixture) => + Effect.gen(function* () { + yield* Effect.promise(() => + NodeFSP.writeFile(NodePath.join(fixture.root, "top.ts"), "export const top = 1;\n"), + ); + const { refresh, removeCheckoutIndex, indexRoot } = yield* checkoutService( + fixture.root, + () => ["src/defs.ts", "src/use.ts", "top.ts"], + ); + const root = indexRoot(fixture.root); + assert.deepEqual(yield* refresh(), { _tag: "Published", generation: 1 }); + assert.isTrue(NodeFS.lstatSync(NodePath.join(root, "top.ts")).isSymbolicLink()); + + yield* removeCheckoutIndex(fixture.root); + assertClearedIndex(root); + // Every source is still there, byte for byte. + assert.equal( + yield* Effect.promise(() => + NodeFSP.readFile(NodePath.join(fixture.root, "src/defs.ts"), "utf8"), + ), + 'const prefix = "🧪é"; export function greet() { return prefix; }\n', + ); + assert.isTrue(NodeFS.existsSync(NodePath.join(fixture.root, "src/use.ts"))); + assert.isTrue(NodeFS.existsSync(NodePath.join(fixture.root, "src/main.go"))); + assert.equal( + yield* Effect.promise(() => + NodeFSP.readFile(NodePath.join(fixture.root, "top.ts"), "utf8"), + ), + "export const top = 1;\n", + ); + }), + ), +); + +it.effect("removes an orphaned index without touching a source checkout", () => + withFixture((fixture) => + Effect.gen(function* () { + const { refresh, removeOrphanedIndexes, indexRoot } = yield* checkoutService( + fixture.root, + () => ["src/defs.ts"], + ); + assert.deepEqual(yield* refresh(), { _tag: "Published", generation: 1 }); + yield* removeOrphanedIndexes(() => false); + assertClearedIndex(indexRoot(fixture.root)); + assert.equal( + yield* Effect.promise(() => + NodeFSP.readFile(NodePath.join(fixture.root, "src/defs.ts"), "utf8"), + ), + 'const prefix = "🧪é"; export function greet() { return prefix; }\n', + ); + }), + ), +); + +it.effect("sweeps indexes for checkouts that are gone or no longer known", () => + withFixture((fixture) => + Effect.gen(function* () { + const shared = yield* checkoutConfig(); + const known = yield* checkoutService(fixture.root, () => ["src/defs.ts"], { config: shared }); + const gone = yield* checkoutService(fixture.worktree, () => ["src/work.ts"], { + config: shared, + }); + assert.deepInclude(yield* known.refresh(), { _tag: "Published" }); + assert.deepInclude(yield* gone.refresh(), { _tag: "Published" }); + const knownRoot = known.indexRoot(fixture.root); + const goneRoot = gone.indexRoot(fixture.worktree); + assert.isTrue(NodeFS.existsSync(knownRoot)); + assert.isTrue(NodeFS.existsSync(goneRoot)); + + // A root with no owner record is a leftover, whatever the predicate says. + const orphanRoot = NodePath.join(known.codeIndexDir, "0".repeat(32)); + yield* Effect.promise(() => NodeFSP.mkdir(orphanRoot, { recursive: true })); + yield* Effect.promise(() => + NodeFSP.writeFile(NodePath.join(orphanRoot, "stray.txt"), "leftover"), + ); + + yield* known.removeOrphanedIndexes((checkoutPath) => checkoutPath === fixture.root); + // The known checkout is kept, and so are its sources. + assert.isTrue(NodeFS.existsSync(knownRoot)); + assert.isTrue(NodeFS.existsSync(NodePath.join(fixture.root, "src/defs.ts"))); + // A checkout `isKnown` rejects is removed, leaving its files intact. + assertClearedIndex(goneRoot); + assert.equal( + yield* Effect.promise(() => + NodeFSP.readFile(NodePath.join(fixture.worktree, "src/work.ts"), "utf8"), + ), + "export function workOnly() { return 2; }\n", + ); + assertClearedIndex(orphanRoot); + }), + ), +); + +it.effect("removes the index of a checkout that was deleted", () => + withFixture((fixture) => + Effect.gen(function* () { + const { refresh, removeOrphanedIndexes, indexRoot } = yield* checkoutService( + fixture.worktree, + () => ["src/work.ts"], + ); + assert.deepInclude(yield* refresh(), { _tag: "Published" }); + const root = indexRoot(fixture.worktree); + assert.isTrue(NodeFS.existsSync(root)); + yield* Effect.promise(() => NodeFSP.rm(fixture.worktree, { recursive: true, force: true })); + // The checkout is still known by path, but it no longer exists. + yield* removeOrphanedIndexes(() => true); + assertClearedIndex(root); + }), + ), +); + +it.effect("follows a new top-level entry on the next build and drops a removed one", () => + withFixture((fixture) => + Effect.gen(function* () { + const listed = ["src/defs.ts"]; + const { run, refresh, indexRoot } = yield* checkoutService(fixture.root, () => listed); + const root = indexRoot(fixture.root); + yield* refresh(); + assert.equal((yield* run({ operation: "symbols", name: "tool" })).nodes.length, 0); + + yield* Effect.promise(() => + NodeFSP.writeFile(NodePath.join(fixture.root, "tools.ts"), "export function tool() {}\n"), + ); + listed.push("tools.ts"); + yield* refresh(); + assert.ok((yield* run({ operation: "symbols", name: "tool" })).nodes.length > 0); + // The link the build added points at the checkout. + assert.isTrue(NodeFS.existsSync(NodePath.join(root, "tools.ts"))); + + // Removing the top-level entry leaves no dangling link in the root. + yield* Effect.promise(() => NodeFSP.rm(NodePath.join(fixture.root, "tools.ts"))); + listed.splice(listed.indexOf("tools.ts"), 1); + yield* refresh(); + assert.notInclude(yield* Effect.promise(() => NodeFSP.readdir(root)), "tools.ts"); + assert.equal((yield* run({ operation: "symbols", name: "tool" })).nodes.length, 0); + // The sources it shared the root with are still there. + assert.isTrue(NodeFS.existsSync(NodePath.join(root, "src/defs.ts"))); + assert.equal( + yield* Effect.promise(() => + NodeFSP.readFile(NodePath.join(fixture.root, "src/defs.ts"), "utf8"), + ), + 'const prefix = "🧪é"; export function greet() { return prefix; }\n', + ); }), ), ); +it.effect("keeps files the graph holds under a top-level entry a truncated listing dropped", () => + withFixture((fixture) => + Effect.gen(function* () { + let listed = ["src/defs.ts", "src/use.ts"]; + const { run, refresh, indexRoot } = yield* checkoutService(fixture.root, () => listed, { + truncated: true, + }); + const root = indexRoot(fixture.root); + yield* refresh(); + assert.ok((yield* run({ operation: "symbols", name: "greet" })).nodes.length > 0); + + // The cap drops `src` from the listing entirely; the files the graph already holds must + // still be reachable, so the build links that entry back from the graph's own paths. + yield* Effect.promise(() => + NodeFSP.writeFile(NodePath.join(fixture.root, "tools.ts"), "export function tool() {}\n"), + ); + listed = ["tools.ts"]; + assert.deepInclude(yield* refresh(), { _tag: "Published", generation: 2 }); + assert.isTrue(NodeFS.existsSync(NodePath.join(root, "src"))); + // The index still answers for the files it held under the dropped entry. + assert.equal(yield* indexedSymbolCount(root, "greet"), 1); + assert.equal(yield* indexedSymbolCount(root, "tool"), 1); + assert.equal((yield* run({ operation: "freshness" })).index?.changedFiles, 0); + }), + ), +); + +it.effect.each(["codegraph.json", ".gitignore"] as const)( + "honours changed exclude patterns in an unlisted %s", + (configName) => + withFixture((fixture) => + Effect.gen(function* () { + const configPath = NodePath.join(fixture.root, configName); + const excluded = + configName === "codegraph.json" + ? JSON.stringify({ exclude: ["src/use.ts"] }) + : "src/use.ts\n"; + yield* Effect.promise(() => NodeFSP.writeFile(configPath, excluded)); + // The listing never mentions the config file; only the link T3 plants reaches it. + const { run, refresh } = yield* checkoutService(fixture.root, () => [ + "src/defs.ts", + "src/use.ts", + ]); + yield* refresh(); + const status = yield* run({ operation: "status" }); + assert.equal(status.stats?.files, 1); + assert.equal((yield* run({ operation: "symbols", name: "caller" })).nodes.length, 0); + yield* refresh(); + + yield* Effect.promise(() => + NodeFSP.writeFile(configPath, configName === "codegraph.json" ? "{}" : ""), + ); + assert.equal((yield* run({ operation: "freshness" })).index?.state, "stale"); + assert.deepEqual(yield* refresh(), { _tag: "Published", generation: 2 }); + assert.equal((yield* run({ operation: "status" })).stats?.files, 2); + assert.equal((yield* run({ operation: "symbols", name: "caller" })).nodes.length, 1); + yield* refresh(); + + yield* Effect.promise(() => NodeFSP.writeFile(configPath, excluded)); + assert.deepEqual(yield* refresh(), { _tag: "Published", generation: 3 }); + assert.equal((yield* run({ operation: "status" })).stats?.files, 1); + assert.equal((yield* run({ operation: "symbols", name: "caller" })).nodes.length, 0); + yield* refresh(); + yield* Effect.promise(() => NodeFSP.unlink(configPath)); + assert.deepEqual(yield* refresh(), { _tag: "Published", generation: 4 }); + assert.equal((yield* run({ operation: "symbols", name: "caller" })).nodes.length, 1); + }), + ), +); + +it.effect.each(["codegraph.json", ".gitignore"] as const)( + "keeps indexing when %s cannot be read as a file", + (configName) => + withFixture((fixture) => + Effect.gen(function* () { + yield* Effect.promise(() => NodeFSP.mkdir(NodePath.join(fixture.root, configName))); + const { run, refresh } = yield* checkoutService(fixture.root, () => ["src/defs.ts"]); + assert.deepEqual(yield* refresh(), { _tag: "Published", generation: 1 }); + assert.equal((yield* run({ operation: "freshness" })).index?.state, "ready"); + }), + ), +); + it.effect("answers from the published generation while its successor builds", () => withFixture((fixture) => Effect.gen(function* () { @@ -1356,7 +2207,7 @@ it.effect("keeps the previous generation serving when a build fails, and says so yield* Effect.promise(() => rewriteGeneratedSources(fixture.root, files)); const held = yield* gate.holdNext; const building = yield* refresh().pipe(Effect.forkScoped); - process.kill(yield* Deferred.await(held.entered), "SIGABRT"); + yield* abortProcess(yield* Deferred.await(held.entered)); assert.deepEqual(yield* Fiber.join(building), { _tag: "Failed", reason: "worker-failed" }); const freshness = yield* run({ operation: "freshness" }); @@ -1433,7 +2284,7 @@ it.effect("coalesces refresh requests made while a build is in flight into one f // Every query and every build lists the checkout once. let listings = 0; const gate = buildGate(); - const { run, refresh } = yield* checkoutService( + const { run, refresh, indexRoot } = yield* checkoutService( fixture.root, () => { listings += 1; @@ -1457,9 +2308,10 @@ it.effect("coalesces refresh requests made while a build is in flight into one f assert.deepEqual(yield* Fiber.join(fiber), { _tag: "Unchanged" }); } assert.equal(listings, 3 + 2, "three queries and two builds listed the checkout"); - assert.deepEqual(yield* Effect.promise(() => generationDirectories(fixture.root)), [ - ".codegraph-t3-g1", - ]); + assert.deepEqual( + yield* Effect.promise(() => generationDirectories(indexRoot(fixture.root))), + [".codegraph-t3-g1"], + ); }), ), ); @@ -1487,12 +2339,13 @@ it.effect("deletes a retired generation only after its last reader finishes", () withFixture((fixture) => Effect.gen(function* () { const gate = listingGate(); - const { run, refresh } = yield* checkoutService( + const { run, refresh, indexRoot } = yield* checkoutService( fixture.root, () => ["src/defs.ts", "src/use.ts"], gate, ); - const generations = () => Effect.promise(() => generationDirectories(fixture.root)); + const generations = () => + Effect.promise(() => generationDirectories(indexRoot(fixture.root))); const edit = (body: string) => Effect.promise(() => NodeFSP.writeFile( @@ -1525,10 +2378,10 @@ it.effect("deletes a retired generation only after its last reader finishes", () const indexLocksModule = new URL("./codeIndexLocks.ts", import.meta.url).href; /** - * Another server process on the same checkout. It takes the index locks the way a server does, + * Another server process on the same index root. It takes the index locks the way a server does, * one stdin command at a time: `lock` the build, `pin` the published generation, or `release`. */ -function otherServerProcess(root: string) { +function otherServerProcess(indexRoot: string) { return Effect.acquireRelease( Effect.promise(async () => { const child = NodeChildProcess.spawn( @@ -1539,17 +2392,17 @@ function otherServerProcess(root: string) { "--input-type=module", "-e", `const locks = await import(process.argv[1]); - const checkout = process.argv[2]; + const indexRoot = process.argv[2]; const held = []; const commands = { lock: () => { - const release = locks.tryLockBuild(checkout); + const release = locks.tryLockBuild(indexRoot); if (release === null) return "busy"; held.push(release); return "locked"; }, pin: () => { - const pin = locks.pinPublishedGeneration(checkout); + const pin = locks.pinPublishedGeneration(indexRoot); held.push(pin.release); return "pinned " + pin.generation; }, @@ -1562,7 +2415,7 @@ function otherServerProcess(root: string) { lines.on("line", (command) => process.stdout.write(commands[command]() + "\\n")); process.stdout.write("ready\\n");`, indexLocksModule, - root, + indexRoot, ], { stdio: ["pipe", "pipe", "pipe"] }, ); @@ -1620,8 +2473,11 @@ function editDefinition(root: string, body: string) { it.effect("refuses to build while another server process builds the checkout", () => withFixture((fixture) => Effect.gen(function* () { - const other = yield* otherServerProcess(fixture.root); - const { run, refresh } = yield* checkoutService(fixture.root, () => ["src/defs.ts"]); + const { run, refresh, indexRoot } = yield* checkoutService(fixture.root, () => [ + "src/defs.ts", + ]); + const root = indexRoot(fixture.root); + const other = yield* otherServerProcess(root); assert.deepEqual(yield* refresh(), { _tag: "Published", generation: 1 }); yield* editDefinition(fixture.root, "second"); @@ -1633,7 +2489,7 @@ it.effect("refuses to build while another server process builds the checkout", ( generation: 1, refresh: "building", }); - assert.deepEqual(yield* Effect.promise(() => generationDirectories(fixture.root)), [ + assert.deepEqual(yield* Effect.promise(() => generationDirectories(root)), [ ".codegraph-t3-g1", ]); @@ -1647,9 +2503,10 @@ it.effect("refuses to build while another server process builds the checkout", ( it.effect("keeps a generation another server process reads until that reader releases it", () => withFixture((fixture) => Effect.gen(function* () { - const other = yield* otherServerProcess(fixture.root); - const { refresh } = yield* checkoutService(fixture.root, () => ["src/defs.ts"]); - const generations = () => Effect.promise(() => generationDirectories(fixture.root)); + const { refresh, indexRoot } = yield* checkoutService(fixture.root, () => ["src/defs.ts"]); + const root = indexRoot(fixture.root); + const other = yield* otherServerProcess(root); + const generations = () => Effect.promise(() => generationDirectories(root)); assert.deepEqual(yield* refresh(), { _tag: "Published", generation: 1 }); assert.equal(yield* other.send("pin"), "pinned 1"); @@ -1670,8 +2527,9 @@ it.effect("keeps a generation another server process reads until that reader rel it.effect("recovers the build lock and reader of a server process that crashed", () => withFixture((fixture) => Effect.gen(function* () { - const other = yield* otherServerProcess(fixture.root); - const { refresh } = yield* checkoutService(fixture.root, () => ["src/defs.ts"]); + const { refresh, indexRoot } = yield* checkoutService(fixture.root, () => ["src/defs.ts"]); + const root = indexRoot(fixture.root); + const other = yield* otherServerProcess(root); assert.deepEqual(yield* refresh(), { _tag: "Published", generation: 1 }); assert.equal(yield* other.send("pin"), "pinned 1"); assert.equal(yield* other.send("lock"), "locked"); @@ -1680,9 +2538,128 @@ it.effect("recovers the build lock and reader of a server process that crashed", yield* other.crash; assert.deepEqual(yield* refresh(), { _tag: "Published", generation: 2 }); - assert.deepEqual(yield* Effect.promise(() => generationDirectories(fixture.root)), [ + assert.deepEqual(yield* Effect.promise(() => generationDirectories(root)), [ ".codegraph-t3-g2", ]); }), ), ); + +it.effect("announces a checkout's first build once, however many builds follow it", () => + withFixture((fixture) => + Effect.gen(function* () { + const { run, refresh, subscribe, indexRoot } = yield* checkoutService(fixture.root, () => [ + "src/defs.ts", + ]); + const other = yield* otherServerProcess(indexRoot(fixture.root)); + const events = yield* subscribe(); + assert.equal(yield* other.send("lock"), "locked"); + // Each of these queries requests a build that ends at once, another server holding the + // lock. Signalling every time would restart it forever. + for (let attempt = 0; attempt < 3; attempt += 1) { + yield* run({ operation: "symbols", name: "greet" }); + } + assert.deepEqual(yield* refresh(), { _tag: "BuildingElsewhere" }); + + assert.equal(yield* other.send("release"), "released"); + assert.deepEqual(yield* refresh(), { _tag: "Published", generation: 1 }); + assert.deepEqual( + yield* events.pipe( + Stream.takeUntil((event) => event.outcome === "published" && event.generation === 1), + Stream.runCollect, + ), + [ + { outcome: "published", generation: null }, + { outcome: "published", generation: 1 }, + ], + ); + }), + ), +); + +it.effect("removes a checkout's index and asks a mounted reader to read again", () => + withFixture((fixture) => + Effect.gen(function* () { + const { run, refresh, subscribe, removeCheckoutIndex, indexRoot } = yield* checkoutService( + fixture.root, + () => ["src/defs.ts"], + ); + const root = indexRoot(fixture.root); + assert.deepEqual(yield* refresh(), { _tag: "Published", generation: 1 }); + const events = yield* subscribe(); + yield* removeCheckoutIndex(fixture.root); + assert.deepEqual(yield* events.pipe(Stream.take(1), Stream.runCollect), [ + { outcome: "published", generation: null }, + ]); + assertClearedIndex(root); + assert.deepEqual(yield* Effect.promise(() => generationDirectories(root)), []); + const freshness = (yield* run({ operation: "freshness" })).index; + assert.equal(freshness?.state, "unindexed"); + assert.notProperty(freshness, "refresh"); + + // The checkout starts over: its rebuild announces itself and restarts the numbering. + assert.deepEqual(yield* refresh(), { _tag: "Published", generation: 1 }); + assert.deepEqual(yield* events.pipe(Stream.take(2), Stream.runCollect), [ + { outcome: "published", generation: null }, + { outcome: "published", generation: 1 }, + ]); + }), + ), +); + +it.effect("stops a running build and asks a mounted reader to read again", () => + withFixture((fixture) => + Effect.gen(function* () { + const gate = buildGate(); + const { run, refresh, subscribe, stopBuilds } = yield* checkoutService( + fixture.root, + () => ["src/defs.ts"], + gate, + ); + const events = yield* subscribe(); + const held = yield* gate.holdNext; + const building = yield* refresh().pipe(Effect.forkScoped); + const pid = yield* Deferred.await(held.entered); + yield* stopBuilds; + assert.isFalse(isProcessAlive(pid), "the build process is gone once stopping returns"); + yield* Fiber.await(building); + // The first announces the build, the second its end. + assert.deepEqual(yield* events.pipe(Stream.take(2), Stream.runCollect), [ + { outcome: "published", generation: null }, + { outcome: "published", generation: null }, + ]); + const freshness = (yield* run({ operation: "freshness" })).index; + assert.equal(freshness?.state, "unindexed"); + assert.notProperty(freshness, "refresh"); + + // The next first build announces itself again. + assert.deepEqual(yield* refresh(), { _tag: "Published", generation: 1 }); + // A waiter resumes ahead of the build's bookkeeping; this answer proves that finished. + yield* run({ operation: "freshness" }); + // A stop with nothing running has nothing to say, so the next event is the publication. + yield* stopBuilds; + yield* editDefinition(fixture.root, "edited"); + assert.deepEqual(yield* refresh(), { _tag: "Published", generation: 2 }); + assert.deepEqual(yield* events.pipe(Stream.take(3), Stream.runCollect), [ + { outcome: "published", generation: null }, + { outcome: "published", generation: 1 }, + { outcome: "published", generation: 2 }, + ]); + }), + ), +); + +it.effect("can build again after a build is stopped before its first step", () => + withFixture((fixture) => + Effect.gen(function* () { + const { ensureProjectIndexed, refresh, stopBuilds } = yield* checkoutService( + fixture.root, + () => ["src/defs.ts"], + { repository: true }, + ); + yield* ensureProjectIndexed(); + yield* stopBuilds; + assert.deepEqual(yield* refresh(), { _tag: "Published", generation: 1 }); + }), + ), +); diff --git a/apps/server/src/project/knowledge/CodeIntelligence.ts b/apps/server/src/project/knowledge/CodeIntelligence.ts index 0809632427390..242d2d69d59ae 100644 --- a/apps/server/src/project/knowledge/CodeIntelligence.ts +++ b/apps/server/src/project/knowledge/CodeIntelligence.ts @@ -1,6 +1,8 @@ // @effect-diagnostics nodeBuiltinImport:off - this boundary owns the analysis process, its IPC, and the index generation directories. import * as NodeChildProcess from "node:child_process"; +import * as NodeFS from "node:fs"; import * as NodeFSP from "node:fs/promises"; +import * as NodeOS from "node:os"; import * as NodePath from "node:path"; import * as NodeURL from "node:url"; @@ -8,6 +10,7 @@ import { ProjectCodeQueryResult, type ProjectCodeIndexEvent, type ProjectCodeIndexTarget, + type ProjectId, type ProjectCodeQuery as ProjectCodeQueryType, type ProjectCodeQueryInput as ProjectCodeQueryInputType, type ProjectCodeQueryResult as ProjectCodeQueryResultType, @@ -16,38 +19,48 @@ import * as HostProcess from "@t3tools/shared/HostProcess"; import * as Context from "effect/Context"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; +import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as PubSub from "effect/PubSub"; import * as Schema from "effect/Schema"; import type * as Scope from "effect/Scope"; +import * as Semaphore from "effect/Semaphore"; import * as Stream from "effect/Stream"; import * as ProjectionStore from "../../orchestration-v2/ProjectionStore.ts"; import * as ProjectStore from "../../orchestration-v2/ProjectStore.ts"; +import * as ServerConfig from "../../config.ts"; +import * as ServerSettings from "../../serverSettings.ts"; +import * as VcsDriver from "../../vcs/VcsDriver.ts"; import * as WorkspaceEntries from "../../workspace/WorkspaceEntries.ts"; import * as WorkspacePaths from "../../workspace/WorkspacePaths.ts"; import { INDEX_DIRECTORY, - IndexOutsideCheckout, + checkoutPathKey, generationDirectory, + indexRootFor, isIndexPath, listGenerations, publishGeneration, + readIndexedCheckout, readPublishedGeneration, - removeLegacyIndexFiles, + writeIndexedCheckout, } from "./codeIndexLayout.ts"; import { + hasLegacyIndexFiles, isBuildRunning, pinPublishedGeneration, removeGenerationUnlessRead, + removeIndexRoot, + removeLegacyIndexFromCheckout, tryLockBuild, } from "./codeIndexLocks.ts"; import type { CodeIntelligenceCommand } from "./codeIntelligenceWorkerThread.ts"; -const MAX_LISTED_FILES = 10_000; - -const isIndexOutsideCheckout = Schema.is(IndexOutsideCheckout); +// Each build is a process parsing a whole checkout, and indexing starts without anyone asking. +const MAX_CONCURRENT_BUILDS = 2; export class CodeIntelligenceProjectNotFound extends Schema.TaggedError()( "CodeIntelligenceProjectNotFound", @@ -93,12 +106,7 @@ export class CodeIntelligenceInspectionFailed extends Schema.TaggedError()( "CodeIntelligenceWorkerError", { - reason: Schema.Literals([ - "worker-failed", - "invalid-result", - "path-outside-checkout", - "index-outside-checkout", - ]), + reason: Schema.Literals(["worker-failed", "invalid-result", "path-outside-checkout"]), }, ) {} @@ -137,10 +145,43 @@ export class CodeIntelligence extends Context.Service< ) => Effect.Effect; /** * Requests a coalesced rebuild of a checkout that already has a published index and returns - * at once. A checkout nobody has queried is left unindexed. + * at once. A checkout nobody has queried is left unindexed, and so is every checkout while + * automatic indexing is off. */ readonly refreshIfIndexed: (cwd: string) => Effect.Effect; - /** Publications and failed builds of the target's checkout from the moment this returns. */ + /** + * Starts the first index of a project's own checkout and returns at once. It does nothing once + * one exists, after a failed build, or outside a repository. Worktrees are covered by + * `refreshForActiveAgent` instead, so merely viewing a worktree thread never indexes it. + */ + readonly ensureProjectIndexed: (projectId: ProjectId) => Effect.Effect; + /** + * Keeps a checkout indexed while an agent works in it. Indexes a worktree that has none and + * refreshes one that does, without waiting for either. + */ + readonly refreshForActiveAgent: (target: ProjectCodeIndexTarget) => Effect.Effect; + /** + * Stops this process's builds of a checkout and removes its index, leaving any generation + * a reader still pins to the next sweep. + */ + readonly removeCheckoutIndex: (checkoutPath: string) => Effect.Effect; + /** Clears an older version's folders from each checkout, leaving its current index alone. */ + readonly removeLegacyIndexes: (checkoutPaths: ReadonlyArray) => Effect.Effect; + /** Whether the checkout has an index, current or left by an older version. */ + readonly hasCheckoutIndex: (checkoutPath: string) => Effect.Effect; + /** + * Removes indexes for checkouts that are gone or that `isKnown` rejects. A running build always + * holds its lock and has a record, so this only finds leftovers. + */ + readonly removeOrphanedIndexes: ( + isKnown: (checkoutPath: string) => boolean, + ) => Effect.Effect; + /** Interrupts every build this process is running, for when automatic indexing is turned off. */ + readonly stopBuilds: Effect.Effect; + /** + * Publications and failed builds of the target's checkout from the moment this returns. + * Watching a project also starts its first index. + */ readonly subscribeIndex: ( input: ProjectCodeIndexTarget, ) => Effect.Effect, CheckoutLookupError, Scope.Scope>; @@ -282,6 +323,15 @@ function runInAnalysisProcess( resume(Effect.fail(new CodeIntelligenceWorkerError({ reason: "worker-failed" }))); return; } + // A first build can hold a core for many minutes and nobody may be waiting on it, while a + // query has a caller. Set before the build's own threads exist so they inherit it. + if (command.type === "build" && child.pid !== undefined) { + try { + NodeOS.setPriority(child.pid, NodeOS.constants.priority.PRIORITY_BELOW_NORMAL); + } catch { + // Only ordering; a process that refuses it still builds correctly. + } + } let terminal: | { readonly _tag: "reply"; readonly value: WorkerTerminal } | { readonly _tag: "error"; readonly error: CodeIntelligenceWorkerError } @@ -308,11 +358,7 @@ function runInAnalysisProcess( if ("error" in decoded) { fail( new CodeIntelligenceWorkerError({ - reason: - decoded.error === "path-outside-checkout" || - decoded.error === "index-outside-checkout" - ? decoded.error - : "worker-failed", + reason: decoded.error === "path-outside-checkout" ? decoded.error : "worker-failed", }), ); } else terminal = { _tag: "reply", value: decoded }; @@ -372,6 +418,11 @@ interface CheckoutIndex { queued: Deferred.Deferred | null; // Whether the most recent build failed, leaving an older generation answering. lastFailed: boolean; + // The loop running those builds, interrupted to stop them when the checkout is removed. + fiber: Fiber.Fiber | null; + // Whether this checkout's first build already asked clients to read again. Once per checkout, + // or a build that ends at once loops through a query starting the next one. + firstBuildAnnounced: boolean; } const REFRESH_LIMITATIONS = { @@ -379,19 +430,18 @@ const REFRESH_LIMITATIONS = { failed: "The last index refresh failed; these results come from the generation before it.", } as const; -const removeGeneration = (checkoutPath: string, generation: number) => +const removeGeneration = (indexRoot: string, generation: number) => Effect.promise(() => - // `rm` unlinks a planted link instead of following it. - NodeFSP.rm(NodePath.join(checkoutPath, generationDirectory(generation)), { + NodeFSP.rm(NodePath.join(indexRoot, generationDirectory(generation)), { recursive: true, force: true, }).catch(() => undefined), ); // Best effort: a generation left behind is swept by the next build. -const removeUnlessRead = (checkoutPath: string, generation: number) => +const removeUnlessRead = (indexRoot: string, generation: number) => Effect.try({ - try: () => removeGenerationUnlessRead(checkoutPath, generation), + try: () => removeGenerationUnlessRead(indexRoot, generation), catch: () => undefined, }).pipe(Effect.ignore); @@ -400,11 +450,25 @@ export interface CodeIntelligenceOptions { readonly beforeBuildCommand?: (analysisPid: number) => Effect.Effect; } +const removeLegacyIndexFiles = (checkoutPath: string) => + Effect.sync(() => { + if (hasLegacyIndexFiles(checkoutPath)) removeLegacyIndexFromCheckout(checkoutPath); + }).pipe( + Effect.catchCause((cause) => + Effect.logWarning("failed to remove a checkout's old index files", { checkoutPath, cause }), + ), + ); + const make = Effect.fn("CodeIntelligence.make")(function* (options: CodeIntelligenceOptions) { const projects = yield* ProjectStore.ProjectStoreV2; const threads = yield* ProjectionStore.ProjectionStoreV2; const workspacePaths = yield* WorkspacePaths.WorkspacePaths; const workspaceEntries = yield* WorkspaceEntries.WorkspaceEntries; + const vcs = yield* VcsDriver.VcsDriver; + const settings = yield* ServerSettings.ServerSettingsService; + const config = yield* ServerConfig.ServerConfig; + const builds = yield* Semaphore.make(MAX_CONCURRENT_BUILDS); + const indexRootOf = (checkoutPath: string) => indexRootFor(config.codeIndexDir, checkoutPath); const launch: AnalysisLaunch = { executablePath: yield* HostProcess.ExecutablePath, isExecutable: yield* HostProcess.IsExecutable, @@ -412,16 +476,34 @@ const make = Effect.fn("CodeIntelligence.make")(function* (options: CodeIntellig const serviceScope = yield* Effect.scope; const checkouts = new Map(); const stateOf = (checkoutPath: string) => { - const existing = checkouts.get(checkoutPath); + const existing = checkouts.get(checkoutPathKey(checkoutPath)); if (existing !== undefined) return existing; - const created: CheckoutIndex = { running: null, queued: null, lastFailed: false }; - checkouts.set(checkoutPath, created); + const created: CheckoutIndex = { + running: null, + queued: null, + lastFailed: false, + fiber: null, + firstBuildAnnounced: false, + }; + checkouts.set(checkoutPathKey(checkoutPath), created); return created; }; + // Off, only a code query indexes a checkout; every proactive path reads this per call. + const autoIndexEnabled = settings.getSettings.pipe( + Effect.map((current) => current.autoCodeIndexEnabled), + Effect.orElseSucceed(() => false), + ); const indexChanges = yield* PubSub.unbounded<{ readonly checkoutPath: string; readonly event: ProjectCodeIndexEvent; }>(); + // Clients only treat events as a re-read signal, and older ones cannot decode a new outcome, so + // "read again" is a publication with no generation. + const publishReRead = (checkoutPath: string) => + PubSub.publish(indexChanges, { + checkoutPath, + event: { outcome: "published", generation: null }, + }); const resolveCheckout = Effect.fn("CodeIntelligence.resolveCheckout")(function* ( input: ProjectCodeIndexTarget, @@ -445,20 +527,36 @@ const make = Effect.fn("CodeIntelligence.make")(function* (options: CodeIntellig }); const listCheckout = Effect.fn("CodeIntelligence.listCheckout")(function* (checkoutPath: string) { + const indexRoot = indexRootOf(checkoutPath); + // Git lists a repository in full. The finder caps its listing, and holds only part of a + // checkout until its first scan ends, which is exactly when a new project is indexed. + const repository = yield* vcs.listWorkspaceFiles(checkoutPath).pipe(Effect.option); + if (Option.isSome(repository)) { + // An unmerged path is listed once per stage. + const files = [...new Set(repository.value.paths)].filter((path) => !isIndexPath(path)); + return { + checkoutPath, + indexRoot, + files, + filesListed: files.length, + listingTruncated: repository.value.truncated, + }; + } yield* workspaceEntries.refresh(checkoutPath); const listing = yield* workspaceEntries.list({ cwd: checkoutPath }); const files = listing.entries - // The index lives in the checkout; counting it would report its files as unsupported - // source. + // A checkout that still holds an older version's index would otherwise report its files + // as unsupported source. .filter( (entry) => entry.kind === "file" && entry.ignored !== true && !isIndexPath(entry.path), ) .map((entry) => entry.path); return { checkoutPath, - files: files.slice(0, MAX_LISTED_FILES), + indexRoot, + files, filesListed: files.length, - listingTruncated: listing.truncated || files.length > MAX_LISTED_FILES, + listingTruncated: listing.truncated, }; }); @@ -468,21 +566,18 @@ const make = Effect.fn("CodeIntelligence.make")(function* (options: CodeIntellig CodeIndexRefreshOutcome, CodeIntelligenceWorkerError | WorkspaceEntries.WorkspaceEntriesError > { + const indexRoot = indexRootOf(checkoutPath); const plan = yield* Effect.try({ - try: () => { - const published = readPublishedGeneration(checkoutPath); - removeLegacyIndexFiles(checkoutPath); - return { published, existing: listGenerations(checkoutPath) }; - }, - catch: (cause) => - new CodeIntelligenceWorkerError({ - reason: isIndexOutsideCheckout(cause) ? "index-outside-checkout" : "worker-failed", - }), + try: () => ({ + published: readPublishedGeneration(indexRoot), + existing: listGenerations(indexRoot), + }), + catch: () => new CodeIntelligenceWorkerError({ reason: "worker-failed" }), }); // Interrupted builds, and superseded generations whose last reader is gone. The build lock // keeps every other builder out, so none of these is being written. for (const existing of plan.existing) { - if (existing !== plan.published) yield* removeUnlessRead(checkoutPath, existing); + if (existing !== plan.published) yield* removeUnlessRead(indexRoot, existing); } const generation = Math.max(plan.published ?? 0, ...plan.existing) + 1; const listing = yield* listCheckout(checkoutPath); @@ -498,41 +593,47 @@ const make = Effect.fn("CodeIntelligence.make")(function* (options: CodeIntellig }, }, options.beforeBuildCommand, - ).pipe(Effect.onError(() => removeGeneration(checkoutPath, generation))); + ).pipe(Effect.onError(() => removeGeneration(indexRoot, generation))); if (!("built" in reply)) { - yield* removeGeneration(checkoutPath, generation); + yield* removeGeneration(indexRoot, generation); return yield* new CodeIntelligenceWorkerError({ reason: "invalid-result" }); } if (!reply.built.changed) return { _tag: "Unchanged" } as const; yield* Effect.try({ - try: () => publishGeneration(checkoutPath, generation), - catch: (cause) => - new CodeIntelligenceWorkerError({ - reason: isIndexOutsideCheckout(cause) ? "index-outside-checkout" : "worker-failed", - }), - }).pipe(Effect.onError(() => removeGeneration(checkoutPath, generation))); + try: () => publishGeneration(indexRoot, generation), + catch: () => new CodeIntelligenceWorkerError({ reason: "worker-failed" }), + }).pipe(Effect.onError(() => removeGeneration(indexRoot, generation))); const previous = plan.published; // New readers pin the new generation from here on; a reader of the old one removes it. - if (previous !== null) yield* removeUnlessRead(checkoutPath, previous); + if (previous !== null) yield* removeUnlessRead(indexRoot, previous); return { _tag: "Published", generation } as const; }); const buildOnce = (checkoutPath: string): Effect.Effect => Effect.acquireUseRelease( Effect.try({ - try: () => tryLockBuild(checkoutPath), - catch: (cause) => - new CodeIntelligenceWorkerError({ - reason: isIndexOutsideCheckout(cause) ? "index-outside-checkout" : "worker-failed", - }), + try: () => { + const indexRoot = indexRootOf(checkoutPath); + const release = tryLockBuild(indexRoot); + // One step: a concurrent removal must never see a locked root without a record. + if (release !== null) writeIndexedCheckout(indexRoot, checkoutPath); + return release; + }, + catch: () => new CodeIntelligenceWorkerError({ reason: "worker-failed" }), }), // Waiting would tie this server to another's build; that build covers the checkout. (release) => release === null ? Effect.succeed({ _tag: "BuildingElsewhere" } as const) - : buildGeneration(checkoutPath), + : Effect.gen(function* () { + // Folders an older version left in the checkout go on the first build that sees them. + yield* removeLegacyIndexFiles(checkoutPath); + return yield* buildGeneration(checkoutPath); + }), (release) => Effect.sync(() => release?.()), ).pipe( + // Outside the lock: a checkout waiting its turn must not read as building to other servers. + builds.withPermits(1), Effect.catch((error) => Effect.succeed({ _tag: "Failed", @@ -546,59 +647,48 @@ const make = Effect.fn("CodeIntelligence.make")(function* (options: CodeIntellig ), ); - const runBuilds = Effect.fn("CodeIntelligence.runBuilds")( - function* (checkoutPath: string, state: CheckoutIndex) { - let current = state.running; - while (current !== null) { - const outcome = yield* buildOnce(checkoutPath); - // Subscribers query again on failure; repeating that event would keep starting builds - // for a persistent failure even when nobody explicitly asks to retry. - if ( - outcome._tag === "Published" || - (outcome._tag === "Failed" && !state.lastFailed) || - (outcome._tag === "Unchanged" && state.lastFailed) - ) { - state.lastFailed = outcome._tag === "Failed"; - const generation = - outcome._tag === "Published" - ? outcome.generation - : yield* Effect.sync(() => { - try { - return readPublishedGeneration(checkoutPath); - } catch { - return null; - } - }); - // Existing clients understand a published generation becoming available again. - const event: ProjectCodeIndexEvent = { - outcome: outcome._tag === "Failed" ? "failed" : "published", - generation, - }; - yield* PubSub.publish(indexChanges, { checkoutPath, event }); - } - yield* Deferred.succeed(current, outcome); - // Promoting and reading the next build in one step keeps a concurrent request from - // starting a second loop for the same checkout. - current = yield* Effect.sync(() => { - state.running = state.queued; - state.queued = null; - return state.running; - }); + const runBuilds = Effect.fn("CodeIntelligence.runBuilds")(function* ( + checkoutPath: string, + state: CheckoutIndex, + ) { + let current = state.running; + while (current !== null) { + const outcome = yield* buildOnce(checkoutPath); + // Subscribers query again on failure; repeating that event would keep starting builds + // for a persistent failure even when nobody explicitly asks to retry. + if ( + outcome._tag === "Published" || + (outcome._tag === "Failed" && !state.lastFailed) || + (outcome._tag === "Unchanged" && state.lastFailed) + ) { + state.lastFailed = outcome._tag === "Failed"; + const generation = + outcome._tag === "Published" + ? outcome.generation + : yield* Effect.sync(() => { + try { + return readPublishedGeneration(indexRootOf(checkoutPath)); + } catch { + return null; + } + }); + // Existing clients understand a published generation becoming available again. + const event: ProjectCodeIndexEvent = { + outcome: outcome._tag === "Failed" ? "failed" : "published", + generation, + }; + yield* PubSub.publish(indexChanges, { checkoutPath, event }); } - }, - (effect, _checkoutPath, state) => - effect.pipe( - Effect.onInterrupt(() => - Effect.gen(function* () { - const pending = [state.running, state.queued]; - state.running = null; - state.queued = null; - for (const deferred of pending) - if (deferred !== null) yield* Deferred.interrupt(deferred); - }), - ), - ), - ); + yield* Deferred.succeed(current, outcome); + // Promoting and reading the next build in one step keeps a concurrent request from + // starting a second loop for the same checkout. + current = yield* Effect.sync(() => { + state.running = state.queued; + state.queued = null; + return state.running; + }); + } + }); /** The build that will cover a request made now; starts one when the checkout is idle. */ const requestBuild = (checkoutPath: string) => @@ -611,8 +701,39 @@ const make = Effect.fn("CodeIntelligence.make")(function* (options: CodeIntellig return Effect.succeed(next); } state.running = next; - return runBuilds(checkoutPath, state).pipe(Effect.forkIn(serviceScope), Effect.as(next)); - }); + return Effect.gen(function* () { + // Once per checkout: a build that ends at once (another server holds the lock, or the + // index path is refused) would otherwise be restarted by the query its own signal runs. + if (!state.firstBuildAnnounced) { + const unindexed = yield* Effect.sync(() => { + try { + return readPublishedGeneration(indexRootOf(checkoutPath)) === null; + } catch { + return false; + } + }); + if (unindexed) { + state.firstBuildAnnounced = true; + yield* publishReRead(checkoutPath); + } + } + const fiber = yield* runBuilds(checkoutPath, state).pipe(Effect.forkIn(serviceScope)); + // Outside the loop: a fiber interrupted before its first step runs none of its own code. + fiber.addObserver((exit) => { + if (Exit.isSuccess(exit)) return; + const pending = [state.running, state.queued]; + state.running = null; + state.queued = null; + for (const deferred of pending) { + if (deferred !== null) Deferred.doneUnsafe(deferred, Exit.failCause(exit.cause)); + } + }); + state.fiber = fiber; + return next; + }); + // Interruption here would leave `running` set with no loop behind it, wedging the checkout + // until restart. The forked loop itself stays interruptible. + }).pipe(Effect.uninterruptible); const refresh: CodeIntelligence["Service"]["refresh"] = Effect.fn("CodeIntelligence.refresh")( function* (input) { @@ -625,18 +746,17 @@ const make = Effect.fn("CodeIntelligence.make")(function* (options: CodeIntellig function* (input) { const checkoutPath = yield* resolveCheckout(input); const normalizedQuery = yield* normalizeQueryPaths(input.query, checkoutPath, workspacePaths); + const indexRoot = indexRootOf(checkoutPath); const { generation: published } = yield* Effect.acquireRelease( Effect.try({ - try: () => pinPublishedGeneration(checkoutPath), - catch: (cause) => - isIndexOutsideCheckout(cause) - ? new CodeIntelligencePathOutsideCheckout({ path: INDEX_DIRECTORY }) - : new CodeIntelligenceInspectionFailed({ - projectId: input.projectId, - threadId: input.threadId ?? null, - path: INDEX_DIRECTORY, - reason: "worker-failed", - }), + try: () => pinPublishedGeneration(indexRoot), + catch: () => + new CodeIntelligenceInspectionFailed({ + projectId: input.projectId, + threadId: input.threadId ?? null, + path: checkoutPath, + reason: "worker-failed", + }), }), (pin) => Effect.sync(pin.release), ); @@ -662,9 +782,6 @@ const make = Effect.fn("CodeIntelligence.make")(function* (options: CodeIntellig if (reason === "path-outside-checkout" && queryPath !== undefined) { return new CodeIntelligencePathOutsideCheckout({ path: queryPath }); } - if (reason === "index-outside-checkout") { - return new CodeIntelligencePathOutsideCheckout({ path: INDEX_DIRECTORY }); - } return new CodeIntelligenceInspectionFailed({ projectId: input.projectId, threadId: input.threadId ?? null, @@ -687,13 +804,13 @@ const make = Effect.fn("CodeIntelligence.make")(function* (options: CodeIntellig }, }).pipe(Effect.mapError((cause) => toQueryError(cause.reason))); if (!("result" in reply)) return yield* toQueryError("invalid-result"); - const state = checkouts.get(checkoutPath); + const state = checkouts.get(checkoutPathKey(checkoutPath)); const needsRefresh = reply.result.index?.state === "unindexed" || reply.result.index?.state === "indexing" || reply.result.index?.state === "stale"; const refreshing: keyof typeof REFRESH_LIMITATIONS | undefined = - needsRefresh && (state?.running != null || isBuildRunning(checkoutPath)) + needsRefresh && (state?.running != null || isBuildRunning(indexRoot)) ? "building" : state?.lastFailed ? "failed" @@ -717,11 +834,12 @@ const make = Effect.fn("CodeIntelligence.make")(function* (options: CodeIntellig const refreshIfIndexed: CodeIntelligence["Service"]["refreshIfIndexed"] = Effect.fn( "CodeIntelligence.refreshIfIndexed", )(function* (cwd) { + if (!(yield* autoIndexEnabled)) return; const checkoutPath = yield* workspacePaths.normalizeWorkspaceRoot(cwd).pipe(Effect.option); if (Option.isNone(checkoutPath)) return; const published = yield* Effect.sync(() => { try { - return readPublishedGeneration(checkoutPath.value); + return readPublishedGeneration(indexRootOf(checkoutPath.value)); } catch { return null; } @@ -729,18 +847,176 @@ const make = Effect.fn("CodeIntelligence.make")(function* (options: CodeIntellig if (published !== null) yield* requestBuild(checkoutPath.value); }); + const ensureProjectIndexed: CodeIntelligence["Service"]["ensureProjectIndexed"] = Effect.fn( + "CodeIntelligence.ensureProjectIndexed", + )(function* (projectId) { + if (!(yield* autoIndexEnabled)) return; + const checkoutPath = yield* resolveCheckout({ projectId }).pipe(Effect.option); + if (Option.isNone(checkoutPath)) return; + const state = checkouts.get(checkoutPathKey(checkoutPath.value)); + // A failed build waits for a code query; retrying here would repeat a persistent failure each + // time a client looked at the project. + if (state?.running != null || state?.lastFailed === true) return; + const unindexed = yield* Effect.sync(() => { + try { + return readPublishedGeneration(indexRootOf(checkoutPath.value)) === null; + } catch { + return false; + } + }); + if (!unindexed) return; + // A folder outside any repository may be anything, including something huge. + const isRepository = yield* vcs + .isInsideWorkTree(checkoutPath.value) + .pipe(Effect.orElseSucceed(() => false)); + if (isRepository) yield* requestBuild(checkoutPath.value); + }); + + const refreshForActiveAgent: CodeIntelligence["Service"]["refreshForActiveAgent"] = Effect.fn( + "CodeIntelligence.refreshForActiveAgent", + )(function* (target) { + if (!(yield* autoIndexEnabled)) return; + const checkoutPath = yield* resolveCheckout(target).pipe(Effect.option); + if (Option.isNone(checkoutPath)) return; + const published = yield* Effect.sync(() => { + try { + return readPublishedGeneration(indexRootOf(checkoutPath.value)); + } catch { + return null; + } + }); + if (published !== null) { + yield* requestBuild(checkoutPath.value); + return; + } + // A worktree always indexes for its agent; a folder outside any repository does not. + const isRepository = yield* vcs + .isInsideWorkTree(checkoutPath.value) + .pipe(Effect.orElseSucceed(() => false)); + if (isRepository) yield* requestBuild(checkoutPath.value); + }); + + const removeCheckoutIndex: CodeIntelligence["Service"]["removeCheckoutIndex"] = Effect.fn( + "CodeIntelligence.removeCheckoutIndex", + )(function* (checkoutPath) { + // A checkout that is already gone no longer normalizes; its index still needs removing. + const path = yield* workspacePaths + .normalizeWorkspaceRoot(checkoutPath) + .pipe(Effect.orElseSucceed(() => NodePath.resolve(checkoutPath))); + const state = checkouts.get(checkoutPathKey(path)); + const hadBuild = state?.fiber != null && state.running !== null; + // The interrupt returns only after the build process is killed and its lock released. + if (state?.fiber != null) yield* Fiber.interrupt(state.fiber); + checkouts.delete(checkoutPathKey(path)); + const hadFiles = yield* Effect.sync( + () => listGenerations(indexRootOf(path)).length > 0 || hasLegacyIndexFiles(path), + ); + const removal = yield* Effect.exit( + Effect.sync(() => { + removeIndexRoot(indexRootOf(path)); + if (hasLegacyIndexFiles(path)) removeLegacyIndexFromCheckout(path); + }), + ); + if (Exit.isFailure(removal)) { + yield* Effect.logWarning("failed to remove a checkout's code index", { + checkoutPath: path, + cause: removal.cause, + }); + } + // A mounted badge reads again on any event; without one it would keep showing Ready. + if (hadFiles || hadBuild) yield* publishReRead(path); + }); + + const removeLegacyIndexes: CodeIntelligence["Service"]["removeLegacyIndexes"] = Effect.fn( + "CodeIntelligence.removeLegacyIndexes", + )(function* (checkoutPaths) { + yield* Effect.forEach(checkoutPaths, removeLegacyIndexFiles, { discard: true }); + }); + + const hasCheckoutIndex: CodeIntelligence["Service"]["hasCheckoutIndex"] = Effect.fn( + "CodeIntelligence.hasCheckoutIndex", + )(function* (checkoutPath) { + const path = yield* workspacePaths + .normalizeWorkspaceRoot(checkoutPath) + .pipe(Effect.orElseSucceed(() => NodePath.resolve(checkoutPath))); + return yield* Effect.sync( + () => + listGenerations(indexRootOf(path)).length > 0 || + isBuildRunning(indexRootOf(path)) || + hasLegacyIndexFiles(path), + ); + }); + + const removeOrphanedIndexes: CodeIntelligence["Service"]["removeOrphanedIndexes"] = Effect.fn( + "CodeIntelligence.removeOrphanedIndexes", + )(function* (isKnown) { + const roots = yield* Effect.sync(() => { + try { + return NodeFS.readdirSync(config.codeIndexDir, { withFileTypes: true }).flatMap((entry) => + entry.isDirectory() ? [entry.name] : [], + ); + } catch { + return []; + } + }); + for (const name of roots) { + yield* Effect.gen(function* () { + const indexRoot = NodePath.join(config.codeIndexDir, name); + const owner = readIndexedCheckout(indexRoot); + // A build always holds the lock and has a record, so a root without one is a leftover. + if (owner === null) { + yield* Effect.sync(() => removeIndexRoot(indexRoot)); + return; + } + if (isKnown(owner) && NodeFS.existsSync(owner)) return; + yield* removeCheckoutIndex(owner); + }).pipe( + Effect.catchCause((cause) => + Effect.logWarning("failed to remove an orphaned code index", { indexRoot: name, cause }), + ), + ); + } + }); + + const stopBuilds: CodeIntelligence["Service"]["stopBuilds"] = Effect.gen(function* () { + for (const [path, state] of checkouts) { + // `running` is cleared when a loop ends, so a finished build is not stopped or announced. + if (state.fiber === null || state.running === null) continue; + yield* Fiber.interrupt(state.fiber); + // The checkout can build again later, so it announces its next first build. + state.firstBuildAnnounced = false; + // A badge stuck on Indexing has nothing to read again unless it hears about this. + yield* publishReRead(path); + } + }); + const subscribeIndex: CodeIntelligence["Service"]["subscribeIndex"] = Effect.fn( "CodeIntelligence.subscribeIndex", )(function* (input) { const checkoutPath = yield* resolveCheckout(input); const subscription = yield* PubSub.subscribe(indexChanges); + yield* ensureProjectIndexed(input.projectId); return Stream.fromSubscription(subscription).pipe( - Stream.filter((change) => change.checkoutPath === checkoutPath), + Stream.filter( + (change) => checkoutPathKey(change.checkoutPath) === checkoutPathKey(checkoutPath), + ), Stream.map((change) => change.event), ); }); - return CodeIntelligence.of({ query, refresh, refreshIfIndexed, subscribeIndex }); + return CodeIntelligence.of({ + query, + refresh, + refreshIfIndexed, + ensureProjectIndexed, + refreshForActiveAgent, + removeCheckoutIndex, + removeLegacyIndexes, + hasCheckoutIndex, + removeOrphanedIndexes, + stopBuilds, + subscribeIndex, + }); }); export const layer = Layer.effect(CodeIntelligence, make({})); diff --git a/apps/server/src/project/knowledge/codeIndexLayout.ts b/apps/server/src/project/knowledge/codeIndexLayout.ts index bb58ab119bab5..1eeb2b5eae222 100644 --- a/apps/server/src/project/knowledge/codeIndexLayout.ts +++ b/apps/server/src/project/knowledge/codeIndexLayout.ts @@ -1,27 +1,20 @@ // @effect-diagnostics nodeBuiltinImport:off - shared by the server and the analysis worker, which runs outside the Effect runtime; every call is a bounded synchronous filesystem step. +import * as NodeCrypto from "node:crypto"; import * as NodeFS from "node:fs"; import * as NodePath from "node:path"; -import * as Schema from "effect/Schema"; /** - * The control directory holds only the `published` pointer. Each index generation is a sibling - * `.codegraph-t3-g` because the SDK takes its data directory as one path segment in the - * checkout root, and its indexer skips every `.codegraph-*` sibling. + * The index root is `/` in T3's data directory, never the checkout: the SDK + * takes its data directory as one path segment inside its project root and ignores absolute + * paths, so T3 gives it a private root that links back to the checkout's top-level entries. */ export const INDEX_DIRECTORY = ".codegraph-t3"; const GENERATION_PREFIX = `${INDEX_DIRECTORY}-g`; const GENERATION_NAME = /^\.codegraph-t3-g([1-9][0-9]{0,8})$/; -const PUBLISHED_FILE = "published"; -const INDEX_GITIGNORE = "# T3 code index, local to this machine.\n*\n"; - -export class IndexOutsideCheckout extends Schema.TaggedError()( - "IndexOutsideCheckout", - {}, -) { - override get message(): string { - return "index-outside-checkout"; - } -} +export const PUBLISHED_FILE = "published"; +const INDEXED_CHECKOUT_FILE = "checkout"; +// Changes what the SDK indexes, and a truncated listing can drop them. +export const CHECKOUT_CONFIG_ENTRIES = ["codegraph.json", ".gitignore"]; export const generationDirectory = (generation: number) => `${GENERATION_PREFIX}${generation}`; @@ -30,62 +23,57 @@ function generationOf(name: string): number | null { return match === null ? null : Number(match[1]); } -const isIndexOutsideCheckout = Schema.is(IndexOutsideCheckout); - -/** Whether a checkout-relative path lies in the control directory or a generation. */ +/** Whether a root-relative path lies in the control directory or a generation. */ export function isIndexPath(relativePath: string): boolean { const top = relativePath.split("/", 1)[0]!; return top === INDEX_DIRECTORY || generationOf(top) !== null; } -/** Generation directories present in the checkout root, whatever their state. */ -export function listGenerations(checkoutPath: string): number[] { - return NodeFS.readdirSync(checkoutPath).flatMap((name) => { +/** The index root one checkout's index lives under. */ +export function indexRootFor(codeIndexDir: string, checkoutPath: string): string { + const digest = NodeCrypto.createHash("sha256") + .update(checkoutPathKey(checkoutPath)) + .digest("hex"); + return NodePath.join(codeIndexDir, digest.slice(0, 32)); +} + +/** Identity only: keep the original path when opening files or recording the checkout owner. */ +export function checkoutPathKey(checkoutPath: string): string { + const resolved = NodePath.resolve(checkoutPath); + return NodePath.sep === "\\" ? resolved.toLowerCase() : resolved; +} + +/** Generation directories present in the index root, whatever their state. */ +export function listGenerations(indexRoot: string): number[] { + let names: string[]; + try { + names = NodeFS.readdirSync(indexRoot); + } catch { + return []; + } + return names.flatMap((name) => { const generation = generationOf(name); return generation === null ? [] : [generation]; }); } -/** - * Refuses an index directory that is a link, resolves outside the checkout, or holds anything but - * plain entries. The SDK opens the database, its -wal/-shm sidecars, its lock and .gitignore by - * path and follows a link or shared inode out of the checkout, and T3 writes the pointer and - * metadata the same way. A missing directory passes. - */ -export function validateIndexDirectory(checkoutPath: string, name: string): void { - const indexPath = NodePath.join(checkoutPath, name); +/** The checkout this index root belongs to, or null when it holds no record. */ +export function readIndexedCheckout(indexRoot: string): string | null { try { - const stat = NodeFS.lstatSync(indexPath); - if (stat.isSymbolicLink() || !stat.isDirectory()) throw new IndexOutsideCheckout(); - const root = NodeFS.realpathSync(checkoutPath); - const relative = NodePath.relative(root, NodeFS.realpathSync(indexPath)); - if ( - relative === "" || - relative === ".." || - relative.startsWith(`..${NodePath.sep}`) || - NodePath.isAbsolute(relative) - ) { - throw new IndexOutsideCheckout(); - } - for (const child of NodeFS.readdirSync(indexPath)) { - const entry = NodeFS.lstatSync(NodePath.join(indexPath, child)); - if (!(entry.isDirectory() || (entry.isFile() && entry.nlink === 1))) { - throw new IndexOutsideCheckout(); - } - } - } catch (cause) { - if (isIndexOutsideCheckout(cause)) throw cause; - if ((cause as NodeJS.ErrnoException).code !== "ENOENT") throw cause; + return NodeFS.readFileSync( + NodePath.join(indexRoot, INDEX_DIRECTORY, INDEXED_CHECKOUT_FILE), + "utf8", + ); + } catch { + return null; } } -/** Replaces `name` in the directory without writing through whatever already sits there. */ +/** Replaces `name` in the directory in one step, so a reader sees the old contents or the new. */ function replaceFile(directory: string, name: string, contents: string): void { const target = NodePath.join(directory, name); const temporary = `${target}.${process.pid}.tmp`; try { - // An exclusive create never follows a planted link, and the rename replaces the entry - // rather than writing through it. NodeFS.writeFileSync(temporary, contents, { flag: "wx" }); NodeFS.renameSync(temporary, target); } catch (cause) { @@ -94,39 +82,82 @@ function replaceFile(directory: string, name: string, contents: string): void { } } +/** Records which checkout an index root belongs to, for sweeps that outlive the project. */ +export function writeIndexedCheckout(indexRoot: string, checkoutPath: string): void { + const control = NodePath.join(indexRoot, INDEX_DIRECTORY); + NodeFS.mkdirSync(control, { recursive: true }); + replaceFile(control, INDEXED_CHECKOUT_FILE, checkoutPath); +} + /** - * The SDK's .gitignore keeps itself tracked, which leaves the index directory in the user's - * `git status`; T3's index is private to the machine, so the file ignores itself too. + * Points the index root at each top-level entry of the checkout the build may read, so the SDK + * finds sources at the paths the index stores. A missing target is skipped: it has nothing to + * index. */ -export function ignoreIndexDirectory(checkoutPath: string, name: string): void { - const directory = NodePath.join(checkoutPath, name); - try { - if (NodeFS.readFileSync(NodePath.join(directory, ".gitignore"), "utf8") === INDEX_GITIGNORE) - return; - } catch {} - try { - replaceFile(directory, ".gitignore", INDEX_GITIGNORE); - } catch {} +export function linkCheckoutSources( + indexRoot: string, + checkoutPath: string, + paths: ReadonlyArray, +): void { + if ( + NodePath.sep === "\\" && + NodePath.toNamespacedPath(checkoutPath).toUpperCase().startsWith("\\\\?\\UNC\\") + ) + throw new Error("Windows junctions cannot target a network checkout."); + const names = new Set(CHECKOUT_CONFIG_ENTRIES); + for (const path of paths) names.add(path.split("/", 1)[0]!); + for (const name of names) { + if (isIndexPath(name)) continue; + const link = NodePath.join(indexRoot, name); + const target = NodePath.join(checkoutPath, name); + let isDirectory: boolean; + try { + // Followed: Windows needs the link's type to match what a checkout link points at. + isDirectory = NodeFS.statSync(target).isDirectory(); + } catch { + continue; + } + if (NodeFS.lstatSync(link, { throwIfNoEntry: false }) !== undefined) continue; + try { + // `junction` needs no privilege on Windows, and the type argument is ignored on POSIX. + NodeFS.symlinkSync(target, link, isDirectory ? "junction" : "file"); + } catch (cause) { + // Windows file links need a privilege; a copy is rebuilt on every build. + if (!isDirectory && (cause as NodeJS.ErrnoException).code === "EPERM") { + NodeFS.copyFileSync(target, link); + continue; + } + throw cause; + } + } } -export function readFileNoFollow(path: string): string { - const descriptor = NodeFS.openSync( - path, - NodeFS.constants.O_RDONLY | (NodeFS.constants.O_NOFOLLOW ?? 0), - ); +/** Removes the links and copies a build left in the index root, never their targets. */ +export function clearCheckoutSources(indexRoot: string): void { + let names: string[]; try { - return NodeFS.readFileSync(descriptor, "utf8"); - } finally { - NodeFS.closeSync(descriptor); + names = NodeFS.readdirSync(indexRoot); + } catch { + return; + } + for (const name of names) { + if (isIndexPath(name)) continue; + const entry = NodePath.join(indexRoot, name); + let stats: NodeFS.Stats; + try { + stats = NodeFS.lstatSync(entry); + } catch { + continue; + } + if (stats.isSymbolicLink() || stats.isFile()) NodeFS.unlinkSync(entry); } } -/** The published generation, or null for a checkout with none. Validates the control directory. */ -export function readPublishedGeneration(checkoutPath: string): number | null { - validateIndexDirectory(checkoutPath, INDEX_DIRECTORY); +/** The published generation, or null for an index root with none. */ +export function readPublishedGeneration(indexRoot: string): number | null { let text: string; try { - text = readFileNoFollow(NodePath.join(checkoutPath, INDEX_DIRECTORY, PUBLISHED_FILE)); + text = NodeFS.readFileSync(NodePath.join(indexRoot, INDEX_DIRECTORY, PUBLISHED_FILE), "utf8"); } catch (cause) { if ((cause as NodeJS.ErrnoException).code === "ENOENT") return null; throw cause; @@ -135,27 +166,8 @@ export function readPublishedGeneration(checkoutPath: string): number | null { } /** Atomically points readers at `generation`. */ -export function publishGeneration(checkoutPath: string, generation: number): void { - const directory = NodePath.join(checkoutPath, INDEX_DIRECTORY); +export function publishGeneration(indexRoot: string, generation: number): void { + const directory = NodePath.join(indexRoot, INDEX_DIRECTORY); NodeFS.mkdirSync(directory, { recursive: true }); - validateIndexDirectory(checkoutPath, INDEX_DIRECTORY); - ignoreIndexDirectory(checkoutPath, INDEX_DIRECTORY); replaceFile(directory, PUBLISHED_FILE, String(generation)); } - -/** - * Index files an older T3 wrote into the control directory, when it was also the SDK's data - * directory. Nothing reads them any more. - */ -export function removeLegacyIndexFiles(checkoutPath: string): void { - const directory = NodePath.join(checkoutPath, INDEX_DIRECTORY); - for (const name of [ - "codegraph.db", - "codegraph.db-wal", - "codegraph.db-shm", - "codegraph.lock", - "t3-index.json", - ]) { - NodeFS.rmSync(NodePath.join(directory, name), { force: true }); - } -} diff --git a/apps/server/src/project/knowledge/codeIndexLocks.test.ts b/apps/server/src/project/knowledge/codeIndexLocks.test.ts new file mode 100644 index 0000000000000..313cfbf254446 --- /dev/null +++ b/apps/server/src/project/knowledge/codeIndexLocks.test.ts @@ -0,0 +1,171 @@ +// @effect-diagnostics nodeBuiltinImport:off - isolated filesystem and SQLite fixtures exercise the cross-process lock protocol. +import * as NodeFS from "node:fs"; +import * as NodeOS from "node:os"; +import * as NodePath from "node:path"; +import { symlinksSupported } from "@t3tools/shared/testing/symlinks"; +import { expect, it } from "vite-plus/test"; + +import { + hasLegacyIndexFiles, + pinPublishedGeneration, + removeIndexRoot, + removeLegacyIndexFromCheckout, + tryLockBuild, +} from "./codeIndexLocks.ts"; +import { + linkCheckoutSources, + publishGeneration, + readPublishedGeneration, + writeIndexedCheckout, +} from "./codeIndexLayout.ts"; + +it("removes checkout links while preserving a pinned generation and its source files", () => { + const temporary = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-index-cleanup-")); + const checkout = NodePath.join(temporary, "checkout"); + const root = NodePath.join(temporary, "index"); + let pin: ReturnType | undefined; + try { + NodeFS.mkdirSync(NodePath.join(checkout, "src"), { recursive: true }); + NodeFS.writeFileSync(NodePath.join(checkout, "src/main.ts"), "export const main = 1;"); + NodeFS.writeFileSync(NodePath.join(checkout, "top.ts"), "export const top = 1;"); + writeIndexedCheckout(root, checkout); + linkCheckoutSources(root, checkout, ["src/main.ts", "top.ts"]); + publishGeneration(root, 1); + const generation = NodePath.join(root, ".codegraph-t3-g1"); + NodeFS.mkdirSync(generation); + NodeFS.writeFileSync(NodePath.join(generation, "codegraph.db"), "index payload"); + pin = pinPublishedGeneration(root); + + removeIndexRoot(root); + + expect(readPublishedGeneration(root)).toBeNull(); + expect(NodeFS.existsSync(generation)).toBe(true); + expect(NodeFS.existsSync(NodePath.join(root, "src"))).toBe(false); + expect(NodeFS.existsSync(NodePath.join(root, "top.ts"))).toBe(false); + expect(NodeFS.readFileSync(NodePath.join(checkout, "src/main.ts"), "utf8")).toBe( + "export const main = 1;", + ); + expect(NodeFS.readFileSync(NodePath.join(checkout, "top.ts"), "utf8")).toBe( + "export const top = 1;", + ); + + pin.release(); + pin = undefined; + removeIndexRoot(root); + expect(NodeFS.existsSync(root)).toBe(false); + } finally { + pin?.release(); + NodeFS.rmSync(temporary, { recursive: true, force: true }); + } +}); + +// Windows refuses to delete a file another handle has open, so there the first lock simply stands. +it.each([ + ["deleted", () => null], + ["deleted and taken by another builder", tryLockBuild], +] as const)("never lets two builders hold a lock %s before it landed", (_name, afterRemoval) => { + const root = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-index-lock-")); + const holders: Array<(() => void) | null> = []; + try { + holders.push( + tryLockBuild(root, () => { + removeIndexRoot(root); + holders.push(afterRemoval(root)); + }), + ); + holders.push(tryLockBuild(root)); + expect(holders.filter((holder) => holder !== null)).toHaveLength(1); + } finally { + for (const holder of holders) holder?.(); + NodeFS.rmSync(root, { recursive: true, force: true }); + } +}); + +const legacyCheckout = (files: ReadonlyArray) => { + const checkout = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-index-legacy-")); + NodeFS.writeFileSync(NodePath.join(checkout, "keep.txt"), "mine"); + for (const file of files) { + NodeFS.mkdirSync(NodePath.dirname(NodePath.join(checkout, file)), { recursive: true }); + NodeFS.writeFileSync(NodePath.join(checkout, file), ""); + } + return checkout; +}; + +it.each([ + [ + "a published index", + [ + ".codegraph-t3/.gitignore", + ".codegraph-t3/published", + ".codegraph-t3/build.lock", + ".codegraph-t3/building.lock", + ".codegraph-t3-g3/.gitignore", + ".codegraph-t3-g3/codegraph.db", + ".codegraph-t3-g3/codegraph.db-shm", + ".codegraph-t3-g3/codegraph.db-wal", + ".codegraph-t3-g3/readers.lock", + ".codegraph-t3-g3/t3-index.json", + ], + ], + ["only lock files", [".codegraph-t3/.gitignore", ".codegraph-t3/build.lock"]], + ["a generation with no control folder", [".codegraph-t3-g3/codegraph.db"]], +])("leaves nothing of T3's in a checkout holding %s", (_name, files) => { + const checkout = legacyCheckout(files); + try { + expect(hasLegacyIndexFiles(checkout)).toBe(true); + removeLegacyIndexFromCheckout(checkout); + expect(NodeFS.readdirSync(checkout)).toEqual(["keep.txt"]); + expect(hasLegacyIndexFiles(checkout)).toBe(false); + } finally { + NodeFS.rmSync(checkout, { recursive: true, force: true }); + } +}); + +it("keeps the user's files in an older version's control folder", () => { + const checkout = legacyCheckout([ + ".codegraph-t3/.gitignore", + ".codegraph-t3/published", + ".codegraph-t3/build.lock", + ".codegraph-t3/notes.txt", + ".codegraph-t3/mine/a.txt", + ".codegraph-t3-g3/codegraph.db", + ".codegraph-t3-g3/notes.txt", + ]); + try { + removeLegacyIndexFromCheckout(checkout); + expect( + NodeFS.readdirSync(NodePath.join(checkout, ".codegraph-t3"), { recursive: true }).toSorted(), + ).toEqual(["mine", NodePath.join("mine", "a.txt"), "notes.txt"]); + expect(NodeFS.readdirSync(NodePath.join(checkout, ".codegraph-t3-g3")).toSorted()).toEqual([ + "codegraph.db", + "notes.txt", + ]); + expect(hasLegacyIndexFiles(checkout)).toBe(false); + } finally { + NodeFS.rmSync(checkout, { recursive: true, force: true }); + } +}); + +it.skipIf(!symlinksSupported).each([ + [".codegraph-t3", "build.lock", "published"], + [".codegraph-t3-g1", "readers.lock", "codegraph.db"], +])("leaves linked locks inside a legacy %s untouched", (directory, lock, marker) => { + const temporary = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-index-link-")); + const checkout = NodePath.join(temporary, "checkout"); + const target = NodePath.join(temporary, "outside.sqlite"); + const legacy = NodePath.join(checkout, directory); + try { + NodeFS.mkdirSync(legacy, { recursive: true }); + NodeFS.writeFileSync(NodePath.join(legacy, marker), "untouched"); + NodeFS.symlinkSync(target, NodePath.join(legacy, lock), "file"); + + removeLegacyIndexFromCheckout(checkout); + + expect(NodeFS.existsSync(target)).toBe(false); + expect(NodeFS.readFileSync(NodePath.join(legacy, marker), "utf8")).toBe("untouched"); + expect(NodeFS.lstatSync(NodePath.join(legacy, lock)).isSymbolicLink()).toBe(true); + expect(hasLegacyIndexFiles(checkout)).toBe(false); + } finally { + NodeFS.rmSync(temporary, { recursive: true, force: true }); + } +}); diff --git a/apps/server/src/project/knowledge/codeIndexLocks.ts b/apps/server/src/project/knowledge/codeIndexLocks.ts index 2678771fe09dd..cddf8fab6ddc2 100644 --- a/apps/server/src/project/knowledge/codeIndexLocks.ts +++ b/apps/server/src/project/knowledge/codeIndexLocks.ts @@ -5,15 +5,16 @@ import * as NodeSqlite from "node:sqlite"; import { INDEX_DIRECTORY, + PUBLISHED_FILE, + clearCheckoutSources, generationDirectory, - ignoreIndexDirectory, + listGenerations, readPublishedGeneration, - validateIndexDirectory, } from "./codeIndexLayout.ts"; /** - * Servers on one machine share a checkout's index, so its builder and readers coordinate through - * SQLite's file locks (fcntl on POSIX, LockFileEx on Windows) on empty lock databases. The kernel + * Servers sharing a data directory share a checkout's index, so its builder and readers coordinate + * through SQLite's file locks (fcntl on POSIX, LockFileEx on Windows) on empty lock databases. The kernel * drops those locks when their process exits, however it exits, so a crashed server leaves no * record for a PID check to judge, and a recycled PID holds nothing. */ @@ -24,6 +25,7 @@ const BUILDING_LOCK = "building.lock"; const READERS_LOCK = "readers.lock"; // Waits out the momentary locks of probes and failed deletion attempts, never a build or a read. const MOMENTARY_LOCK_WAIT_MS = 1_000; +const isBuildLockFile = (name: string) => name === BUILD_LOCK || name === BUILDING_LOCK; const SQLITE_BUSY = 5; const SQLITE_CANTOPEN = 14; @@ -32,12 +34,28 @@ const sqliteCode = (cause: unknown) => ? cause.errcode & 0xff : undefined; +const fileIdentity = (path: string) => + NodeFS.statSync(path, { bigint: true, throwIfNoEntry: false })?.ino; + /** The connection holding `path`'s exclusive lock, or null while another connection holds a lock. */ -function lockExclusive(path: string, waitMs: number): NodeSqlite.DatabaseSync | null { +function lockExclusive( + path: string, + waitMs: number, + onOpened?: () => void, +): NodeSqlite.DatabaseSync | null { const database = new NodeSqlite.DatabaseSync(path); try { - // With no journal the lock never creates a -journal file beside it. - database.exec(`PRAGMA busy_timeout = ${waitMs}; PRAGMA journal_mode = OFF; BEGIN EXCLUSIVE;`); + const opened = fileIdentity(path); + onOpened?.(); + // Defensive SQLite ignores OFF; MEMORY avoids an open journal that Windows cannot delete. + database.exec( + `PRAGMA busy_timeout = ${waitMs}; PRAGMA journal_mode = MEMORY; BEGIN EXCLUSIVE;`, + ); + // A removal can delete this file before the lock lands, and a lock on it then excludes nobody. + if (opened === undefined || fileIdentity(path) !== opened) { + database.close(); + return null; + } return database; } catch (cause) { database.close(); @@ -70,13 +88,14 @@ function lockShared(path: string, role: "reader" | "probe") { } } -/** Takes the checkout's build lock and returns its release, or null while another builder holds it. */ -export function tryLockBuild(checkoutPath: string): (() => void) | null { - const control = NodePath.join(checkoutPath, INDEX_DIRECTORY); +/** Takes the index root's build lock and returns its release, or null while another builder holds it. */ +export function tryLockBuild( + indexRoot: string, + onBuildLockOpened?: () => void, +): (() => void) | null { + const control = NodePath.join(indexRoot, INDEX_DIRECTORY); NodeFS.mkdirSync(control, { recursive: true }); - validateIndexDirectory(checkoutPath, INDEX_DIRECTORY); - ignoreIndexDirectory(checkoutPath, INDEX_DIRECTORY); - const build = lockExclusive(NodePath.join(control, BUILD_LOCK), 0); + const build = lockExclusive(NodePath.join(control, BUILD_LOCK), 0, onBuildLockOpened); if (build === null) return null; try { const building = lockExclusive(NodePath.join(control, BUILDING_LOCK), MOMENTARY_LOCK_WAIT_MS); @@ -95,10 +114,10 @@ const closeLock = (lock: ReturnType) => { if (lock !== null && lock !== "missing") lock.close(); }; -/** Whether any process, this one included, is building the checkout's index now. */ -export function isBuildRunning(checkoutPath: string): boolean { +/** Whether any process, this one included, is building the index now. */ +export function isBuildRunning(indexRoot: string): boolean { try { - const probe = lockShared(NodePath.join(checkoutPath, INDEX_DIRECTORY, BUILDING_LOCK), "probe"); + const probe = lockShared(NodePath.join(indexRoot, INDEX_DIRECTORY, BUILDING_LOCK), "probe"); closeLock(probe); return probe === null; } catch { @@ -111,16 +130,8 @@ export function isBuildRunning(checkoutPath: string): boolean { * instead. Callers ensure no build writes it: the builder holds the build lock, and a reader * removes only generations older than the published one. */ -export function removeGenerationUnlessRead(checkoutPath: string, generation: number): void { - const name = generationDirectory(generation); - const directory = NodePath.join(checkoutPath, name); - try { - validateIndexDirectory(checkoutPath, name); - } catch { - // Nothing reads a generation that fails validation; `rm` unlinks a planted link unfollowed. - NodeFS.rmSync(directory, { recursive: true, force: true }); - return; - } +export function removeGenerationUnlessRead(indexRoot: string, generation: number): void { + const directory = NodePath.join(indexRoot, generationDirectory(generation)); let lock: NodeSqlite.DatabaseSync | null; try { lock = lockExclusive(NodePath.join(directory, READERS_LOCK), 0); @@ -142,6 +153,177 @@ export function removeGenerationUnlessRead(checkoutPath: string, generation: num NodeFS.rmSync(directory, { recursive: true, force: true }); } +function clearControlFiles(control: string): void { + for (const name of NodeFS.readdirSync(control)) { + if (isBuildLockFile(name)) continue; + NodeFS.rmSync(NodePath.join(control, name), { recursive: true, force: true }); + } +} + +/** False while a process holds one open: Windows refuses to delete an open file. */ +function removeBuildLockFiles(control: string): boolean { + try { + for (const name of [BUILDING_LOCK, BUILD_LOCK]) { + NodeFS.rmSync(NodePath.join(control, name), { force: true }); + } + return true; + } catch (cause) { + const code = (cause as NodeJS.ErrnoException).code; + if (code === "EBUSY" || code === "EPERM" || code === "EACCES") return false; + throw cause; + } +} + +const removeIfEmpty = (directory: string) => { + try { + NodeFS.rmdirSync(directory); + } catch { + // What remains is in use or the user's; a later sweep retries. + } +}; + +/** + * Runs `clear` under the build lock, then removes the lock files and their folder. The files go + * while held where the OS allows it: a later builder gets a new file, and an earlier one that had + * only opened its file finds it gone. Elsewhere they go on release, which an open file refuses. + */ +function clearUnderBuildLock(root: string, clear: (control: string) => void): void { + const control = NodePath.join(root, INDEX_DIRECTORY); + const release = tryLockBuild(root); + if (release === null) return; + let removed = false; + try { + clear(control); + removed = removeBuildLockFiles(control); + } finally { + release(); + } + if (!removed) removeBuildLockFiles(control); + removeIfEmpty(control); +} + +/** + * Removes one index root: the published pointer first, so new readers see "unindexed", then every + * generation a reader is not pinning, then the checkout's links. A build elsewhere leaves it all + * for the next sweep. + * + * Nothing here may remove recursively from the root: its entries are links into a user's checkout. + */ +export function removeIndexRoot(indexRoot: string): void { + if (!NodeFS.existsSync(indexRoot)) return; + clearUnderBuildLock(indexRoot, (control) => { + NodeFS.rmSync(NodePath.join(control, PUBLISHED_FILE), { force: true }); + for (const generation of listGenerations(indexRoot)) { + removeGenerationUnlessRead(indexRoot, generation); + } + clearCheckoutSources(indexRoot); + clearControlFiles(control); + }); + removeIfEmpty(indexRoot); +} + +// Files an older T3 left in a checkout when the index lived beside its sources. +const LEGACY_CONTROL_MARKERS = [ + PUBLISHED_FILE, + BUILD_LOCK, + BUILDING_LOCK, + "codegraph.db", + "t3-index.json", +]; +const LEGACY_GENERATION_MARKERS = ["t3-index.json", "codegraph.db"]; +// Everything T3 and the SDK wrote to an old index folder; anything else there is the user's. +const LEGACY_INDEX_FILES = [ + ...LEGACY_CONTROL_MARKERS, + READERS_LOCK, + ".gitignore", + "codegraph.db-shm", + "codegraph.db-wal", + "codegraph.lock", + "daemon.pid", + "rebuild.pid", + "writer.pid", +]; + +const isRealDirectory = (path: string): boolean => { + try { + return NodeFS.lstatSync(path).isDirectory(); + } catch { + return false; + } +}; + +/** Legacy lock paths are checkout-controlled, so none may follow a link or share an outside inode. */ +function isT3Directory( + path: string, + names: ReadonlyArray, + allowed?: ReadonlyArray, +): boolean { + if (!isRealDirectory(path)) return false; + try { + const entries = NodeFS.readdirSync(path); + return ( + entries.some((name) => names.includes(name)) && + (allowed === undefined || entries.every((name) => allowed.includes(name))) && + entries.every((name) => { + const entry = NodeFS.lstatSync(NodePath.join(path, name)); + return entry.isDirectory() || (entry.isFile() && entry.nlink === 1); + }) + ); + } catch { + return false; + } +} + +/** + * The folders an older T3 left inside a checkout, when they are provably T3's. Anything else that + * happens to share a name belongs to the user. + */ +function legacyIndexFolders( + checkoutPath: string, +): { readonly generations: ReadonlyArray } | null { + const control = NodePath.join(checkoutPath, INDEX_DIRECTORY); + let exists: boolean; + try { + exists = NodeFS.lstatSync(control, { throwIfNoEntry: false }) !== undefined; + } catch { + return null; + } + if (exists && !isT3Directory(control, LEGACY_CONTROL_MARKERS)) return null; + const generations = listGenerations(checkoutPath).filter((generation) => + isT3Directory( + NodePath.join(checkoutPath, generationDirectory(generation)), + LEGACY_GENERATION_MARKERS, + LEGACY_INDEX_FILES, + ), + ); + return exists || generations.length > 0 ? { generations } : null; +} + +/** Whether a checkout still holds the folders an older T3 kept inside it, without throwing. */ +export function hasLegacyIndexFiles(checkoutPath: string): boolean { + try { + return legacyIndexFolders(checkoutPath) !== null; + } catch { + return false; + } +} + +/** Removes the index folders an older T3 left inside a checkout, and nothing else there. */ +export function removeLegacyIndexFromCheckout(checkoutPath: string): void { + const legacy = legacyIndexFolders(checkoutPath); + if (legacy === null) return; + // An older server may still be building; it finishes and this runs on a later sweep. + clearUnderBuildLock(checkoutPath, (control) => { + NodeFS.rmSync(NodePath.join(control, PUBLISHED_FILE), { force: true }); + for (const generation of legacy.generations) { + removeGenerationUnlessRead(checkoutPath, generation); + } + for (const name of LEGACY_INDEX_FILES) { + if (!isBuildLockFile(name)) NodeFS.rmSync(NodePath.join(control, name), { force: true }); + } + }); +} + export interface GenerationPin { readonly generation: number | null; readonly release: () => void; @@ -157,15 +339,14 @@ const unpinned = (generation: number | null): GenerationPin => ({ * lock is held: a generation is deleted only after it stops being published, and never published * again, so a pin that still finds it published holds a lock its deleter has to wait for. */ -export function pinPublishedGeneration(checkoutPath: string): GenerationPin { - let generation = readPublishedGeneration(checkoutPath); +export function pinPublishedGeneration(indexRoot: string): GenerationPin { + let generation = readPublishedGeneration(indexRoot); while (generation !== null) { const name = generationDirectory(generation); - validateIndexDirectory(checkoutPath, name); - const lock = lockShared(NodePath.join(checkoutPath, name, READERS_LOCK), "reader"); + const lock = lockShared(NodePath.join(indexRoot, name, READERS_LOCK), "reader"); let current: number | null; try { - current = readPublishedGeneration(checkoutPath); + current = readPublishedGeneration(indexRoot); } catch (cause) { closeLock(lock); throw cause; @@ -179,7 +360,7 @@ export function pinPublishedGeneration(checkoutPath: string): GenerationPin { generation: pinned, release: () => { lock.close(); - retireIfSuperseded(checkoutPath, pinned); + retireIfSuperseded(indexRoot, pinned); }, }; } @@ -189,12 +370,12 @@ export function pinPublishedGeneration(checkoutPath: string): GenerationPin { return unpinned(null); } -function retireIfSuperseded(checkoutPath: string, generation: number): void { +function retireIfSuperseded(indexRoot: string, generation: number): void { try { - const published = readPublishedGeneration(checkoutPath); + const published = readPublishedGeneration(indexRoot); // Builds write only generations newer than the published one. if (published !== null && generation < published) { - removeGenerationUnlessRead(checkoutPath, generation); + removeGenerationUnlessRead(indexRoot, generation); } } catch { // The next build sweeps a generation left behind. diff --git a/apps/server/src/project/knowledge/codeIntelligenceWorker.ts b/apps/server/src/project/knowledge/codeIntelligenceWorker.ts index b28220a262e9b..9d6733b07002c 100644 --- a/apps/server/src/project/knowledge/codeIntelligenceWorker.ts +++ b/apps/server/src/project/knowledge/codeIntelligenceWorker.ts @@ -34,12 +34,14 @@ import type { } from "@t3tools/contracts"; import { - ignoreIndexDirectory, - readFileNoFollow, - validateIndexDirectory, + CHECKOUT_CONFIG_ENTRIES, + clearCheckoutSources, + linkCheckoutSources, } from "./codeIndexLayout.ts"; const CATALOG_REVISION = "@colbymchenry/codegraph@1.6.2"; +// Programming languages outside the SDK's catalog, which it reports as unknown like any asset. +const UNREADABLE_SOURCE = /\.(?:clj|cljs|cr|elm|ex|exs|fs|fsx|gd|hs|jl|ml|mli|nim|pl|pm|rkt|zig)$/i; const MAX_RESULTS = 200; const MAX_FILES_FOR_UNRESOLVED = 50; const MAX_LISTED_FILES = 10_000; @@ -66,6 +68,8 @@ interface CodeGraphModule { interface CheckoutListing { readonly checkoutPath: string; + // The SDK's project root: it holds the index and links back to the checkout's sources. + readonly indexRoot: string; readonly files: ReadonlyArray; readonly filesListed: number; readonly listingTruncated: boolean; @@ -219,16 +223,25 @@ interface IndexMeta { // Listed sources the build deliberately left out, such as build output: coverage gaps, not // newly added files a refresh would pick up. readonly skippedPaths: ReadonlyArray; + // Whether the build ranked an over-cap listing; older builds kept whatever sorted first. + readonly ranked: boolean; + readonly configurationDigest: string | null; } -function readIndexMeta(checkoutPath: string, indexDirectory: string): IndexMeta { +function readIndexMeta(indexRoot: string, indexDirectory: string): IndexMeta { try { const raw: unknown = JSON.parse( - readFileNoFollow(NodePath.join(checkoutPath, indexDirectory, INDEX_META_FILE)), + NodeFS.readFileSync(NodePath.join(indexRoot, indexDirectory, INDEX_META_FILE), "utf8"), ); if (typeof raw !== "object" || raw === null) throw new Error("invalid"); - const { commit, generation, indexedAt, skippedPaths } = raw as Record; + const { commit, generation, indexedAt, skippedPaths, ranked, configurationDigest } = + raw as Record; return { + ranked: ranked === true, + configurationDigest: + typeof configurationDigest === "string" && /^[0-9a-f]{64}$/.test(configurationDigest) + ? configurationDigest + : null, commit: typeof commit === "string" && /^[0-9a-f]{7,64}$/.test(commit) ? commit : null, generation: typeof generation === "number" && Number.isSafeInteger(generation) && generation >= 0 @@ -243,12 +256,37 @@ function readIndexMeta(checkoutPath: string, indexDirectory: string): IndexMeta : [], }; } catch { - return { commit: null, generation: 0, indexedAt: null, skippedPaths: [] }; + return { + commit: null, + generation: 0, + indexedAt: null, + skippedPaths: [], + ranked: false, + configurationDigest: null, + }; + } +} + +function checkoutConfigurationDigest(checkoutPath: string): string { + const digest = NodeCrypto.createHash("sha256"); + for (const name of CHECKOUT_CONFIG_ENTRIES) { + digest.update(name).update("\0"); + try { + const contents = NodeFS.readFileSync(NodePath.join(checkoutPath, name)); + digest.update("present\0").update(contents); + } catch (cause) { + // Anything unreadable, such as a folder of that name, configures nothing. + digest.update( + (cause as NodeJS.ErrnoException).code === "ENOENT" ? "missing\0" : "unreadable\0", + ); + } + digest.update("\0"); } + return digest.digest("hex"); } -function writeIndexMeta(checkoutPath: string, indexDirectory: string, meta: IndexMeta): void { - const target = NodePath.join(checkoutPath, indexDirectory, INDEX_META_FILE); +function writeIndexMeta(indexRoot: string, indexDirectory: string, meta: IndexMeta): void { + const target = NodePath.join(indexRoot, indexDirectory, INDEX_META_FILE); const temporary = `${target}.${process.pid}.tmp`; try { // An exclusive create never follows a planted link, and the rename replaces the entry @@ -565,11 +603,60 @@ function preferDeclarations(tests: ReadonlyArray): CodeGraphNode[ ); } -function authorizedListing(listing: CheckoutListing): string[] { - return listing.files.slice(0, MAX_LISTED_FILES).flatMap((path) => { +type IsSupported = (path: string) => boolean; + +function supportedPaths(sdk: Omit): IsSupported { + const supported = new Set(sdk.getSupportedLanguages()); + return (path) => { + const language = sdk.detectLanguage(path); + return language !== "unknown" && supported.has(language); + }; +} + +/** + * Orders a listing for the file cap: what the analyzer can parse before what it cannot, and + * first-party paths before dot-directories such as vendored references. Otherwise whatever sorts + * first decides what is indexed, and a large `.repos` leaves a project's own code out entirely. + */ +export function rankListing(files: ReadonlyArray, isSupported: IsSupported): string[] { + const ranks: [string[], string[], string[], string[]] = [[], [], [], []]; + for (const path of files) { + const hidden = path.startsWith(".") || path.includes("/."); + ranks[(isSupported(path) ? 0 : 2) + (hidden ? 1 : 0)]!.push(path); + } + return ranks.flat(); +} + +function isListingTruncated(listing: CheckoutListing): boolean { + return listing.listingTruncated || listing.files.length > MAX_LISTED_FILES; +} + +interface AuthorizedListing { + readonly paths: ReadonlyArray; + // Listed paths that resolve outside the checkout. + readonly rejected: number; +} + +function authorizedListing(listing: CheckoutListing, isSupported: IsSupported): AuthorizedListing { + const candidates = + listing.files.length > MAX_LISTED_FILES + ? rankListing(listing.files, isSupported) + : listing.files; + const paths: string[] = []; + let rejected = 0; + for (const path of candidates) { + if (paths.length >= MAX_LISTED_FILES) break; + const absolute = NodePath.resolve(listing.checkoutPath, path); const authorized = authorizedSourcePath(listing.checkoutPath, path); - return authorized === null ? [] : [authorized]; - }); + if (authorized === null) { + // A tracked file deleted from the working tree is no coverage gap; an escaping link is. + if (NodeFS.existsSync(absolute)) rejected += 1; + // Git lists a submodule as a path; handing the SDK a directory would walk past the allowlist. + } else if (NodeFS.statSync(absolute, { throwIfNoEntry: false })?.isDirectory() !== true) { + paths.push(authorized); + } + } + return { paths, rejected }; } /** Indexed sources that still exist but now resolve outside the checkout, such as a new link. */ @@ -593,22 +680,27 @@ export async function buildIndexInWorker( onProgress: (progress: Pick) => void, ): Promise<{ readonly changed: boolean }> { const { CodeGraph, detectLanguage, getSupportedLanguages } = sdk(); - const listedPaths = authorizedListing(input); + const isSupported = supportedPaths({ detectLanguage, getSupportedLanguages }); + const { paths: listedPaths } = authorizedListing(input, isSupported); const commit = headCommit(input.checkoutPath); + const configurationDigest = checkoutConfigurationDigest(input.checkoutPath); let from = input.from; if (from !== null) { - validateIndexDirectory(input.checkoutPath, from); useIndexDirectory(from); - if (!CodeGraph.isInitialized(input.checkoutPath)) from = null; + const meta = readIndexMeta(input.indexRoot, from); + if ( + !CodeGraph.isInitialized(input.indexRoot) || + // Filtering changes must remove old entries as well as reconsider previously skipped files. + meta.configurationDigest !== configurationDigest || + // Successors keep every file they inherit, so one that spent the cap unranked never recovers. + (isListingTruncated(input) && !meta.ranked) + ) + from = null; } if (from !== null) { - const published = await CodeGraph.open(input.checkoutPath, { sync: false, readOnly: true }); + const published = await CodeGraph.open(input.indexRoot, { sync: false, readOnly: true }); try { - const meta = readIndexMeta(input.checkoutPath, from); - const { isSupported } = classifyIndexCoverage(input, listedPaths, published, { - languages: getSupportedLanguages(), - detectLanguage, - }); + const meta = readIndexMeta(input.indexRoot, from); if ( meta.commit === commit && !hasUnauthorizedFiles(input.checkoutPath, published) && @@ -626,27 +718,31 @@ export async function buildIndexInWorker( published.close(); } } - const target = NodePath.join(input.checkoutPath, input.to); + const target = NodePath.join(input.indexRoot, input.to); // Exclusive: a leftover entry, link or not, fails the build instead of being written through. NodeFS.mkdirSync(target); if (from !== null) { for (const name of INDEX_DATABASE_FILES) { - const source = NodePath.join(input.checkoutPath, from, name); + const source = NodePath.join(input.indexRoot, from, name); if (NodeFS.existsSync(source)) { - NodeFS.copyFileSync(source, NodePath.join(target, name), NodeFS.constants.COPYFILE_EXCL); + NodeFS.copyFileSync( + source, + NodePath.join(target, name), + // FICLONE falls back to a normal copy where reflinks are unavailable. + NodeFS.constants.COPYFILE_EXCL | NodeFS.constants.COPYFILE_FICLONE, + ); } } } useIndexDirectory(input.to); let graph = from !== null - ? await CodeGraph.open(input.checkoutPath, { sync: false }) - : await CodeGraph.init(input.checkoutPath, { index: false }); - ignoreIndexDirectory(input.checkoutPath, input.to); + ? await CodeGraph.open(input.indexRoot, { sync: false }) + : await CodeGraph.init(input.indexRoot, { index: false }); try { if (hasUnauthorizedFiles(input.checkoutPath, graph)) { graph.close(); - graph = await CodeGraph.recreate(input.checkoutPath); + graph = await CodeGraph.recreate(input.indexRoot); } const reconcilePaths = new Set(listedPaths); for (const file of graph.getFiles()) { @@ -656,18 +752,24 @@ export async function buildIndexInWorker( else if (!NodeFS.existsSync(NodePath.resolve(input.checkoutPath, file.path))) reconcilePaths.add(file.path); } + // From the reconcile set, not the listing: the cap can drop a top-level entry the graph still + // holds files under, and the SDK would read those as deleted. + clearCheckoutSources(input.indexRoot); + linkCheckoutSources(input.indexRoot, input.checkoutPath, [...reconcilePaths]); // `sync` reads an empty `paths` as "reconcile the whole checkout", a walk that follows // links out of it. It and the no-index `init`/`open`/`recreate` are the SDK's only calls // here, so this allowlist bounds everything the index reads. if (reconcilePaths.size > 0) await graph.sync({ onProgress, paths: [...reconcilePaths] }); const indexedPaths = new Set(graph.getFiles().map((file) => file.path)); - writeIndexMeta(input.checkoutPath, input.to, { + writeIndexMeta(input.indexRoot, input.to, { commit, generation: input.generation, indexedAt: Date.now(), // A completed sync may deliberately skip build output. Those paths are coverage gaps, // not newly added files that the next refresh can fix. skippedPaths: listedPaths.filter((path) => !indexedPaths.has(path)).toSorted(), + ranked: true, + configurationDigest, }); } finally { graph.close(); @@ -680,7 +782,9 @@ export async function inspectCodeInWorker( input: CodeIntelligenceWorkerInput, ): Promise { const { CodeGraph, detectLanguage, getSupportedLanguages } = sdk(); - const listedPaths = authorizedListing(input); + const isSupported = supportedPaths({ detectLanguage, getSupportedLanguages }); + const listing = authorizedListing(input, isSupported); + const listedPaths = listing.paths; const listedPathSet = new Set(listedPaths); const queryPath = input.query.operation === "dependencies" || @@ -709,26 +813,27 @@ export async function inspectCodeInWorker( } const commit = headCommit(input.checkoutPath); if (input.indexDirectory !== null) { - validateIndexDirectory(input.checkoutPath, input.indexDirectory); useIndexDirectory(input.indexDirectory); } - if (input.indexDirectory === null || !CodeGraph.isInitialized(input.checkoutPath)) { + if (input.indexDirectory === null || !CodeGraph.isInitialized(input.indexRoot)) { return unindexedResult(input, getSupportedLanguages(), commit); } - const graph = await CodeGraph.open(input.checkoutPath, { sync: false, readOnly: true }); + const graph = await CodeGraph.open(input.indexRoot, { sync: false, readOnly: true }); try { - const meta = readIndexMeta(input.checkoutPath, input.indexDirectory); + const meta = readIndexMeta(input.indexRoot, input.indexDirectory); if (input.query.operation === "freshness") { - return inspectFreshness(input, listedPaths, graph, meta, commit, { - detectLanguage, - getSupportedLanguages, - }); + return inspectFreshness( + input, + listing, + graph, + meta, + commit, + getSupportedLanguages(), + isSupported, + ); } const languageCatalog = getSupportedLanguages(); - const indexCoverage = classifyIndexCoverage(input, listedPaths, graph, { - languages: languageCatalog, - detectLanguage, - }); + const indexCoverage = classifyIndexCoverage(input, listing, graph, isSupported); const files = indexCoverage.files; const stats = graph.getStats(); const limit = Math.min( @@ -1095,7 +1200,7 @@ export async function inspectCodeInWorker( "Relations are static candidates. Import/name resolution can misresolve shadowed names, and SDK confidence is a heuristic score rather than certainty.", ); limitations.push(...indexCoverage.limitations); - let coverageIncomplete = indexCoverage.limitations.length > 0; + let coverageIncomplete = indexCoverage.incomplete; if (sourceDigests.some((source) => !source.verified)) { coverageIncomplete = true; limitations.push( @@ -1111,15 +1216,21 @@ export async function inspectCodeInWorker( input.checkoutPath, graph, listedPaths, - indexCoverage.isSupported, + isSupported, meta.skippedPaths, ); + const configurationChanged = + meta.configurationDigest !== checkoutConfigurationDigest(input.checkoutPath); if (changed.length > 0) { limitations.push( `${changed.length} source ${changed.length === 1 ? "file has" : "files have"} changed since this index generation was built.`, ); } - const stale = changed.length > 0 || sourceDigests.some((source) => !source.verified); + if (configurationChanged) limitations.push("The checkout's index configuration has changed."); + const stale = + configurationChanged || + changed.length > 0 || + sourceDigests.some((source) => !source.verified); const coverage: ProjectCodeCoverage = { state: stale ? "stale" : coverageIncomplete ? "partial" : "complete", catalogRevision: CATALOG_REVISION, @@ -1178,40 +1289,29 @@ export async function inspectCodeInWorker( /** * How much of the listed checkout the index covers, counted the same way for a query and for - * freshness. Any limitation it reports makes coverage partial. + * freshness. Every limitation but unsupported files makes coverage partial: docs, data and assets + * fill any repository, so they are a gap only when nothing else could be indexed. Source the SDK + * cannot read is a gap once it is a real share of the code, not for one stray script. */ function classifyIndexCoverage( input: CheckoutListing, - listedPaths: ReadonlyArray, + { paths: listedPaths, rejected }: AuthorizedListing, graph: CodeGraphInstance, - catalog: { - readonly languages: ReadonlyArray; - readonly detectLanguage: (path: string) => Language; - }, + isSupported: IsSupported, ) { - const supported = new Set(catalog.languages); - const isSupported = (path: string) => { - const language = catalog.detectLanguage(path); - return language !== "unknown" && supported.has(language); - }; const listed = new Set(listedPaths); const files = graph.getFiles().filter((file) => listed.has(file.path)); const indexedPaths = new Set(files.map((file) => file.path)); - const filesUnsupported = listedPaths.filter((path) => !isSupported(path)).length; + const filesUnsupported = listedPaths.filter( + (path) => !isSupported(path) && !indexedPaths.has(path), + ).length; const filesFailed = files.filter((file) => (file.errors?.length ?? 0) > 0).length; const filesSkipped = - input.files.slice(0, MAX_LISTED_FILES).length - - listedPaths.length + - listedPaths.filter((path) => isSupported(path) && !indexedPaths.has(path)).length; + rejected + listedPaths.filter((path) => isSupported(path) && !indexedPaths.has(path)).length; const pendingReferences = graph.getPendingReferenceCount(); const buildState = graph.getIndexState(); const limitations: string[] = []; - if (input.listingTruncated || input.files.length > MAX_LISTED_FILES) - limitations.push("Workspace file listing is truncated."); - if (filesUnsupported > 0) - limitations.push( - `${filesUnsupported} listed files use languages outside the SDK runtime catalog.`, - ); + if (isListingTruncated(input)) limitations.push("Workspace file listing is truncated."); if (filesSkipped > 0) limitations.push( `${filesSkipped} listed source files were outside the checkout or were not indexed.`, @@ -1223,6 +1323,17 @@ function classifyIndexCoverage( ); if (buildState !== null && buildState !== "complete") limitations.push(`The last full index run reported '${buildState}'.`); + const unreadableSource = listedPaths.filter( + (path) => UNREADABLE_SOURCE.test(path) && !isSupported(path) && !indexedPaths.has(path), + ).length; + const incomplete = + limitations.length > 0 || + (filesUnsupported > 0 && files.length === 0) || + unreadableSource * 20 > files.length + unreadableSource; + if (filesUnsupported > 0) + limitations.push( + `${filesUnsupported} listed files use languages outside the SDK runtime catalog.`, + ); return { files, filesSkipped, @@ -1230,7 +1341,7 @@ function classifyIndexCoverage( filesFailed, pendingReferences, limitations, - isSupported, + incomplete, }; } @@ -1305,25 +1416,25 @@ function unindexedResult( function inspectFreshness( input: CodeIntelligenceWorkerInput, - listedPaths: ReadonlyArray, + listing: AuthorizedListing, graph: CodeGraphInstance, meta: IndexMeta, commit: string | null, - { detectLanguage, getSupportedLanguages }: Omit, + languages: ReadonlyArray, + isSupported: IsSupported, ): ProjectCodeQueryResult { - const languages = getSupportedLanguages(); - const indexCoverage = classifyIndexCoverage(input, listedPaths, graph, { - languages, - detectLanguage, - }); + const indexCoverage = classifyIndexCoverage(input, listing, graph, isSupported); const changed = changedSinceIndex( input.checkoutPath, graph, - listedPaths, - indexCoverage.isSupported, + listing.paths, + isSupported, meta.skippedPaths, ); - const incomplete = indexCoverage.limitations.length > 0; + const configurationChanged = + meta.configurationDigest !== checkoutConfigurationDigest(input.checkoutPath); + const stale = changed.length > 0 || configurationChanged; + const { incomplete } = indexCoverage; const limitations = [ ...(changed.length > 0 ? [ @@ -1331,12 +1442,13 @@ function inspectFreshness( ] : []), ...indexCoverage.limitations, + ...(configurationChanged ? ["The checkout's index configuration has changed."] : []), ]; return emptyResult( input, { ...baseCoverage(input, languages), - state: changed.length > 0 ? "stale" : incomplete ? "partial" : "complete", + state: stale ? "stale" : incomplete ? "partial" : "complete", filesIndexed: indexCoverage.files.length, filesSkipped: indexCoverage.filesSkipped, filesUnsupported: indexCoverage.filesUnsupported, @@ -1346,7 +1458,7 @@ function inspectFreshness( limitations, }, { - state: changed.length > 0 ? "stale" : incomplete ? "partial" : "ready", + state: stale ? "stale" : incomplete ? "partial" : "ready", indexedCommit: meta.commit, headCommit: commit, generation: meta.generation, diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index 74b917bdce8ed..06c431a4e9aab 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -8,6 +8,7 @@ import * as ConversationSummaries from "./project/conversationSummaries/Conversa import * as ModelEvaluations from "./modelEvaluations/ModelEvaluations.ts"; import * as ExternalHistory from "./project/externalHistory/ExternalHistory.ts"; import * as ProjectLoom from "./project/loom/ProjectLoom.ts"; +import * as CodeIndexReactor from "./project/knowledge/CodeIndexReactor.ts"; import * as CodeIntelligence from "./project/knowledge/CodeIntelligence.ts"; import * as FileHistory from "./project/knowledge/FileHistory.ts"; import * as ProviderEventLoggers from "@t3tools/provider-core/server/ProviderEventLoggers"; @@ -657,14 +658,16 @@ const layerProviderInstallationRefresh = Layer.effectDiscard( }), ); +const layerStorageCleanup = StorageCleanup.layer.pipe( + Layer.provide(Layer.merge(ProjectionStoreV2.layer, WorktreeBranches.layer)), +); + const layerRuntimeCoreDependenciesBase = Layer.mergeAll( AgentAwarenessRelay.layer, HeldHooksWaker.layer, layerThreadSettlementWorker, layerIdleCompactionWorker, - StorageCleanup.layer.pipe( - Layer.provide(Layer.merge(ProjectionStoreV2.layer, WorktreeBranches.layer)), - ), + layerStorageCleanup, DatabaseStorage.layer.pipe(Layer.provide(ProjectionMaintenance.layer)), layerThreadPullRequestWorker, DelegatedTaskAnalytics.layer, @@ -694,6 +697,12 @@ const layerRuntimeCoreDependenciesBase = Layer.mergeAll( yield* service.start(); }), ).pipe(Layer.provide(DelegatedTaskQuestionReactor.layer), Layer.provide(ProjectionStoreV2.layer)), + CodeIndexReactor.layer.pipe( + Layer.provide(layerStorageCleanup), + Layer.provide(ProjectionStoreV2.layer), + Layer.provide(ProjectStore.layer), + Layer.provide(layerServerSettings), + ), ProviderUsageLimitsIngestion.layer, layerProviderInstallationRefresh, ReplayMarkers.layer, @@ -788,7 +797,14 @@ const layerRuntimeCoreDependencies = layerRuntimeCoreDependenciesBase.pipe( ), ); -const layerCodeIntelligence = Layer.mergeAll(CodeIntelligence.layer, FileHistory.layer).pipe( +const layerCodeIntelligence = Layer.mergeAll( + CodeIntelligence.layer.pipe( + Layer.provide(GitVcsDriver.layerVcs), + // Layers memoize by reference, so this is the same settings instance the server uses. + Layer.provide(layerServerSettings), + ), + FileHistory.layer, +).pipe( Layer.provideMerge(Layer.mergeAll(ProjectStore.layer, ProjectionStoreV2.layer, layerWorkspace)), ); diff --git a/apps/server/src/storageCleanup.test.ts b/apps/server/src/storageCleanup.test.ts index ad4f7b74e9b15..0c85dda9e172b 100644 --- a/apps/server/src/storageCleanup.test.ts +++ b/apps/server/src/storageCleanup.test.ts @@ -8,7 +8,7 @@ import * as Fiber from "effect/Fiber"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; -import type * as Path from "effect/Path"; +import * as Path from "effect/Path"; import type * as Scope from "effect/Scope"; import * as Stream from "effect/Stream"; import { SCAN_CACHE_FILE_NAME } from "./usage/usageScanCache.ts"; @@ -23,6 +23,7 @@ import { } from "@t3tools/contracts"; import * as HostProcess from "@t3tools/shared/HostProcess"; import * as ServerConfig from "./config.ts"; +import * as CodeIntelligence from "./project/knowledge/CodeIntelligence.ts"; import * as StorageCleanup from "./storageCleanup.ts"; import * as GitVcsDriver from "./vcs/GitVcsDriver.ts"; import * as GitManager from "./git/GitManager.ts"; @@ -321,6 +322,7 @@ const cleanupFixture = ( ) => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; const config = yield* ServerConfig.ServerConfig; const git = yield* GitVcsDriver.GitVcsDriver; const repo = yield* fs.realPath(yield* fs.makeTempDirectoryScoped({ prefix: "cleanup-repo-" })); @@ -334,7 +336,7 @@ const cleanupFixture = ( yield* command(repo, ["add", "."]); yield* command(repo, ["commit", "-m", "initial"]); yield* fs.makeDirectory(config.worktreesDir, { recursive: true }); - const worktree = `${config.worktreesDir}/feature`; + const worktree = path.join(config.worktreesDir, "feature"); yield* command(repo, ["worktree", "add", "-b", "feature", worktree]); let settings: import("@t3tools/contracts").ServerSettings = { ...DEFAULT_SERVER_SETTINGS, @@ -355,9 +357,18 @@ const cleanupFixture = ( let terminalListener: ((event: TerminalMetadataStreamEvent) => Effect.Effect) | undefined; let events: Stream.Stream = Stream.empty; const activated = yield* Deferred.make(); + const indexRemovals: string[] = []; + let onIndexRemoval: Effect.Effect = Effect.void; const context = yield* Layer.build( StorageCleanup.layer.pipe(Layer.provide(WorktreeBranches.layer)), ).pipe( + Effect.provideService(CodeIntelligence.CodeIntelligence, { + removeCheckoutIndex: (checkoutPath: string) => + Effect.gen(function* () { + yield* onIndexRemoval; + indexRemovals.push(checkoutPath); + }), + } as unknown as CodeIntelligence.CodeIntelligence["Service"]), Effect.provideService(GitVcsDriver.GitVcsDriver, { ...git, execute: (input) => @@ -522,6 +533,7 @@ const cleanupFixture = ( editBeforeRemoval = true; }, fs, + path, worktree, config, command, @@ -596,6 +608,10 @@ const cleanupFixture = ( }, }) ?? Effect.void, fullStatusCalls: () => fullStatusCalls, + indexRemovals: () => indexRemovals, + setOnIndexRemoval: (hook: Effect.Effect) => { + onIndexRemoval = hook; + }, snapshotReads: () => snapshotReads, pullRequestLookups, activate: (domainEvents: Stream.Stream) => { @@ -801,27 +817,45 @@ it.live("keeps submodules when stored repository metadata is malformed", () => ), ); -it.live("preserves late tracked edits when submodules were never initialized", () => - runCleanupTest( - Effect.gen(function* () { - const fixture = yield* cleanupFixture(); - const { service, fs, worktree, command } = fixture; - const head = (yield* command(worktree, ["rev-parse", "HEAD"])).stdout.trim(); - yield* fs.writeFileString( - `${worktree}/.gitmodules`, - '[submodule "module"]\n\tpath = module\n\turl = /unused\n', - ); - yield* command(worktree, ["add", ".gitmodules"]); - yield* command(worktree, ["update-index", "--add", "--cacheinfo", `160000,${head},module`]); - yield* command(worktree, ["commit", "-m", "record uninitialized module"]); - yield* fs.makeDirectory(`${worktree}/module`); - fixture.editBeforeRemoval(); - expect( - (yield* service.runNow).entries.find((entry) => entry.path === worktree), - ).toMatchObject({ outcome: "failed" }); - expect(yield* fs.readFileString(`${worktree}/tracked.txt`)).toBe("late edit\n"); - }), - ), +it.live.each([ + ["uninitialized", "git removal", "failed"], + ["initialized", "index shutdown", "kept"], +] as const)( + "preserves late tracked edits with %s submodules during %s", + ([initialization, timing, outcome]) => + runCleanupTest( + Effect.gen(function* () { + const fixture = yield* initialization === "initialized" + ? submoduleCleanupFixture() + : cleanupFixture(); + const { service, fs, worktree, command } = fixture; + if (initialization === "uninitialized") { + const head = (yield* command(worktree, ["rev-parse", "HEAD"])).stdout.trim(); + yield* fs.writeFileString( + `${worktree}/.gitmodules`, + '[submodule "module"]\n\tpath = module\n\turl = /unused\n', + ); + yield* command(worktree, ["add", ".gitmodules"]); + yield* command(worktree, [ + "update-index", + "--add", + "--cacheinfo", + `160000,${head},module`, + ]); + yield* command(worktree, ["commit", "-m", "record uninitialized module"]); + yield* fs.makeDirectory(`${worktree}/module`); + } + if (timing === "index shutdown") { + fixture.setOnIndexRemoval( + fs.writeFileString(`${worktree}/tracked.txt`, "late edit\n").pipe(Effect.orDie), + ); + } else fixture.editBeforeRemoval(); + expect( + (yield* service.runNow).entries.find((entry) => entry.path === worktree), + ).toMatchObject({ outcome }); + expect(yield* fs.readFileString(`${worktree}/tracked.txt`)).toBe("late edit\n"); + }), + ), ); describe("subagent worktree ownership", () => { @@ -1272,6 +1306,35 @@ describe("deleted thread worktrees", () => { ), ); + it.live.each([ + ["reflog", "Worktree history exceeds the inspection limit"], + ["worktree", "Worktree list exceeds the inspection limit"], + ] as const)("keeps a worktree when its %s inspection is truncated", ([command, reason]) => + runCleanupTest( + Effect.gen(function* () { + const git = yield* GitVcsDriver.GitVcsDriver; + const { service, fs, worktree } = yield* cleanupFixture().pipe( + Effect.provideService(GitVcsDriver.GitVcsDriver, { + ...git, + execute: (input) => + git + .execute(input) + .pipe( + Effect.map((result) => + input.operation === "StorageCleanup.checkoutBlocker" && + input.args[0] === command + ? { ...result, stdoutTruncated: true } + : result, + ), + ), + }), + ); + expect((yield* service.runNow).entries[0]).toMatchObject({ outcome: "kept", reason }); + expect(yield* fs.exists(worktree)).toBe(true); + }), + ), + ); + it.live("keeps a deleted owner's worktree while only its reflog holds a commit", () => runCleanupTest( Effect.gen(function* () { @@ -1363,6 +1426,44 @@ describe("deleted thread worktrees", () => { ), ); + it.live("does not keep a detached worktree for a same-named branch in another project", () => + runCleanupTest( + Effect.gen(function* () { + const { service, fs, repo, worktree, command, config, setThreads, setProjectRoots } = + yield* cleanupFixture(); + const other = yield* fs.realPath( + yield* fs.makeTempDirectoryScoped({ prefix: "cleanup-unrelated-" }), + ); + yield* command(other, ["init", "-b", "main"]); + yield* command(other, ["config", "user.email", "other@example.test"]); + yield* command(other, ["config", "user.name", "Other"]); + yield* command(other, ["commit", "--allow-empty", "-m", "other repo"]); + const otherWorktree = `${config.worktreesDir}/unrelated-feature`; + yield* command(other, ["worktree", "add", "-b", "feature", otherWorktree]); + setProjectRoots([repo, other]); + yield* command(worktree, ["switch", "--detach"]); + setThreads([ + shell({ branch: "feature", worktreePath: worktree }), + shell({ + id: ThreadId.make("thread-2"), + projectId: ProjectId.make("project-2"), + branch: "feature", + worktreePath: otherWorktree, + lineage: { + rootThreadId: ThreadId.make("thread-2"), + parentThreadId: null, + relationshipToParent: null, + }, + }), + ]); + const report = yield* service.runNow; + expect(report.entries.find((entry) => entry.threadId === "thread-1")).toMatchObject({ + outcome: "removed", + }); + }), + ), + ); + it.live("rebinds a thread that stores its worktree path with a trailing slash", () => runCleanupTest( Effect.gen(function* () { @@ -1850,8 +1951,8 @@ describe("orphan worktree reports", () => { for (let index = 0; index < 205; index++) { yield* fixture.fs.makeDirectory(`${fixture.config.worktreesDir}/busy-${index}`); } - const orphans = ["one", "two"].map( - (name) => `${fixture.config.worktreesDir}/repository/${name}`, + const orphans = ["one", "two"].map((name) => + fixture.path.join(fixture.config.worktreesDir, "repository", name), ); for (const orphan of orphans) { yield* fixture.fs.makeDirectory(orphan, { recursive: true }); @@ -1907,7 +2008,7 @@ describe("orphan worktree reports", () => { const loop = `${fixture.repo}/loop`; yield* fixture.fs.symlink(loop, loop); fixture.setProjectRoots([fixture.repo, loop]); - const orphan = `${fixture.config.worktreesDir}/repository/orphan`; + const orphan = fixture.path.join(fixture.config.worktreesDir, "repository", "orphan"); yield* fixture.fs.makeDirectory(orphan, { recursive: true }); yield* fixture.fs.utimes(orphan, 1, 1); expect((yield* fixture.service.runNow).entries).toEqual([ @@ -1924,7 +2025,11 @@ describe("orphan worktree reports", () => { Effect.gen(function* () { const fixture = yield* cleanupFixture(); fixture.setThreads([]); - const orphan = `${fixture.config.worktreesDir}/${location === "nested" ? "repository/" : ""}orphan`; + const orphan = fixture.path.join( + fixture.config.worktreesDir, + location === "nested" ? "repository" : "", + "orphan", + ); yield* fixture.command(fixture.repo, ["worktree", "add", "-b", "orphan", orphan]); const sizes = yield* Effect.forEach(yield* fixture.fs.readDirectory(orphan), (name) => allocatedBytes(fixture.fs, `${orphan}/${name}`), @@ -2044,13 +2149,13 @@ describe("orphan worktree reports", () => { fixture.setThreads([]); const outside = yield* fixture.fs.makeTempDirectoryScoped(); yield* fixture.fs.makeDirectory(`${outside}/checkout`); - const projectDirectory = `${fixture.config.worktreesDir}/repository`; + const projectDirectory = fixture.path.join(fixture.config.worktreesDir, "repository"); yield* fixture.fs.makeDirectory(projectDirectory); yield* fixture.fs.symlink(outside, `${fixture.config.worktreesDir}/linked-project`); yield* fixture.fs.symlink(`${outside}/checkout`, `${projectDirectory}/linked-checkout`); yield* fixture.fs.symlink(`${outside}/missing`, `${projectDirectory}/dangling-link`); expect((yield* fixture.service.runNow).entries).toEqual([]); - const orphan = `${projectDirectory}/orphan`; + const orphan = fixture.path.join(projectDirectory, "orphan"); yield* fixture.fs.makeDirectory(orphan); fixture.disable(); expect((yield* fixture.service.runNow).entries).toEqual([]); @@ -2064,7 +2169,7 @@ describe("orphan worktree reports", () => { Effect.gen(function* () { const fixture = yield* cleanupFixture(); fixture.setThreads([]); - const folder = `${fixture.config.worktreesDir}/repository/leftover`; + const folder = fixture.path.join(fixture.config.worktreesDir, "repository", "leftover"); yield* fixture.fs.makeDirectory(folder, { recursive: true }); yield* fixture.fs.writeFileString(`${folder}/payload`, "leftover data"); yield* fixture.fs.utimes(folder, 1, 1); @@ -2091,7 +2196,7 @@ describe("orphan worktree reports", () => { const fixture = yield* cleanupFixture(); fixture.setThreads([]); const blocked = `${fixture.config.worktreesDir}/a-inaccessible`; - const orphan = `${fixture.config.worktreesDir}/z-repository/orphan`; + const orphan = fixture.path.join(fixture.config.worktreesDir, "z-repository", "orphan"); yield* fixture.fs.makeDirectory(blocked); yield* fixture.fs.makeDirectory(orphan, { recursive: true }); yield* fixture.fs.writeFileString(`${orphan}/file`, "x"); @@ -2117,14 +2222,15 @@ describe("orphan worktree reports", () => { (platform) => runCleanupTest( Effect.gen(function* () { + const hostIsWindows = yield* HostProcess.isWindows; const fixture = yield* cleanupFixture().pipe( Effect.provideService(HostProcess.Platform, platform), ); fixture.setThreads([]); - const parent = `${fixture.config.worktreesDir}/repository`; - const referenced = `${parent}/Referenced`; - const project = `${parent}/Project`; - const nested = `${parent}/Nested`; + const parent = fixture.path.join(fixture.config.worktreesDir, "repository"); + const referenced = fixture.path.join(parent, "Referenced"); + const project = fixture.path.join(parent, "Project"); + const nested = fixture.path.join(parent, "Nested"); for (const folder of [referenced, project, nested]) { yield* fixture.fs.makeDirectory(`${folder}/child`, { recursive: true }); yield* fixture.fs.utimes(folder, 1, 1); @@ -2133,7 +2239,8 @@ describe("orphan worktree reports", () => { fixture.setProjectRoots([fixture.repo, project, `${nested}/child`]); const report = yield* fixture.service.runNow; expect(report.entries.map((entry) => entry.path).sort()).toEqual( - platform === "win32" ? [] : [referenced], + // The real Windows path layer folds case whatever platform the service is told it runs on. + platform === "win32" || hostIsWindows ? [] : [referenced], ); }), ), @@ -2450,6 +2557,10 @@ describe("storage cleanup reports and local file policies", () => { }), ); const root = yield* fs.realPath(fixture.worktree); + const sizes = yield* Effect.forEach(yield* fs.readDirectory(root), (name) => + allocatedBytes(fs, fixture.path.join(root, name)), + ); + const expectedBytes = sizes.reduce((sum, size) => sum + size, 0); fixture.setOpenSessionCwd(fixture.worktree); const kept = yield* fixture.service.runNow; expect(kept.entries[0]).toMatchObject({ @@ -2460,8 +2571,8 @@ describe("storage cleanup reports and local file policies", () => { expect(walked.filter((directory) => directory.startsWith(root))).toEqual([]); fixture.setSessionLive(false); const removed = yield* fixture.service.runNow; - expect(removed.entries[0]).toMatchObject({ outcome: "removed" }); - expect(removed.entries[0]?.bytes).toBeGreaterThan(0); + expect(removed.entries[0]).toMatchObject({ outcome: "removed", bytes: expectedBytes }); + expect(removed.bytesFreed).toBe(expectedBytes); expect(walked).toContain(root); }), ), @@ -2504,10 +2615,14 @@ describe("storage cleanup reports and local file policies", () => { expect(locked?.reason).toContain("Git index is locked"); expect(yield* fs.readFileString(`${worktree}/tracked.txt`)).toBe("original\n"); yield* fs.remove(`${index.trim()}.lock`); - const gitFile = yield* fs.readFileString(`${worktree}/.git`); - yield* fs.writeFileString(`${worktree}/.git`, "gitdir: /missing-git-directory\n"); + const gitFile = `${worktree}/.git`; + const savedGitFile = `${worktree}/.git-original`; + // Git marks this file hidden on Windows, where overwriting it fails with EPERM. + yield* fs.rename(gitFile, savedGitFile); + yield* fs.writeFileString(gitFile, "gitdir: /missing-git-directory\n"); expect((yield* service.runNow).entries[0]).toMatchObject(changed); - yield* fs.writeFileString(`${worktree}/.git`, gitFile); + yield* fs.remove(gitFile); + yield* fs.rename(savedGitFile, gitFile); expect((yield* service.runNow).entries[0]).toMatchObject({ outcome: "removed" }); expect(fullStatusCalls()).toBe(0); }), @@ -2572,6 +2687,19 @@ describe("storage cleanup reports and local file policies", () => { }), ), ); + it.live("removes a worktree's code index with the worktree, not while keeping it", () => + runCleanupTest( + Effect.gen(function* () { + const { service, fs, worktree, indexRemovals } = yield* cleanupFixture(); + yield* fs.writeFileString(`${worktree}/tracked.txt`, "edited\n"); + expect((yield* service.runNow).entries[0]?.outcome).toBe("kept"); + expect(indexRemovals()).toEqual([]); + yield* fs.writeFileString(`${worktree}/tracked.txt`, "original\n"); + expect((yield* service.runNow).entries[0]?.outcome).toBe("removed"); + expect(indexRemovals()).toEqual([worktree]); + }), + ), + ); it.live("follows a branch the agent switched to and removes the worktree next pass", () => runCleanupTest( Effect.gen(function* () { @@ -2666,38 +2794,44 @@ describe("storage cleanup reports and local file policies", () => { }), ), ); - it.live("keeps a detached worktree when its holding ref goes during the final checks", () => - runCleanupTest( - Effect.gen(function* () { - const git = yield* GitVcsDriver.GitVcsDriver; - let localChangeChecks = 0; - let repo = ""; - const { service, command, worktree, ...fixture } = yield* cleanupFixture().pipe( - Effect.provideService(GitVcsDriver.GitVcsDriver, { - ...git, - execute: (input) => - Effect.gen(function* () { - if (input.operation === "StorageCleanup.localChanges" && ++localChangeChecks === 2) - yield* git.execute({ - operation: "test", - cwd: repo, - args: ["tag", "--delete", "keeper"], - }); - return yield* git.execute(input); - }), - }), - ); - repo = fixture.repo; - yield* command(worktree, ["switch", "--detach"]); - yield* command(worktree, ["commit", "--allow-empty", "-m", "detached work"]); - yield* command(worktree, ["tag", "keeper"]); - expect((yield* service.runNow).entries[0]).toMatchObject({ - outcome: "kept", - reason: "Worktree history has commits no branch or tag holds", - }); - expect(yield* fixture.fs.exists(worktree)).toBe(true); - }), - ), + it.live.each(["final status checks", "code index cleanup"] as const)( + "keeps a detached worktree when its holding ref goes during %s", + (phase) => + runCleanupTest( + Effect.gen(function* () { + const git = yield* GitVcsDriver.GitVcsDriver; + let localChangeChecks = 0; + let repo = ""; + const removeHoldingRef = Effect.suspend(() => + git.execute({ operation: "test", cwd: repo, args: ["tag", "--delete", "keeper"] }), + ).pipe(Effect.orDie, Effect.asVoid); + const { service, command, worktree, ...fixture } = yield* cleanupFixture().pipe( + Effect.provideService(GitVcsDriver.GitVcsDriver, { + ...git, + execute: (input) => + Effect.gen(function* () { + if ( + phase === "final status checks" && + input.operation === "StorageCleanup.localChanges" && + ++localChangeChecks === 2 + ) + yield* removeHoldingRef; + return yield* git.execute(input); + }), + }), + ); + repo = fixture.repo; + if (phase === "code index cleanup") fixture.setOnIndexRemoval(removeHoldingRef); + yield* command(worktree, ["switch", "--detach"]); + yield* command(worktree, ["commit", "--allow-empty", "-m", "detached work"]); + yield* command(worktree, ["tag", "keeper"]); + expect((yield* service.runNow).entries[0]).toMatchObject({ + outcome: "kept", + reason: "Worktree history has commits no branch or tag holds", + }); + expect(yield* fixture.fs.exists(worktree)).toBe(true); + }), + ), ); it.live.each([ ["any-local-files", ".env", "kept"], @@ -2871,7 +3005,11 @@ describe("storage cleanup reports and local file policies", () => { it.live("caps entries while preserving totals and prioritizing removals", () => runCleanupTest( Effect.gen(function* () { - const { service, fs, worktree, setThreads } = yield* cleanupFixture(); + const { service, fs, path, worktree, setThreads } = yield* cleanupFixture(); + const sizes = yield* Effect.forEach(yield* fs.readDirectory(worktree), (name) => + allocatedBytes(fs, path.join(worktree, name)), + ); + const expectedBytes = sizes.reduce((sum, size) => sum + size, 0); const threads = []; for (let i = 0; i < 205; i++) { const worktreePath = `${worktree}-${i}`; @@ -2891,9 +3029,8 @@ describe("storage cleanup reports and local file policies", () => { expect(report.entries).toHaveLength(200); expect(report.omittedCount).toBe(6); expect(report.counts).toEqual({ removed: 1, kept: 205, failed: 0 }); - expect(report.entries[0]?.outcome).toBe("removed"); - expect(report.entries[0]?.bytes).toBeGreaterThan(0); - expect(report.bytesFreed).toBe(report.entries[0]?.bytes); + expect(report.entries[0]).toMatchObject({ outcome: "removed", bytes: expectedBytes }); + expect(report.bytesFreed).toBe(expectedBytes); }), ), ); diff --git a/apps/server/src/storageCleanup.ts b/apps/server/src/storageCleanup.ts index 07ddde641a4cc..37f84c1a98edb 100644 --- a/apps/server/src/storageCleanup.ts +++ b/apps/server/src/storageCleanup.ts @@ -43,6 +43,7 @@ import * as ProjectionStore from "./orchestration-v2/ProjectionStore.ts"; import * as ProviderSessionManager from "./orchestration-v2/ProviderSessionManager.ts"; import { threadHasQueuedTurnStart } from "./orchestration-v2/ThreadSettlementService.ts"; import * as WorktreeBranches from "./orchestration-v2/WorktreeBranches.ts"; +import * as CodeIntelligence from "./project/knowledge/CodeIntelligence.ts"; import { forkParked } from "./serverActivation.ts"; import * as Settings from "./serverSettings.ts"; import * as TerminalManager from "./terminal/Manager.ts"; @@ -203,7 +204,7 @@ export function storageCleanupActivityAt(thread: OrchestrationV2ThreadShell): nu } /** Only a complete subagent or side chat chain within this worktree shares ownership. */ -function worktreeOwner>( +export function worktreeOwner>( threads: ReadonlyArray, ) { if (threads.length === 1) return threads[0]; @@ -231,7 +232,7 @@ function worktreeOwner, now: number, @@ -303,6 +304,7 @@ const make = Effect.gen(function* () { const sql = yield* SqlClient.SqlClient; const git = yield* GitVcsDriver.GitVcsDriver; const worktreeBranches = yield* WorktreeBranches.WorktreeBranches; + const codeIntelligence = yield* CodeIntelligence.CodeIntelligence; const gitManager = yield* GitManager.GitManager; const terminals = yield* TerminalManager.TerminalManager; const fs = yield* FileSystem.FileSystem; @@ -623,6 +625,8 @@ const make = Effect.gen(function* () { args: ["reflog", "show", "--format=%H", "HEAD"], maxOutputBytes: 16 * 1024 * 1024, }); + if (reflog.stdoutTruncated || reflog.stderrTruncated) + return "worktree history exceeds the inspection limit"; // Only refs that stay count. A user cannot recover from refs/t3, and a routine // `git fetch --prune` can drop any remote branch, even one a stale local HEAD calls the default. const unheld = yield* git.execute({ @@ -637,6 +641,8 @@ const make = Effect.gen(function* () { cwd, args: ["worktree", "list", "--porcelain"], }); + if (worktrees.stdoutTruncated || worktrees.stderrTruncated) + return "worktree list exceeds the inspection limit"; const held = new Set( [...worktrees.stdout.matchAll(/^branch refs\/heads\/(.+?)\r?$/gm)].map((match) => match[1]), ); @@ -935,6 +941,7 @@ const make = Effect.gen(function* () { checkout._tag !== "Bound" && snapshot.threads.some( (other) => + other.projectId === thread.projectId && other.worktreePath !== null && path.resolve(other.worktreePath) !== worktreePath && group.some((member) => member.branch !== null && member.branch === other.branch), @@ -1159,8 +1166,10 @@ const make = Effect.gen(function* () { const blocked = yield* blockedSinceCheck(); if (blocked !== null) return keep(blocked); const bytes = yield* measureWorktree(worktreePath); - const blockedWhileMeasuring = yield* blockedSinceCheck(); - if (blockedWhileMeasuring !== null) return keep(blockedWhileMeasuring); + // Stop builds while their sources still exist, then recheck activity and checkout safety. + yield* codeIntelligence.removeCheckoutIndex(worktreePath); + const blockedBeforeRemoval = yield* blockedSinceCheck(); + if (blockedBeforeRemoval !== null) return keep(blockedBeforeRemoval); const observed = yield* worktreeBranches.observe(worktreePath); if ( checkout._tag === "Detached" @@ -1195,7 +1204,7 @@ const make = Effect.gen(function* () { const changes = yield* localChangesReason(worktreePath, settings); if (changes !== null) return keep(changes); } - // Last, so nothing that runs after this can drop the ref that holds the commit. + // Recheck recovery refs immediately before removal. const finalBlocker = yield* checkCheckout(); if (finalBlocker !== null) return keep(finalBlocker); yield* removeLinkedWorktree({ diff --git a/apps/web/src/components/codeExplorer/CodeExplorer.browser.test.tsx b/apps/web/src/components/codeExplorer/CodeExplorer.browser.test.tsx index f6835651d9ef6..56d184ab49ebe 100644 --- a/apps/web/src/components/codeExplorer/CodeExplorer.browser.test.tsx +++ b/apps/web/src/components/codeExplorer/CodeExplorer.browser.test.tsx @@ -406,19 +406,19 @@ it("warns when the index is stale", async () => { await expect.element(screen.getByText("Stale")).toBeVisible(); }); -it("says the index is unavailable instead of drawing an empty graph", async () => { +it("says the index is not ready instead of drawing an empty graph", async () => { const registry = setup({ empty: true, coverage: "unavailable", index: "unindexed" }); await using screen = await renderHost(registry); await expect .element( screen.getByText( - "The code index is unavailable for this checkout, so no calls can be shown.", + "The code index is not ready for this checkout, so no calls can be shown yet.", ), ) .toBeVisible(); await expect.element(screen.getByText(/No static callers or callees found/)).toBeVisible(); expect(graph(screen, "greet").query()).toBeNull(); - await expect.element(screen.getByText("Unavailable")).toBeVisible(); + await expect.element(screen.getByText("Not indexed")).toBeVisible(); }); it("keeps the legacy explorer usable without sending unsupported path or freshness queries", async () => { diff --git a/apps/web/src/components/codeExplorer/CodeExplorer.tsx b/apps/web/src/components/codeExplorer/CodeExplorer.tsx index 8222f6d9eefc9..fca18562f7d54 100644 --- a/apps/web/src/components/codeExplorer/CodeExplorer.tsx +++ b/apps/web/src/components/codeExplorer/CodeExplorer.tsx @@ -510,7 +510,7 @@ export default function CodeExplorer(

{coverage.state === "stale" ? "The index is stale for some of these files. Verify calls in source before relying on them." - : "The code index is unavailable for this checkout, so no calls can be shown."} + : "The code index is not ready for this checkout, so no calls can be shown yet."}

) : null} {farPending || farError !== null ? ( diff --git a/apps/web/src/components/files/CodeIndexBadge.tsx b/apps/web/src/components/files/CodeIndexBadge.tsx index 59e814220b57f..11e6783ccf2f0 100644 --- a/apps/web/src/components/files/CodeIndexBadge.tsx +++ b/apps/web/src/components/files/CodeIndexBadge.tsx @@ -10,8 +10,8 @@ import { serverEnvironment } from "~/state/server"; const VARIANT = { ok: "success", warn: "warning", muted: "secondary" } as const; /** - * Plain-text freshness of a checkout's code index. It reads the index without building one, - * and shows nothing until the server answers rather than guessing a state. + * Plain-text freshness of a checkout's code index. It shows nothing until the server answers + * rather than guessing a state. */ export function CodeIndexBadge(props: { readonly environmentId: EnvironmentId; diff --git a/apps/web/src/components/files/FileCodeRelationships.tsx b/apps/web/src/components/files/FileCodeRelationships.tsx index 4d12c72b55ef3..015429a71dde3 100644 --- a/apps/web/src/components/files/FileCodeRelationships.tsx +++ b/apps/web/src/components/files/FileCodeRelationships.tsx @@ -308,7 +308,7 @@ function SymbolsTab(props: { Refresh - {data.nodes.length === 0 ? ( + {data.coverage.state === "unavailable" ? null : data.nodes.length === 0 ? ( No indexed declarations in this file. Its language may be unsupported, or it declares nothing the index recognizes. diff --git a/apps/web/src/components/files/codeRelationshipRows.tsx b/apps/web/src/components/files/codeRelationshipRows.tsx index ff276bfd038d7..ab1a8b5689edf 100644 --- a/apps/web/src/components/files/codeRelationshipRows.tsx +++ b/apps/web/src/components/files/codeRelationshipRows.tsx @@ -28,7 +28,7 @@ function coverageSummary(coverage: ProjectCodeCoverage): string { case "stale": return "Stale · some returned files changed since indexing. Verify in source."; case "unavailable": - return "The code index is unavailable for this checkout."; + return "The code index is not ready for this checkout."; } } diff --git a/apps/web/src/components/settings/LabsSettingsPanel.tsx b/apps/web/src/components/settings/LabsSettingsPanel.tsx index 2fdff310dd966..c9756cdd76475 100644 --- a/apps/web/src/components/settings/LabsSettingsPanel.tsx +++ b/apps/web/src/components/settings/LabsSettingsPanel.tsx @@ -26,6 +26,7 @@ export function LabsSettingsPanel() { {/* Clerk is only mounted when this build has T3 Connect. */} {hasCloudPublicConfig() ? : } + @@ -166,6 +167,27 @@ function LinkedEnvironmentsRow() { ); } +function AutoCodeIndexRow() { + const settings = useScopedSettings(); + const updateSettings = useUpdateScopedSettings(); + return ( + updateSettings({ autoCodeIndexEnabled: Boolean(checked) })} + aria-label="Automatic code indexing" + /> + } + /> + ); +} + function SimlockHostsRow() { const settings = useScopedSettings(); const updateSettings = useUpdateScopedSettings(); diff --git a/apps/web/src/components/settings/settingsSearch.ts b/apps/web/src/components/settings/settingsSearch.ts index 9c0f5bd012a83..090ae15fd1359 100644 --- a/apps/web/src/components/settings/settingsSearch.ts +++ b/apps/web/src/components/settings/settingsSearch.ts @@ -509,6 +509,13 @@ export const SETTINGS_SEARCH_ITEMS = [ scope: "environment", searchTerms: ["beta link another machine vps peer agents work there delegate"], }, + { + id: "auto-code-index", + title: "Automatic code indexing", + to: "/settings/labs", + scope: "environment", + searchTerms: ["beta code graph index worktree automatic settle cleanup background"], + }, { id: "simlock-hosts", title: "Simlock device hosts (alpha)", diff --git a/docs/user/code-navigation.md b/docs/user/code-navigation.md index 828f979a79149..be33b5e3bf16b 100644 --- a/docs/user/code-navigation.md +++ b/docs/user/code-navigation.md @@ -1,11 +1,25 @@ # Code navigation Open a file from a thread's file browser and expand **Code relationships** below the preview to -see how it connects to the rest of the project. Nothing is analyzed until you expand it. The -first expansion on a checkout starts indexing it in the background, and until that finishes the -section says the checkout is not indexed yet. Results come from the thread's own checkout, its +see how it connects to the rest of the project. Results come from the thread's own checkout, its worktree when it has one. +A project in a git repository starts indexing in the background when you add it, or the first time +you open or run a thread in it. A worktree has its own index, built and kept current while an agent +works in it, or the first time something asks it a code question, such as expanding this section or +an agent using the code graph. An idle worktree is not refreshed until an agent is active in it +again. A worktree's index is removed when its thread settles, unless other unsettled threads share +the worktree. It is also removed when Maintenance removes the worktree, and after 7 days with no +activity on the worktree's threads, unless one of them is pinned or marked active. The next agent +run rebuilds it. Until an index is ready, the section says the checkout is not indexed yet. Folders +outside a git repository index only on a code question. + +Indexes are stored in T3 Code's data directory, not in your repository. Folders an earlier version +left in a project or worktree are removed automatically when T3 Code starts. + +Turn automatic indexing off in **Settings → Labs → Automatic code indexing**. Then a checkout only +indexes when something asks it a code question. + - **Symbols** lists the file's declarations. Select one to see its callers, callees, references, impact (files that depend on it), and **Tests**: test files that reach it within a few calls. - **Decisions** shows project knowledge whose entities name this file's path or one of its @@ -22,12 +36,16 @@ values or dynamic dispatch and can mistake one same-named symbol for another. Ea that nothing calls the code or that it is untested. The badge next to **Code relationships**, and **Code index** in a thread's details, shows whether -the index matches the checkout: **Ready**, **Partial** (some files are not indexed), **Stale** -(files changed since indexing; **Stale @abc1234** when the checkout has moved past the commit it -was indexed at), **Indexing** (building the first index), or **Unavailable** (not indexed yet). -Checking the badge never starts indexing. Code queries and finished turns refresh a stale index -in the background, and results keep coming from the previous index until the new one is ready. -If a refresh fails, the badge says so and the previous index keeps answering. +the index matches the checkout: **Ready**, **Partial** (source files are missing from the index), +**Stale** (files changed since indexing; **Stale @abc1234** when the checkout has moved past the +commit it was indexed at), **Indexing** (building the first index), **Not indexed**, or **Index +failed**. Files with no code to analyze, such as docs and images, don't count against **Ready**. +Code queries refresh a stale index in the background, and so do agent turns while automatic +indexing is on. Results keep coming from the previous index until the new one is ready. If a +refresh fails, the badge says so and the previous index keeps answering. + +An index covers at most 10,000 files. A larger repository shows **Partial**, with your own source +indexed ahead of files in hidden folders. On mobile, the file screen has a compact **Code relationships** section with symbols, callers, and impact. diff --git a/packages/client-runtime/src/projectCode/index.test.ts b/packages/client-runtime/src/projectCode/index.test.ts index 983df7c4749ad..b53321d200d89 100644 --- a/packages/client-runtime/src/projectCode/index.test.ts +++ b/packages/client-runtime/src/projectCode/index.test.ts @@ -261,10 +261,18 @@ describe("project code atoms", () => { expect(projectCodeIndexBadge({ ...index, headCommit: "1234567890abcdef" }).label).toBe( "Stale @abcdef0", ); - expect(projectCodeIndexBadge({ ...index, state: "unindexed" }).label).toBe("Unavailable"); + expect(projectCodeIndexBadge({ ...index, state: "unindexed" }).label).toBe("Not indexed"); expect(projectCodeIndexBadge({ ...index, state: "unindexed", refresh: "building" }).label).toBe( "Indexing", ); + // No earlier generation exists to answer, so a failed first build must not claim one does. + const firstIndexFailed = projectCodeIndexBadge({ + ...index, + state: "unindexed", + refresh: "failed", + }); + expect(firstIndexFailed).toMatchObject({ label: "Index failed", tone: "warn" }); + expect(firstIndexFailed.detail).not.toContain("still answering"); // A failed refresh leaves the older generation answering, and the badge says so. expect(projectCodeIndexBadge({ ...index, refresh: "failed" })).toMatchObject({ label: "Stale", @@ -273,4 +281,72 @@ describe("project code atoms", () => { // A server from before index tracking cannot claim the index is ready. expect(projectCodeIndexBadge(undefined).label).toBe("Unavailable"); }); + + it.effect("re-reads a query only for a real publication or a failure", () => + Effect.scoped( + Effect.gen(function* () { + const { atoms, registry, requests, events, subscribed } = yield* setup(); + registry.set( + sessions(target.environmentId), + AsyncResult.success(session([AuthFilesystemReadScope])), + ); + const checkout = { projectId: input.projectId, threadId: ThreadId.make("mounted") }; + const query = atoms.query({ + environmentId: target.environmentId, + input: { ...input, ...checkout }, + }); + const freshness = atoms.freshness({ + environmentId: target.environmentId, + input: { ...checkout, query: { operation: "freshness" } }, + }); + const queryRuns = () => + requests.filter( + (request) => + request.threadId === checkout.threadId && request.query.operation === "symbols", + ).length; + const freshnessRuns = () => + requests.filter((request) => request.query.operation === "freshness").length; + let onQuery = () => {}; + let onFreshness = () => {}; + const stopQuery = registry.subscribe(query, () => onQuery()); + const stopFreshness = registry.subscribe(freshness, () => onFreshness()); + yield* Effect.addFinalizer(() => + Effect.sync(() => { + stopQuery(); + stopFreshness(); + }), + ); + yield* AtomRegistry.getResult(registry, query, { suspendOnWaiting: true }); + yield* AtomRegistry.getResult(registry, freshness, { suspendOnWaiting: true }); + yield* Deferred.await(subscribed); + expect([queryRuns(), freshnessRuns()]).toEqual([1, 1]); + + // One event at a time: a stream atom keeps only the last value of a chunk. + const publish = Effect.fn("TestProjectCode.publish")(function* ( + event: ProjectCodeIndexEvent, + expected: readonly [query: number, freshness: number], + ) { + const reached = yield* Deferred.make(); + onQuery = onFreshness = () => { + if (queryRuns() >= expected[0] && freshnessRuns() >= expected[1]) { + Deferred.doneUnsafe(reached, Exit.void); + } + }; + yield* PubSub.publish(events, { target: checkout, event }); + yield* Deferred.await(reached); + expect([queryRuns(), freshnessRuns()]).toEqual(expected); + }); + // Freshness re-reads on every event. The query only re-runs for a real publication, + // because a query starts the build a settle removal just stopped. + const reRead: ProjectCodeIndexEvent = { outcome: "published", generation: null }; + yield* publish(reRead, [1, 2]); + yield* publish({ outcome: "published", generation: 1 }, [2, 3]); + // A settle removes the index, the next turn announces its rebuild with the same signal, + // and generation numbers restart once the files are gone. + yield* publish(reRead, [2, 4]); + yield* publish(reRead, [2, 5]); + yield* publish({ outcome: "published", generation: 1 }, [3, 6]); + }), + ), + ); }); diff --git a/packages/client-runtime/src/projectCode/index.ts b/packages/client-runtime/src/projectCode/index.ts index 63bff84382e6c..3bc2629427936 100644 --- a/packages/client-runtime/src/projectCode/index.ts +++ b/packages/client-runtime/src/projectCode/index.ts @@ -8,7 +8,7 @@ import { } from "@t3tools/contracts"; import * as Effect from "effect/Effect"; import * as Stream from "effect/Stream"; -import type { Atom } from "effect/reactivity"; +import { AsyncResult, Atom, type AtomRegistry } from "effect/reactivity"; import type { EnvironmentRegistry } from "../connection/registry.ts"; import * as EnvironmentSupervisor from "../connection/supervisor.ts"; import { @@ -36,6 +36,8 @@ const requireProjectCode = Effect.fn("ProjectCode.requireCapability")(function* export function createProjectCodeAtoms( runtime: Atom.AtomRuntime, ) { + // An atom drops a value equal to its last one, and the same signal can arrive twice running. + let received = 0; const indexEvents = createEnvironmentSubscriptionAtomFamily(runtime, { label: "project-code:index-events", idleTtlMs: 0, @@ -48,7 +50,7 @@ export function createProjectCodeAtoms( : Stream.never, ), ), - ), + ).pipe(Stream.map((event) => ({ ...event, received: ++received }))), }); const refreshTrigger = ({ environmentId, @@ -64,12 +66,29 @@ export function createProjectCodeAtoms( ...(input.threadId === undefined ? {} : { threadId: input.threadId }), }, }); + // Queries start builds, so only a real publication or a failure re-runs one; the re-read signal + // a settle removal or a stopped build sends would otherwise start the build it just ended. + // Per registry, as `refreshOnSignalWhileRead` keeps its own bookkeeping. + const queryRefreshTrigger = (target: { + readonly environmentId: EnvironmentId; + readonly input: ProjectCodeQueryInput; + }) => { + const signal = refreshTrigger(target); + const lastSignificant = new WeakMap(); + return Atom.transform(signal, (get) => { + const event = AsyncResult.getOrElse(get(signal), () => null); + if (event !== null && (event.outcome !== "published" || event.generation !== null)) { + lastSignificant.set(get.registry, event.received); + } + return lastSignificant.get(get.registry) ?? 0; + }); + }; return { query: createEnvironmentGuardedRpcQueryAtomFamily(runtime, { label: "project-code:query", tag: WS_METHODS.projectCodeQuery, staleTimeMs: 0, - refreshTrigger, + refreshTrigger: queryRefreshTrigger, execute: Effect.fn("ProjectCode.query")(function* (input: ProjectCodeQueryInput) { yield* requireProjectCode(); return yield* requestGuarded(WS_METHODS.projectCodeQuery, input); @@ -110,6 +129,19 @@ const REFRESH_DETAIL = { failed: "The last refresh failed, so this index is still answering.", } as const; +const FIRST_INDEX_BADGE: Record<"building" | "failed", ProjectCodeIndexBadge> = { + building: { + label: "Indexing", + tone: "muted", + detail: "Building the first index for this checkout.", + }, + failed: { + label: "Index failed", + tone: "warn", + detail: "Building the first index for this checkout failed. The next code query tries again.", + }, +}; + /** * The plain-text badge every client shows for an index. Stale names the commit the index was * built at when HEAD has moved since, otherwise how many files changed under it. @@ -119,13 +151,7 @@ export function projectCodeIndexBadge( ): ProjectCodeIndexBadge { const badge = describeIndexState(index); if (index?.refresh === undefined) return badge; - if (index.state === "unindexed" && index.refresh === "building") { - return { - label: "Indexing", - tone: "muted", - detail: "Building the first index for this checkout.", - }; - } + if (index.state === "unindexed") return FIRST_INDEX_BADGE[index.refresh]; return { ...badge, detail: `${badge.detail} ${REFRESH_DETAIL[index.refresh]}` }; } @@ -161,7 +187,7 @@ function describeIndexState(index: ProjectCodeIndexInfo | undefined): ProjectCod return { label: moved ? `Stale @${indexedAt}` : "Stale", tone: "warn", - detail: `${files} The next code query or turn refreshes the index.`, + detail: `${files} Code queries refresh the index, and so do agent turns while automatic indexing is on.`, }; } case "indexing": @@ -172,7 +198,7 @@ function describeIndexState(index: ProjectCodeIndexInfo | undefined): ProjectCod }; case "unindexed": return { - label: "Unavailable", + label: "Not indexed", tone: "muted", detail: "This checkout has not been indexed yet. The first code query indexes it.", }; diff --git a/packages/contracts/src/settings.test.ts b/packages/contracts/src/settings.test.ts index 6c4f4265dcfd7..a31bbeef37778 100644 --- a/packages/contracts/src/settings.test.ts +++ b/packages/contracts/src/settings.test.ts @@ -57,6 +57,15 @@ describe("opt-in experimental settings", () => { ); }); +describe("automatic code indexing setting", () => { + it("defaults on and patches like any server setting", () => { + expect(decodeServerSettings({}).autoCodeIndexEnabled).toBe(true); + expect(decodeServerSettingsPatch({ autoCodeIndexEnabled: false })).toEqual({ + autoCodeIndexEnabled: false, + }); + }); +}); + describe("storage cleanup settings", () => { it("keeps cleanup disabled for existing installations", () => { expect(decodeServerSettings({}).worktreeCleanup).toBeNull(); diff --git a/packages/contracts/src/settings.ts b/packages/contracts/src/settings.ts index 527b4cb25f7c5..041ab327bec2a 100644 --- a/packages/contracts/src/settings.ts +++ b/packages/contracts/src/settings.ts @@ -1410,6 +1410,11 @@ export const ServerSettings = Schema.Struct({ * and tasks on another machine; tasks already running there finish. */ linkedEnvironmentsEnabled: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(false))), + /** + * Index a project when it is added and a worktree while an agent works in it. Off, a checkout + * only indexes when something asks it a code question. + */ + autoCodeIndexEnabled: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(true))), sidebarAutoSettleKeepPullRequests: Schema.Boolean.pipe( Schema.withDecodingDefault(Effect.succeed(false)), ), @@ -1825,6 +1830,7 @@ export const ServerSettingsPatch = Schema.Struct({ idleCompaction: Schema.optionalKey(IdleCompactionMode), sidebarAutoSettleKeepPullRequests: Schema.optionalKey(Schema.Boolean), sidebarAutoSettleOnMerge: Schema.optionalKey(Schema.Boolean), + autoCodeIndexEnabled: Schema.optionalKey(Schema.Boolean), autoResumeLimitedThreads: Schema.optionalKey(Schema.Boolean), snoozeLimitedThreads: Schema.optionalKey(Schema.Boolean), snoozePresets: Schema.optionalKey(SnoozePresetSettings), diff --git a/packages/shared/src/testing/providerEnv.ts b/packages/shared/src/testing/providerEnv.ts index ac07867c20473..50db357a45f9e 100644 --- a/packages/shared/src/testing/providerEnv.ts +++ b/packages/shared/src/testing/providerEnv.ts @@ -10,6 +10,8 @@ const KEPT_ENV = new Set([ "T3CODE_TRANSFER_BUDGET_RESULT_PATH", // Opt-in: points the HtmlRender tests at a headless browser, which they skip without. "T3CODE_TEST_HEADLESS_SHELL", + // The mobile test script sets this; tests must see the platform their code was compiled for. + "T3_MOBILE_TEST_PLATFORM", ]); if (process.env.T3CODE_TEST_KEEP_PROVIDER_ENV !== "1") { diff --git a/scripts/lib/code-intelligence-smoke.ts b/scripts/lib/code-intelligence-smoke.ts index 681055f7ad168..549872bfaaaea 100644 --- a/scripts/lib/code-intelligence-smoke.ts +++ b/scripts/lib/code-intelligence-smoke.ts @@ -68,17 +68,21 @@ export const smokeCodeIntelligenceProcess = Effect.fn("smokeCodeIntelligenceProc const callers = yield* Effect.acquireUseRelease( Effect.promise(async () => { const checkout = await NodeFSP.mkdtemp(NodePath.join(NodeOS.tmpdir(), "t3-code-smoke-")); + const indexRoot = await NodeFSP.mkdtemp( + NodePath.join(NodeOS.tmpdir(), "t3-code-smoke-index-"), + ); await NodeFSP.mkdir(NodePath.join(checkout, "src")); await NodeFSP.writeFile( NodePath.join(checkout, "src/a.ts"), "export function greet() { return 1; }\nexport function caller() { return greet(); }\n", ); - return checkout; + return { checkout, indexRoot }; }), - (checkout) => + ({ checkout, indexRoot }) => Effect.gen(function* () { const listing = { checkoutPath: checkout, + indexRoot, files: ["src/a.ts"], filesListed: 1, listingTruncated: false, @@ -99,7 +103,11 @@ export const smokeCodeIntelligenceProcess = Effect.fn("smokeCodeIntelligenceProc if (!("result" in answer)) return yield* fail("the query returned no result"); return answer.result.nodes.map((node) => node.name); }), - (checkout) => Effect.promise(() => NodeFSP.rm(checkout, { recursive: true, force: true })), + ({ checkout, indexRoot }) => + Effect.promise(async () => { + await NodeFSP.rm(checkout, { recursive: true, force: true }); + await NodeFSP.rm(indexRoot, { recursive: true, force: true }); + }), ); if (!callers.includes("caller")) { return yield* fail(`expected 'caller' among [${callers.join(", ")}]`);