From 7d1a1496efe8f4ee6fcd9d6eb3358549b27a9791 Mon Sep 17 00:00:00 2001 From: Harlan Wilton Date: Thu, 1 Oct 2026 03:17:52 +1000 Subject: [PATCH] feat(artifact-delivery): reuse a ready build of the same commit Over 14 days, 264 public builds covered only 89 distinct builds, and each build makes about 9 GitHub reads. On 2026-09-30 those reads hung and builds failed for commits that were already built and signed. A public build now reuses a ready build of the same source when its attestation still verifies under the current trusted root, its policy and check versions are current, and R2 still holds the bytes. A request pinned to a built commit makes no GitHub read. An unpinned request still resolves on GitHub, so it never gets an older commit. The signer signs a fresh statement through the normal path. --- .../server/utils/artifact-storage.ts | 37 +- .../artifact-delivery/server/utils/build.ts | 426 +++++++++++-- .../artifact-delivery/server/utils/state.ts | 131 ++++ migrations/0131_artifact_build_reuse.sql | 22 + nuxt.config.ts | 5 + test/unit/artifact-build-reuse.test.ts | 600 ++++++++++++++++++ 6 files changed, 1155 insertions(+), 66 deletions(-) create mode 100644 migrations/0131_artifact_build_reuse.sql create mode 100644 test/unit/artifact-build-reuse.test.ts diff --git a/layers/artifact-delivery/server/utils/artifact-storage.ts b/layers/artifact-delivery/server/utils/artifact-storage.ts index c417961b..d32dbbf6 100644 --- a/layers/artifact-delivery/server/utils/artifact-storage.ts +++ b/layers/artifact-delivery/server/utils/artifact-storage.ts @@ -27,16 +27,33 @@ export async function putImmutableArtifact( return { _tag: 'stored', key: input.key } const existing = await bucket.head(input.key) - if ( - existing - && existing.size === input.bytes.byteLength - && checksumHex(existing.checksums.sha256) === input.contentSha256 - && existing.customMetadata?.contentSha256 === input.contentSha256 - && existing.customMetadata?.format === 'skilld-tar-v1' - ) { - return { _tag: 'existing', key: input.key } - } - return { _tag: 'mutation-rejected', key: input.key } + return matchesImmutableArtifact(existing, { contentSha256: input.contentSha256, contentBytes: input.bytes.byteLength }) + ? { _tag: 'existing', key: input.key } + : { _tag: 'mutation-rejected', key: input.key } +} + +/** + * Whether R2 still holds the exact public Artifact bytes under this key. + * + * It reads only object metadata. The artifact signer still reads and hashes the + * whole object before it signs. + */ +export async function hasImmutableArtifact( + bucket: R2Bucket, + input: { key: string, contentSha256: string, contentBytes: number }, +): Promise { + return matchesImmutableArtifact(await bucket.head(input.key), input) +} + +function matchesImmutableArtifact( + object: R2Object | null, + expected: { contentSha256: string, contentBytes: number }, +): boolean { + return object !== null + && object.size === expected.contentBytes + && checksumHex(object.checksums.sha256) === expected.contentSha256 + && object.customMetadata?.contentSha256 === expected.contentSha256 + && object.customMetadata?.format === 'skilld-tar-v1' } function checksumHex(value: ArrayBuffer | undefined): string | null { diff --git a/layers/artifact-delivery/server/utils/build.ts b/layers/artifact-delivery/server/utils/build.ts index d6321f2b..d07a4672 100644 --- a/layers/artifact-delivery/server/utils/build.ts +++ b/layers/artifact-delivery/server/utils/build.ts @@ -1,4 +1,6 @@ import type { + ArtifactAttestation, + ArtifactFile, CheckResult, ProblemCode, ResolvedSource, @@ -7,25 +9,29 @@ import type { import type { ArtifactSigner } from './attestation' import type { CheckedArtifactSource } from './checks' import type { ArtifactSourceFile, PublicGithubSourceClient, SourceRejection } from './github-source' -import type { ResolutionRow } from './state' +import type { ReadyBuildLookup, ResolutionPatch, ResolutionRow } from './state' import type { TrustedRoot } from './trusted-root' +import { createWideEvent } from '@harlan-zw/nuxt-wide-events/standalone' +import { emitOperationalEvent } from '#server/utils/operational-event' import { artifactAttestationSchema, artifactAttestationStatementSchema, resolvedSourceSchema, sourceRequestSchema, } from '../schemas/contracts' -import { artifactR2Key, putImmutableArtifact } from './artifact-storage' +import { artifactR2Key, hasImmutableArtifact, putImmutableArtifact } from './artifact-storage' import { completeAttestation, createAttestationStatement, encodeAttestationStatement, verifyArtifactAttestation, } from './attestation' -import { checkArtifactSource, checksBlockArtifact } from './checks' -import { digestHex } from './encoding' +import { checkArtifactSource, checksBlockArtifact, checksPermitSigning } from './checks' +import { canonicalJson, digestHex } from './encoding' import { isRetryableProblem } from './github-source' import { + ARTIFACT_POLICY_VERSION, + findReadyPublicBuild, getResolution, parseCheckResults, publishArtifactRecord, @@ -56,21 +62,76 @@ export interface ArtifactBuildDependencies { signer: ArtifactSigner trustedRoot: TrustedRoot now: () => number + /** + * Receives each decision to reuse a ready build or not. The default emits the + * `artifact-build-reuse` wide event, which the daily check-in can count. + */ + reportReuse?: (report: ArtifactBuildReuseReport) => void } +/** Why a build did not reuse a ready build of the same commit. */ +export type ReuseMissReason + = 'no-ready-build' + | 'attestation-untrusted' + | 'policy-changed' + | 'checks-changed' + | 'record-mismatch' + | 'bytes-missing' + +export type ArtifactBuildReuseReport + = { _tag: 'hit', lookup: ReadyBuildLookup['_tag'], resolutionId: string, reusedFrom: string } + | { _tag: 'miss', lookup: ReadyBuildLookup['_tag'], resolutionId: string, reason: ReuseMissReason } + export type ArtifactBuildOutcome = { _tag: 'ready' | 'blocked' | 'failed' | 'unchanged', resolutionId: string } | { _tag: 'superseded', resolutionId: string } +/** + * A ready public build of the same commit, verified against the current trusted + * root and still stored in R2. Its files and check results come from its signed + * attestation, never from unsigned columns. + */ +interface ReusableBuild { + resolutionId: string + source: ResolvedSource + artifactId: string + contentSha256: string + contentBytes: number + files: ArtifactFile[] + checkResults: CheckResult[] +} + +type ReuseDecision + = { _tag: 'hit', build: ReusableBuild } + | { _tag: 'miss', reason: ReuseMissReason } + +interface LoadedBuild { + _tag: 'loaded' + source: ResolvedSource + files: ArtifactSourceFile[] + checked: CheckedArtifactSource +} + +interface ReusedBuild { + _tag: 'reused' + build: ReusableBuild +} + type BuildLoad - = { _tag: 'loaded', source: ResolvedSource, files: ArtifactSourceFile[], checked: CheckedArtifactSource } + = LoadedBuild + | ReusedBuild | { _tag: 'rejected', rejection: SourceRejection } +type StagedStatement + = { _tag: 'staged', patch: ResolutionPatch } + | { _tag: 'content-changed' } + /** - * The loaded source carried between the `fetching`, `packaging` and `signing` - * states of one invocation. A large build spends its subrequest budget on the - * blob fetches, so one invocation loads at most once; a resumed invocation - * starts with no carried load and loads again. + * The loaded or reused source carried between the `fetching`, `packaging` and + * `signing` states of one invocation. A large build spends its subrequest + * budget on the blob fetches, so one invocation loads at most once. A resumed + * invocation starts with no carried load: it reuses a ready build again when + * one matches, and loads from GitHub otherwise. */ type CarriedBuildLoad = BuildLoad | null @@ -96,6 +157,18 @@ export async function processArtifactBuild( } if (row.state === 'resolving') { + // A request pinned to a commit that was built before needs no GitHub + // read. An unpinned request always resolves on GitHub, so it never gets + // an older commit than the branch or tag names now. + const pinned = await reusePinnedBuild(dependencies, row) + if (pinned) { + const advanced = await transitionResolution(dependencies.db, row, 'fetching', resolvedSourcePatch(pinned.build.source), now) + if (advanced._tag === 'superseded') + return { _tag: 'superseded', resolutionId } + row = advanced.row + carried = pinned + continue + } const github = await githubForResolution(dependencies, row) if (isSourceRejection(github)) return await failWithRejection(dependencies, row, github) @@ -105,14 +178,7 @@ export async function processArtifactBuild( const source = result.source if (source.visibility !== row.visibility) return await failResolution(dependencies, row, 'SOURCE_NOT_FOUND', false) - const advanced = await transitionResolution(dependencies.db, row, 'fetching', { - repositoryId: source.repositoryId, - resolvedOwner: source.owner, - resolvedRepository: source.repository, - commitSha: source.commitSha, - treeSha: source.treeSha, - skillPath: source.skillPath, - }, now) + const advanced = await transitionResolution(dependencies.db, row, 'fetching', resolvedSourcePatch(source), now) if (advanced._tag === 'superseded') return { _tag: 'superseded', resolutionId } row = advanced.row @@ -120,11 +186,13 @@ export async function processArtifactBuild( } if (row.state === 'fetching') { - const loaded = await loadAndCheck(dependencies, row, resolvedSourceFromRow(row)) - carried = loaded._tag === 'loaded' ? loaded : null + const loaded: BuildLoad = carried ?? await reuseOrLoad(dependencies, row) + carried = loaded._tag === 'rejected' ? null : loaded const checkResults = loaded._tag === 'loaded' ? loaded.checked.checkResults - : rejectionCheckResults(loaded.rejection) + : loaded._tag === 'reused' + ? loaded.build.checkResults + : rejectionCheckResults(loaded.rejection) if (loaded._tag === 'rejected' && loaded.rejection.code !== 'INVALID_SOURCE') return await failWithRejection(dependencies, row, loaded.rejection) const advanced = await transitionResolution(dependencies.db, row, 'checking', { @@ -149,16 +217,17 @@ export async function processArtifactBuild( } if (row.state === 'packaging') { - const loaded: BuildLoad = carried ?? await requirePassingSource(dependencies, row) + const loaded: BuildLoad = carried ?? await resumeLoad(dependencies, row) if (loaded._tag === 'rejected') return await failWithRejection(dependencies, row, loaded.rejection) carried = loaded - const archive = createDeterministicUstar(loaded.files) - const contentSha256 = await digestHex('SHA-256', archive) + const content = loaded._tag === 'reused' + ? { contentSha256: loaded.build.contentSha256, contentBytes: loaded.build.contentBytes } + : await packagedContent(loaded.files) const advanced = await transitionResolution(dependencies.db, row, 'signing', { - artifactId: `sha256:${contentSha256}`, - contentSha256, - contentBytes: archive.byteLength, + artifactId: `sha256:${content.contentSha256}`, + contentSha256: content.contentSha256, + contentBytes: content.contentBytes, }, now) if (advanced._tag === 'superseded') return { _tag: 'superseded', resolutionId } @@ -169,42 +238,23 @@ export async function processArtifactBuild( if (row.state === 'signing') { if (!row.artifact_id || !row.content_sha256 || !row.content_bytes) throw new Error('Signing Resolution has no Artifact content identity') + const content = { + artifactId: row.artifact_id, + contentSha256: row.content_sha256, + contentBytes: row.content_bytes, + } if (!row.attestation_statement_json) { - const loaded: BuildLoad = carried ?? await requirePassingSource(dependencies, row) + const loaded: BuildLoad = carried ?? await resumeLoad(dependencies, row) if (loaded._tag === 'rejected') return await failWithRejection(dependencies, row, loaded.rejection) carried = loaded - const archive = createDeterministicUstar(loaded.files) - const contentSha256 = await digestHex('SHA-256', archive) - if (contentSha256 !== row.content_sha256 || archive.byteLength !== row.content_bytes) + const statement = loaded._tag === 'reused' + ? stageReusedStatement(row, content, loaded.build) + : await stageLoadedStatement(dependencies, row, content, loaded) + if (statement._tag === 'content-changed') return await failResolution(dependencies, row, 'INVALID_SOURCE', false) - const write = row.visibility === 'private' - ? await putEncryptedPrivateArtifact(dependencies, row, archive, contentSha256) - : await putImmutableArtifact(dependencies.bucket, { - key: artifactR2Key(contentSha256), - bytes: archive, - contentSha256, - }) - if (write._tag === 'mutation-rejected') - throw new Error('Immutable Artifact storage rejected changed bytes') - const statement = createAttestationStatement({ - artifactId: row.artifact_id, - createdAt: new Date(row.created_at * 1000).toISOString(), - source: loaded.source, - contentSha256, - contentBytes: archive.byteLength, - files: loaded.checked.files, - checkResults: loaded.checked.checkResults, - }) - const privateStoragePatch = row.visibility === 'private' - ? privateArtifactStoragePatch(write) - : {} - const staged = await transitionResolution(dependencies.db, row, 'signing', { - r2Key: write.key, - ...privateStoragePatch, - attestationStatementJson: encodeAttestationStatement(statement), - }, now) + const staged = await transitionResolution(dependencies.db, row, 'signing', statement.patch, now) if (staged._tag === 'superseded') return { _tag: 'superseded', resolutionId } row = staged.row @@ -244,6 +294,270 @@ export async function processArtifactBuild( throw new Error('Artifact build exceeded its state transition limit') } +async function packagedContent(files: ArtifactSourceFile[]): Promise<{ contentSha256: string, contentBytes: number }> { + const archive = createDeterministicUstar(files) + return { contentSha256: await digestHex('SHA-256', archive), contentBytes: archive.byteLength } +} + +/** Store the loaded bytes, then stage the statement the signer signs. */ +async function stageLoadedStatement( + dependencies: ArtifactBuildDependencies, + row: ResolutionRow, + content: { artifactId: string, contentSha256: string, contentBytes: number }, + loaded: LoadedBuild, +): Promise { + const archive = createDeterministicUstar(loaded.files) + const contentSha256 = await digestHex('SHA-256', archive) + if (contentSha256 !== content.contentSha256 || archive.byteLength !== content.contentBytes) + return { _tag: 'content-changed' } + const write = row.visibility === 'private' + ? await putEncryptedPrivateArtifact(dependencies, row, archive, contentSha256) + : await putImmutableArtifact(dependencies.bucket, { + key: artifactR2Key(contentSha256), + bytes: archive, + contentSha256, + }) + if (write._tag === 'mutation-rejected') + throw new Error('Immutable Artifact storage rejected changed bytes') + const statement = createAttestationStatement({ + artifactId: content.artifactId, + createdAt: new Date(row.created_at * 1000).toISOString(), + source: loaded.source, + contentSha256, + contentBytes: archive.byteLength, + files: loaded.checked.files, + checkResults: loaded.checked.checkResults, + }) + const privateStoragePatch = row.visibility === 'private' + ? privateArtifactStoragePatch(write) + : {} + return { + _tag: 'staged', + patch: { + r2Key: write.key, + ...privateStoragePatch, + attestationStatementJson: encodeAttestationStatement(statement), + }, + } +} + +/** + * Stage a new statement over bytes that R2 already holds. + * + * The statement names this Resolution's source and creation time, with the + * files and check results of the reused build. The signer then signs it + * through the same path as a fresh build: it hashes the R2 object again, and + * checks the policy version, the check results, and that the statement matches + * this row. No earlier signature is copied. + */ +function stageReusedStatement( + row: ResolutionRow, + content: { artifactId: string, contentSha256: string, contentBytes: number }, + build: ReusableBuild, +): StagedStatement { + if ( + build.artifactId !== content.artifactId + || build.contentSha256 !== content.contentSha256 + || build.contentBytes !== content.contentBytes + ) { + return { _tag: 'content-changed' } + } + const statement = createAttestationStatement({ + artifactId: content.artifactId, + createdAt: new Date(row.created_at * 1000).toISOString(), + source: resolvedSourceFromRow(row), + contentSha256: content.contentSha256, + contentBytes: content.contentBytes, + files: build.files, + checkResults: build.checkResults, + }) + return { + _tag: 'staged', + patch: { + r2Key: artifactR2Key(content.contentSha256), + attestationStatementJson: encodeAttestationStatement(statement), + }, + } +} + +/** A pinned public request reuses a ready build of its commit before any GitHub read. */ +async function reusePinnedBuild( + dependencies: ArtifactBuildDependencies, + row: ResolutionRow, +): Promise { + if (row.visibility !== 'public') + return null + const request = sourceRequestFromRow(row) + if (request.ref?.type !== 'commit') + return null + const decision = await decideReuse(dependencies, row, { + _tag: 'pinned', + owner: request.owner, + repository: request.repository, + commitSha: request.ref.value, + selector: request.selector, + }) + return decision._tag === 'hit' ? { _tag: 'reused', build: decision.build } : null +} + +/** After GitHub resolves a public request, reuse a ready build of that exact source, or load it. */ +async function reuseOrLoad( + dependencies: ArtifactBuildDependencies, + row: ResolutionRow, +): Promise { + const source = resolvedSourceFromRow(row) + if (row.visibility === 'public') { + const decision = await decideReuse(dependencies, row, { _tag: 'resolved', source }) + if (decision._tag === 'hit') + return { _tag: 'reused', build: decision.build } + } + return await loadAndCheck(dependencies, row, source) +} + +/** + * The source for a build that resumes after `fetching` in a new invocation. + * A ready build is reused only when it matches the check results and content + * this Resolution already recorded. + */ +async function resumeLoad( + dependencies: ArtifactBuildDependencies, + row: ResolutionRow, +): Promise { + if (row.visibility === 'public') { + const decision = await decideReuse( + dependencies, + row, + { _tag: 'resolved', source: resolvedSourceFromRow(row) }, + build => canonicalJson(build.checkResults) === canonicalJson(parseCheckResults(row.check_results_json)) + && (row.content_sha256 === null || row.content_sha256 === build.contentSha256) + && (row.content_bytes === null || row.content_bytes === build.contentBytes), + ) + if (decision._tag === 'hit') + return { _tag: 'reused', build: decision.build } + } + return await requirePassingSource(dependencies, row) +} + +async function decideReuse( + dependencies: ArtifactBuildDependencies, + row: ResolutionRow, + lookup: ReadyBuildLookup, + fitsRecord: (build: ReusableBuild) => boolean = () => true, +): Promise { + const found = await findReusableBuild(dependencies, lookup) + const decision: ReuseDecision = found._tag === 'hit' && !fitsRecord(found.build) + ? { _tag: 'miss', reason: 'record-mismatch' } + : found + const report: ArtifactBuildReuseReport = decision._tag === 'hit' + ? { _tag: 'hit', lookup: lookup._tag, resolutionId: row.id, reusedFrom: decision.build.resolutionId } + : { _tag: 'miss', lookup: lookup._tag, resolutionId: row.id, reason: decision.reason } + ;(dependencies.reportReuse ?? emitReuseEvent)(report) + return decision +} + +/** + * The newest ready public build for the lookup, when every input that fixes + * its bytes and its attestation still holds: + * + * - the current trusted root verifies its attestation, so the signing key is + * still trusted and the stored statement is the one that was signed; + * - the statement names the current policy version and format; + * - its check results pass under the current check versions; + * - the statement matches the stored row; + * - R2 still holds the exact bytes. + */ +async function findReusableBuild( + dependencies: ArtifactBuildDependencies, + lookup: ReadyBuildLookup, +): Promise { + const ready = await findReadyPublicBuild(dependencies.db, lookup) + if (!ready) + return { _tag: 'miss', reason: 'no-ready-build' } + const attestation = parseStoredAttestation(ready.attestationJson) + if (!attestation || !await verifyArtifactAttestation(attestation, dependencies.trustedRoot, dependencies.now())) + return { _tag: 'miss', reason: 'attestation-untrusted' } + if (attestation.policyVersion !== ARTIFACT_POLICY_VERSION) + return { _tag: 'miss', reason: 'policy-changed' } + if (!checksPermitSigning(attestation.checkResults)) + return { _tag: 'miss', reason: 'checks-changed' } + if ( + attestation.artifactId !== ready.artifactId + || ready.artifactId !== `sha256:${ready.contentSha256}` + || attestation.contentSha256 !== ready.contentSha256 + || attestation.contentBytes !== ready.contentBytes + || ready.r2Key !== artifactR2Key(ready.contentSha256) + || canonicalJson(attestation.source) !== canonicalJson(ready.source) + ) { + return { _tag: 'miss', reason: 'record-mismatch' } + } + const stored = await hasImmutableArtifact(dependencies.bucket, { + key: ready.r2Key, + contentSha256: ready.contentSha256, + contentBytes: ready.contentBytes, + }) + if (!stored) + return { _tag: 'miss', reason: 'bytes-missing' } + return { + _tag: 'hit', + build: { + resolutionId: ready.resolutionId, + source: ready.source, + artifactId: ready.artifactId, + contentSha256: ready.contentSha256, + contentBytes: ready.contentBytes, + files: attestation.files, + checkResults: attestation.checkResults, + }, + } +} + +function parseStoredAttestation(value: string): ArtifactAttestation | null { + const parsed = artifactAttestationSchema.safeParse(parseJson(value)) + return parsed.success ? parsed.data : null +} + +function parseJson(value: string): unknown { + try { + return JSON.parse(value) as unknown + } + catch { + // Stored text that is not JSON cannot be verified. The caller reports a miss. + return undefined + } +} + +/** One wide event per reuse decision, so the daily check-in can count hits. */ +function emitReuseEvent(report: ArtifactBuildReuseReport): void { + if (report._tag === 'hit') { + emitOperationalEvent(createWideEvent({ + 'operation': 'artifact-build-reuse', + 'outcome': 'hit', + 'artifact.resolutionId': report.resolutionId, + 'artifact.reuseLookup': report.lookup, + 'artifact.reusedFrom': report.reusedFrom, + }), 'info') + return + } + emitOperationalEvent(createWideEvent({ + 'operation': 'artifact-build-reuse', + 'outcome': 'miss', + 'artifact.resolutionId': report.resolutionId, + 'artifact.reuseLookup': report.lookup, + 'reason': report.reason, + }), 'info') +} + +function resolvedSourcePatch(source: ResolvedSource): ResolutionPatch { + return { + repositoryId: source.repositoryId, + resolvedOwner: source.owner, + resolvedRepository: source.repository, + commitSha: source.commitSha, + treeSha: source.treeSha, + skillPath: source.skillPath, + } +} + async function loadAndCheck( dependencies: ArtifactBuildDependencies, row: ResolutionRow, diff --git a/layers/artifact-delivery/server/utils/state.ts b/layers/artifact-delivery/server/utils/state.ts index e5c219ba..dc268731 100644 --- a/layers/artifact-delivery/server/utils/state.ts +++ b/layers/artifact-delivery/server/utils/state.ts @@ -3,6 +3,7 @@ import type { CheckResult, ProblemCode, ResolutionResponse, + ResolvedSource, SourceRequest, } from '../schemas/contracts' import { z } from 'zod' @@ -10,9 +11,18 @@ import { artifactAttestationSchema, checkResultSchema, problemCodeSchema, + resolvedSourceSchema, } from '../schemas/contracts' import { canonicalJson, digestHex } from './encoding' +/** + * The Artifact policy every attestation names. The artifact signer refuses a + * statement under another version. + * + * Bump it when a change to loading, packaging, or checks changes what one + * commit produces. A ready build signed under another version is never reused, + * so the bump also makes every commit load from GitHub once more. + */ export const ARTIFACT_POLICY_VERSION = '2026-08-20.1' export const ACTIVE_BUILD_STATES = [ @@ -251,6 +261,127 @@ export async function transitionResolution( return { _tag: 'advanced', row: advanced } } +/** + * How a build looks for a ready public build of the same commit. + * + * - `resolved`: GitHub has resolved the request. The whole source must match. + * - `pinned`: the request names a commit, and no GitHub read has happened. It + * matches the commit, the owner and Repository name that GitHub reported for + * the ready build, and the Skill path. A Skill name matches only a build + * requested by that same name, because only a resolve by name proved that + * exactly one Skill in the commit has it. + */ +export type ReadyBuildLookup + = { _tag: 'resolved', source: ResolvedSource } + | { + _tag: 'pinned' + owner: string + repository: string + commitSha: string + selector: SourceRequest['selector'] + } + +/** A ready public build as D1 stores it. Nothing here verifies its attestation. */ +export interface ReadyPublicBuild { + resolutionId: string + source: ResolvedSource + artifactId: string + contentSha256: string + contentBytes: number + r2Key: string + attestationJson: string +} + +const readyPublicBuildRowSchema = z.object({ + id: z.string().uuid(), + repository_id: z.number().int().positive().safe(), + resolved_owner: z.string(), + resolved_repository: z.string(), + commit_sha: z.string(), + tree_sha: z.string(), + skill_path: z.string(), + artifact_id: z.string(), + content_sha256: z.string(), + content_bytes: z.number().int().positive(), + r2_key: z.string(), + attestation_json: z.string(), +}) + +/** + * The newest ready public build that a lookup matches, or null. + * + * Only a `ready` Resolution matches, so a revoked, failed or blocked one never + * does. The Artifact must still have the `available` delivery status. The + * caller verifies the attestation and the stored bytes before it reuses them. + * Migration 0131 indexes both lookups. + */ +export async function findReadyPublicBuild( + db: D1Database, + lookup: ReadyBuildLookup, +): Promise { + if (lookup._tag === 'resolved' && lookup.source.visibility !== 'public') + return null + const value = await readyPublicBuildStatement(db, lookup).first>() + if (!value) + return null + const row = readyPublicBuildRowSchema.parse(value) + return { + resolutionId: row.id, + source: resolvedSourceSchema.parse({ + provider: 'github', + repositoryId: row.repository_id, + owner: row.resolved_owner, + repository: row.resolved_repository, + visibility: 'public', + commitSha: row.commit_sha, + treeSha: row.tree_sha, + skillPath: row.skill_path, + }), + artifactId: row.artifact_id, + contentSha256: row.content_sha256, + contentBytes: row.content_bytes, + r2Key: row.r2_key, + attestationJson: row.attestation_json, + } +} + +// The first two filters repeat the WHERE clause of the migration 0131 partial +// index word for word. SQLite uses a partial index only when they match. +// The order is by `created_at`, not `updated_at`: without planner stats, SQLite +// picks `idx_artifact_resolutions_state (state, updated_at)` for a pinned +// lookup when that index can supply the order, and reads every ready row. +const READY_PUBLIC_BUILD_SELECT = `SELECT r.id, r.repository_id, r.resolved_owner, r.resolved_repository, + r.commit_sha, r.tree_sha, r.skill_path, r.artifact_id, r.content_sha256, + r.content_bytes, r.r2_key, r.attestation_json + FROM artifact_resolutions r + JOIN artifacts a ON a.id = r.artifact_id + WHERE r.state = 'ready' AND r.visibility = 'public' + AND a.delivery_status = 'available'` +const NEWEST_READY_BUILD = 'ORDER BY r.created_at DESC, r.id DESC LIMIT 1' + +function readyPublicBuildStatement(db: D1Database, lookup: ReadyBuildLookup): D1PreparedStatement { + if (lookup._tag === 'resolved') { + const { source } = lookup + return db.prepare( + `${READY_PUBLIC_BUILD_SELECT} + AND r.commit_sha = ?1 AND r.repository_id = ?2 AND r.skill_path = ?3 + AND r.tree_sha = ?4 AND r.resolved_owner = ?5 AND r.resolved_repository = ?6 + ${NEWEST_READY_BUILD}`, + ).bind(source.commitSha, source.repositoryId, source.skillPath, source.treeSha, source.owner, source.repository) + } + const selector = lookup.selector.type === 'path' + ? { filter: 'r.skill_path = ?4', value: lookup.selector.path } + : { filter: 'r.selector_type = \'named-skill\' AND r.selector_value = ?4', value: lookup.selector.name } + return db.prepare( + `${READY_PUBLIC_BUILD_SELECT} + AND r.commit_sha = ?1 + AND r.resolved_owner = ?2 COLLATE NOCASE + AND r.resolved_repository = ?3 COLLATE NOCASE + AND ${selector.filter} + ${NEWEST_READY_BUILD}`, + ).bind(lookup.commitSha, lookup.owner, lookup.repository, selector.value) +} + export function parseCheckResults(value: string | null): CheckResult[] { if (!value) return [] diff --git a/migrations/0131_artifact_build_reuse.sql b/migrations/0131_artifact_build_reuse.sql new file mode 100644 index 00000000..837b03de --- /dev/null +++ b/migrations/0131_artifact_build_reuse.sql @@ -0,0 +1,22 @@ +-- Find a ready public build of the same commit, so a repeat Resolution can +-- reuse its Artifact instead of loading the Skill from GitHub again. +-- +-- Over 14 days to 2026-09-30, 264 public builds covered only 89 distinct +-- (Repository, commit, Skill path) builds. Each build makes about 9 GitHub reads. +-- +-- The build looks up a ready row in two ways (`findReadyPublicBuild` in +-- `layers/artifact-delivery/server/utils/state.ts`): +-- +-- - After GitHub resolves the request: by commit, Repository ID, Skill path, +-- tree, owner and Repository name. +-- - Before any GitHub read, for a request pinned to a commit: by commit, owner +-- and Repository name, and the Skill path or name. The Repository ID is not +-- known yet. +-- +-- The index leads with the commit so that both lookups can seek on it. The +-- WHERE clause must stay textually equal to the query's filter, or SQLite +-- ignores the index. Only ready public rows enter it: a private Artifact is +-- never reused. +CREATE INDEX IF NOT EXISTS idx_artifact_resolutions_ready_public_commit + ON artifact_resolutions(commit_sha, repository_id, skill_path) + WHERE state = 'ready' AND visibility = 'public'; diff --git a/nuxt.config.ts b/nuxt.config.ts index 3079d38d..4eabba4d 100644 --- a/nuxt.config.ts +++ b/nuxt.config.ts @@ -106,6 +106,11 @@ export default defineNuxtConfig({ request: true, service: 'skilld', fields: [ + // `artifact-build-reuse`: which Resolution, which lookup matched, and + // the ready Resolution it reused. The daily check-in counts the hits. + 'artifact.resolutionId', + 'artifact.reuseLookup', + 'artifact.reusedFrom', 'attempt', 'batch.count', 'cache.ageSeconds', diff --git a/test/unit/artifact-build-reuse.test.ts b/test/unit/artifact-build-reuse.test.ts new file mode 100644 index 00000000..5dc57596 --- /dev/null +++ b/test/unit/artifact-build-reuse.test.ts @@ -0,0 +1,600 @@ +import type { + ArtifactAttestationStatement, + ResolvedSource, + SourceRequest, +} from '../../layers/artifact-delivery/server/schemas/contracts' +import type { ArtifactBuildDependencies, ArtifactBuildReuseReport } from '../../layers/artifact-delivery/server/utils/build' +import type { ArtifactSourceFile, PublicGithubSourceClient } from '../../layers/artifact-delivery/server/utils/github-source' +import type { ResolutionRow } from '../../layers/artifact-delivery/server/utils/state' +import type { TrustedRoot } from '../../layers/artifact-delivery/server/utils/trusted-root' +import type { ArtifactSignerBindings } from '../../workers/artifact-signer/src/handler' +import { createHash } from 'node:crypto' +import { describe, expect, it, vi } from 'vitest' +import { + artifactAttestationSchema, + resolutionSchema, +} from '../../layers/artifact-delivery/server/schemas/contracts' +import { + completeAttestation, + createArtifactSigner, + createAttestationSignaturePayload, + encodeAttestationStatement, + verifyArtifactAttestation, +} from '../../layers/artifact-delivery/server/utils/attestation' +import { processArtifactBuild } from '../../layers/artifact-delivery/server/utils/build' +import { bytesToBase64Url } from '../../layers/artifact-delivery/server/utils/encoding' +import { ARTIFACT_BUILD_QUEUE_NAME, consumeArtifactBuildBatch } from '../../layers/artifact-delivery/server/utils/queue' +import { resolveSkillPageUrl } from '../../layers/artifact-delivery/server/utils/skill-page' +import { + createResolution, + getResolution, + presentResolution, + resolutionRequestIdentity, + transitionResolution, +} from '../../layers/artifact-delivery/server/utils/state' +import { handleArtifactSignerRequest } from '../../workers/artifact-signer/src/handler' +import { createSqliteD1 } from './helpers/d1-sqlite' + +const NOW = 1_787_227_200 +const MIGRATIONS = [ + 'migrations/0017_users.sql', + 'migrations/0110_artifact_delivery.sql', + 'migrations/0111_github_app_delivery.sql', + 'migrations/0112_private_artifact_keys.sql', + 'migrations/0122_artifact_resolution_retry_after.sql', + 'migrations/0131_artifact_build_reuse.sql', +] +const COMMIT = '0123456789abcdef0123456789abcdef01234567' +const NEXT_COMMIT = 'fedcba9876543210fedcba9876543210fedcba98' +const source: ResolvedSource = { + provider: 'github', + repositoryId: 123456789, + owner: 'skilld-dev', + repository: 'skills', + visibility: 'public', + commitSha: COMMIT, + treeSha: '89abcdef0123456789abcdef0123456789abcdef', + skillPath: 'skills/demo', +} +const pinned: SourceRequest = { + provider: 'github', + owner: 'skilld-dev', + repository: 'skills', + selector: { type: 'path', path: 'skills/demo' }, + ref: { type: 'commit', value: COMMIT }, +} +const onBranch: SourceRequest = { ...pinned, ref: { type: 'branch', value: 'main' } } +const files: ArtifactSourceFile[] = [{ + path: 'SKILL.md', + mode: 420, + bytes: new TextEncoder().encode('---\nname: demo\ndescription: Use this Skill for demo work.\n---\n\n# Demo\n'), + gitBlobSha: 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', +}] + +describe('reusing a ready public build', () => { + it('serves a repeat request pinned to a built commit with no GitHub call', async () => { + const harness = await createReuseHarness() + const first = await harness.build(pinned, githubServing(source)) + harness.advanceClock(120) + const github = githubDown() + + const repeat = await harness.build(pinned, github) + + expect(repeat.outcome).toEqual({ _tag: 'ready', resolutionId: repeat.row.id }) + expect(github.resolve).not.toHaveBeenCalled() + expect(github.load).not.toHaveBeenCalled() + expect(harness.storage.put).toHaveBeenCalledOnce() + const response = resolutionSchema.parse(presentResolution(repeat.row)) + if (response.state !== 'ready') + throw new Error('The repeat Resolution is not ready') + const attestation = response.artifact.attestation + expect(await verifyArtifactAttestation(attestation, harness.trustedRoot(), harness.now())).toBe(true) + expect(response.artifact.artifactId).toBe(first.row.artifact_id) + expect(attestation.source).toEqual(source) + expect(attestation.createdAt).toBe(new Date(repeat.row.created_at * 1000).toISOString()) + expect(attestation.createdAt).not.toBe(parseReady(first.row).createdAt) + expect(harness.reports).toContainEqual({ + _tag: 'hit', + lookup: 'pinned', + resolutionId: repeat.row.id, + reusedFrom: first.row.id, + }) + harness.close() + }) + + it('asks the registry for the same Skill page as the build it reused', async () => { + const harness = await createReuseHarness() + const first = await harness.build(pinned, githubServing(source)) + const repeat = await harness.build(pinned, githubDown()) + const pageUrl = vi.fn(async () => 'https://skilld.dev/gh/skilld-dev/skills/demo') + + const firstUrl = await resolveSkillPageUrl(first.row, pageUrl, reason => expect.fail(reason)) + const repeatUrl = await resolveSkillPageUrl(repeat.row, pageUrl, reason => expect.fail(reason)) + + expect(repeatUrl).toBe(firstUrl) + expect(pageUrl.mock.calls[1]).toEqual(pageUrl.mock.calls[0]) + harness.close() + }) + + it('resolves an unpinned request on GitHub, then skips the load', async () => { + const harness = await createReuseHarness() + const first = await harness.build(pinned, githubServing(source)) + const github = githubServing(source) + + const repeat = await harness.build(onBranch, github) + + expect(repeat.row.state).toBe('ready') + expect(github.resolve).toHaveBeenCalledOnce() + expect(github.load).not.toHaveBeenCalled() + expect(harness.reports).toContainEqual({ + _tag: 'hit', + lookup: 'resolved', + resolutionId: repeat.row.id, + reusedFrom: first.row.id, + }) + harness.close() + }) + + it('does not serve an older build when GitHub cannot resolve an unpinned request', async () => { + const harness = await createReuseHarness() + await harness.build(pinned, githubServing(source)) + const resolutionId = await harness.request(onBranch) + + await expect(processArtifactBuild(harness.dependencies(githubDown()), resolutionId)) + .rejects + .toThrow('The operation was aborted due to timeout') + + expect((await getResolution(harness.db, resolutionId))?.state).toBe('resolving') + harness.close() + }) + + it('loads a different commit from GitHub', async () => { + const harness = await createReuseHarness() + await harness.build(pinned, githubServing(source)) + const next = { ...source, commitSha: NEXT_COMMIT } + const github = githubServing(next) + + const built = await harness.build({ ...pinned, ref: { type: 'commit', value: NEXT_COMMIT } }, github) + + expect(built.row.state).toBe('ready') + expect(github.resolve).toHaveBeenCalledOnce() + expect(github.load).toHaveBeenCalledOnce() + expect(parseReady(built.row).source.commitSha).toBe(NEXT_COMMIT) + harness.close() + }) + + it('resolves a pinned Skill name on GitHub until a build of that name exists', async () => { + const harness = await createReuseHarness() + await harness.build(pinned, githubServing(source)) + const named: SourceRequest = { ...pinned, selector: { type: 'named-skill', name: 'demo' } } + const firstNamed = githubServing(source) + const secondNamed = githubDown() + + const first = await harness.build(named, firstNamed) + const second = await harness.build(named, secondNamed) + + expect(first.row.state).toBe('ready') + expect(firstNamed.resolve).toHaveBeenCalledOnce() + expect(firstNamed.load).not.toHaveBeenCalled() + expect(second.row.state).toBe('ready') + expect(secondNamed.resolve).not.toHaveBeenCalled() + harness.close() + }) + + it.each([ + { + change: 'policy version', + reason: 'policy-changed', + edit: (statement: ArtifactAttestationStatement) => ({ ...statement, policyVersion: '2026-01-01.0' }), + }, + { + change: 'check version', + reason: 'checks-changed', + edit: (statement: ArtifactAttestationStatement) => ({ + ...statement, + checkResults: statement.checkResults.map(check => check.name === 'agent-skills-spec' + ? { ...check, version: '2026-01-01' } + : check), + }), + }, + ])('loads again when the ready build was signed under another $change', async ({ reason, edit }) => { + const harness = await createReuseHarness() + const first = await harness.build(pinned, githubServing(source)) + await harness.resignReadyBuild(first.row.id, edit) + const github = githubServing(source) + + const repeat = await harness.build(pinned, github) + + expect(repeat.row.state).toBe('ready') + expect(github.load).toHaveBeenCalledOnce() + expect(harness.reports).toContainEqual({ _tag: 'miss', lookup: 'pinned', resolutionId: repeat.row.id, reason }) + harness.close() + }) + + it('loads again after its signing key leaves the trusted root', async () => { + const harness = await createReuseHarness() + await harness.build(pinned, githubServing(source)) + await harness.rotateSigningKey() + const github = githubServing(source) + + const repeat = await harness.build(pinned, github) + + expect(github.load).toHaveBeenCalledOnce() + expect(harness.reports).toContainEqual({ + _tag: 'miss', + lookup: 'pinned', + resolutionId: repeat.row.id, + reason: 'attestation-untrusted', + }) + expect(await verifyArtifactAttestation(parseReady(repeat.row), harness.trustedRoot(), harness.now())).toBe(true) + harness.close() + }) + + it.each([ + { + prior: 'revoked Resolution', + spoil: async (harness: ReuseHarness, row: ResolutionRow) => { + await transitionResolution(harness.db, row, 'revoked', { errorCode: 'ARTIFACT_REVOKED' }, harness.now()) + }, + }, + { + prior: 'revoked Artifact', + spoil: async (harness: ReuseHarness, row: ResolutionRow) => { + harness.raw.prepare('UPDATE artifacts SET delivery_status = \'revoked\' WHERE id = ?').run(row.artifact_id) + }, + }, + ])('loads again rather than reuse a $prior', async ({ spoil }) => { + const harness = await createReuseHarness() + const first = await harness.build(pinned, githubServing(source)) + await spoil(harness, first.row) + const github = githubServing(source) + + const repeat = await harness.build(pinned, github) + + expect(repeat.row.state).toBe('ready') + expect(github.load).toHaveBeenCalledOnce() + harness.close() + }) + + it('loads again rather than reuse a failed build of the same commit', async () => { + const harness = await createReuseHarness() + const failedId = await harness.request(pinned) + const signerDown = { + ...harness.dependencies(githubServing(source)), + signer: { sign: vi.fn(async () => Promise.reject(new Error('Artifact signer returned 503'))) }, + } + await consumeArtifactBuildBatch({} as Cloudflare.Env, queueBatch([failedId], 5), () => signerDown) + expect((await getResolution(harness.db, failedId))?.state).toBe('failed') + const github = githubServing(source) + + const repeat = await harness.build(pinned, github) + + expect(repeat.row.state).toBe('ready') + expect(github.load).toHaveBeenCalledOnce() + harness.close() + }) + + it('loads again when the stored bytes are gone', async () => { + const harness = await createReuseHarness() + const first = await harness.build(pinned, githubServing(source)) + harness.storage.remove(first.row.r2_key!) + const github = githubServing(source) + + const repeat = await harness.build(pinned, github) + + expect(repeat.row.state).toBe('ready') + expect(github.load).toHaveBeenCalledOnce() + expect(harness.storage.put).toHaveBeenCalledTimes(2) + expect(harness.reports).toContainEqual({ + _tag: 'miss', + lookup: 'pinned', + resolutionId: repeat.row.id, + reason: 'bytes-missing', + }) + harness.close() + }) + + it.each([ + { recorded: 'the reused check results', edit: (checks: string) => checks, loads: 0, result: 'ready' }, + // The fresh load cannot match what the row recorded either, so the signer + // refuses the statement. The point is that GitHub was asked. + { + recorded: 'other check results', + edit: (checks: string) => checks.replace('"pass"', '"warn"'), + loads: 1, + result: 'Artifact signer returned 409', + }, + ])('resumes a Resolution left in packaging with $recorded', async ({ edit, loads, result }) => { + const harness = await createReuseHarness() + const first = await harness.build(pinned, githubServing(source)) + const resolutionId = await harness.request(pinned) + // A new invocation picks up a Resolution that an earlier one left in `packaging`. + let row = (await getResolution(harness.db, resolutionId))! + for (const [next, patch] of [ + ['resolving', {}], + ['fetching', { + repositoryId: source.repositoryId, + resolvedOwner: source.owner, + resolvedRepository: source.repository, + commitSha: source.commitSha, + treeSha: source.treeSha, + skillPath: source.skillPath, + }], + ['checking', { checkResultsJson: edit(first.row.check_results_json!) }], + ['packaging', {}], + ] as const) { + const advanced = await transitionResolution(harness.db, row, next, patch, harness.now()) + if (advanced._tag !== 'advanced') + throw new Error('Test Resolution was superseded') + row = advanced.row + } + const github = githubServing(source) + + const outcome = await processArtifactBuild(harness.dependencies(github), resolutionId) + .then(value => value._tag, (error: Error) => error.message) + + expect(github.resolve).not.toHaveBeenCalled() + expect(github.load).toHaveBeenCalledTimes(loads) + expect(outcome).toBe(result) + harness.close() + }) + + it('loads GitHub once for two Resolutions of one commit in one Queue batch', async () => { + const harness = await createReuseHarness() + const first = await harness.request(pinned) + const second = await harness.request(onBranch) + const github = githubServing(source) + + await consumeArtifactBuildBatch({} as Cloudflare.Env, queueBatch([first, second], 1), () => harness.dependencies(github)) + + const rows = await Promise.all([first, second].map(id => getResolution(harness.db, id))) + expect(rows.map(row => row?.state)).toEqual(['ready', 'ready']) + expect(rows[1]?.artifact_id).toBe(rows[0]?.artifact_id) + expect(github.load).toHaveBeenCalledOnce() + harness.close() + }) + + it('never gives a private Resolution a public build', async () => { + const harness = await createReuseHarness() + await harness.build(pinned, githubServing(source)) + const privateSource = { ...source, visibility: 'private' as const } + const github = githubServing(privateSource) + harness.raw.prepare( + `INSERT INTO users (id, github_id, login, digest_frequency, digest_hour, timezone, created_at, last_login_at) + VALUES (1, 101, 'skilld-dev', 'weekly', 9, 'UTC', ?, ?)`, + ).run(NOW, NOW) + const resolutionId = await harness.request(pinned, { + visibility: 'private', + accountId: 1, + installationId: 9001, + repositoryId: source.repositoryId, + }) + const dependencies: ArtifactBuildDependencies = { + ...harness.dependencies(githubDown()), + privateGithub: async () => github, + privateArtifacts: { + put: vi.fn(async () => ({ + _tag: 'stored' as const, + key: `v1/private/account/${resolutionId}.bin`, + ciphertextSha256: 'f'.repeat(64), + ciphertextBytes: 1024, + encryptionKeyId: 'account-fixture', + })), + }, + signer: { sign: vi.fn(async () => Promise.reject(new Error('Private signing is out of scope here'))) }, + } + + await expect(processArtifactBuild(dependencies, resolutionId)).rejects.toThrow('Private signing is out of scope here') + + expect(github.resolve).toHaveBeenCalledOnce() + expect(github.load).toHaveBeenCalledOnce() + expect(harness.reports.filter(report => report.resolutionId === resolutionId)).toEqual([]) + harness.close() + }) +}) + +type ReuseHarness = Awaited> + +async function createReuseHarness() { + const sqlite = createSqliteD1(MIGRATIONS) + const storage = memoryBucket() + const reports: ArtifactBuildReuseReport[] = [] + let clock = NOW + let requests = 0 + let signing = await signingKey('skilld-production-2026-08') + const signerBindings: ArtifactSignerBindings = { + DB: sqlite.db, + PUBLIC_ARTIFACTS: storage.bucket, + PRIVATE_ARTIFACTS: storage.bucket, + ARTIFACT_SIGNING_MAX_AGE_SECONDS: '300', + ...signing.bindings, + } + const signer = createArtifactSigner({ + fetch: async (request: Request) => await handleArtifactSignerRequest(request, signerBindings, () => clock), + } as unknown as Fetcher) + + function dependencies(github: PublicGithubSourceClient): ArtifactBuildDependencies { + return { + db: sqlite.db, + github, + bucket: storage.bucket, + signer, + trustedRoot: signing.trustedRoot, + now: () => clock, + reportReuse: report => reports.push(report), + } + } + + async function request( + sourceRequest: SourceRequest, + access?: Parameters[4], + ): Promise { + requests += 1 + const accountId = access?.visibility === 'private' ? access.accountId : undefined + const identity = await resolutionRequestIdentity(sourceRequest, `reuse-test-request-${requests}`, accountId) + const created = await createResolution(sqlite.db, sourceRequest, identity, clock, access) + if (created._tag === 'idempotency-conflict') + throw new Error('Test Resolution conflicted') + return created.row.id + } + + async function build(sourceRequest: SourceRequest, github: PublicGithubSourceClient) { + const resolutionId = await request(sourceRequest) + const outcome = await processArtifactBuild(dependencies(github), resolutionId) + const row = await getResolution(sqlite.db, resolutionId) + if (!row) + throw new Error('Test Resolution was not stored') + return { outcome, row } + } + + /** Replace a ready build's attestation with a changed statement the current key signs. */ + async function resignReadyBuild( + resolutionId: string, + edit: (statement: ArtifactAttestationStatement) => ArtifactAttestationStatement, + ): Promise { + const row = await getResolution(sqlite.db, resolutionId) + if (!row) + throw new Error('Test Resolution was not stored') + const { statement: _statement, signature: _signature, ...statement } = parseReady(row) + const rawStatement = encodeAttestationStatement(edit(statement)) + const signature = await crypto.subtle.sign( + 'Ed25519', + signing.privateKey, + await createAttestationSignaturePayload(new TextEncoder().encode(rawStatement)), + ) + const attestation = completeAttestation(rawStatement, { + algorithm: 'Ed25519', + keyId: signing.keyId, + value: bytesToBase64Url(new Uint8Array(signature)), + }) + sqlite.raw.prepare('UPDATE artifact_resolutions SET attestation_json = ? WHERE id = ?') + .run(JSON.stringify(attestation), resolutionId) + } + + /** The signer moves to a new key, and the trusted root drops the old one. */ + async function rotateSigningKey(): Promise { + signing = await signingKey('skilld-production-2026-11') + Object.assign(signerBindings, signing.bindings) + } + + return { + db: sqlite.db, + raw: sqlite.raw, + storage, + reports, + now: () => clock, + advanceClock: (seconds: number) => { + clock += seconds + }, + trustedRoot: () => signing.trustedRoot, + dependencies, + request, + build, + resignReadyBuild, + rotateSigningKey, + close: sqlite.close, + } +} + +function githubServing(resolved: ResolvedSource) { + return { + resolve: vi.fn(async () => ({ _tag: 'resolved' as const, source: resolved })), + load: vi.fn(async () => ({ _tag: 'loaded' as const, value: { source: resolved, files } })), + } satisfies PublicGithubSourceClient +} + +/** GitHub as it behaved on 2026-09-30: every read hangs until its timeout. */ +function githubDown() { + const fault = async () => Promise.reject(new Error('The operation was aborted due to timeout')) + return { resolve: vi.fn(fault), load: vi.fn(fault) } satisfies PublicGithubSourceClient +} + +function parseReady(row: ResolutionRow) { + return artifactAttestationSchema.parse(JSON.parse(row.attestation_json ?? 'null')) +} + +function queueBatch(resolutionIds: string[], attempts: number) { + return { + queue: ARTIFACT_BUILD_QUEUE_NAME, + messages: resolutionIds.map(resolutionId => ({ + body: { version: 1, resolutionId }, + attempts, + ack: vi.fn(), + retry: vi.fn(), + })), + } as unknown as Parameters[1] +} + +async function signingKey(keyId: string) { + const keyPair = await crypto.subtle.generateKey('Ed25519', true, ['sign', 'verify']) + const publicKey = bytesToBase64Url(new Uint8Array(await crypto.subtle.exportKey('raw', keyPair.publicKey))) + const privateKey = bytesToBase64Url(new Uint8Array(await crypto.subtle.exportKey('pkcs8', keyPair.privateKey))) + const notBefore = new Date((NOW - 60) * 1000).toISOString() + const notAfter = new Date((NOW + 3600) * 1000).toISOString() + const trustedRoot: TrustedRoot = { + version: 1, + rootKeyId: 'skilld-root-2026', + rootPublicKey: publicKey, + keys: [{ + keyId, + algorithm: 'Ed25519', + publicKey, + notBefore, + notAfter, + status: 'active', + statement: bytesToBase64Url(new TextEncoder().encode(JSON.stringify({ keyId }))), + rootSignature: 'C'.repeat(86), + }], + fetchedAt: new Date(NOW * 1000).toISOString(), + } + return { + keyId, + privateKey: keyPair.privateKey, + trustedRoot, + bindings: { + ARTIFACT_SIGNING_KEY_ID: keyId, + ARTIFACT_SIGNING_KEY_NOT_BEFORE: notBefore, + ARTIFACT_SIGNING_KEY_NOT_AFTER: notAfter, + ARTIFACT_SIGNING_PRIVATE_KEY_PKCS8: privateKey, + }, + } +} + +/** An R2 bucket in memory that computes checksums from the bytes, as R2 does. */ +function memoryBucket() { + const objects = new Map }>() + const describeObject = (key: string) => { + const object = objects.get(key) + if (!object) + return null + const digest = createHash('sha256').update(object.bytes).digest() + return { + key, + size: object.bytes.byteLength, + checksums: { sha256: Uint8Array.from(digest).buffer }, + customMetadata: object.customMetadata, + } + } + const put = vi.fn(async (key: string, value: Uint8Array, options?: R2PutOptions) => { + if (objects.has(key)) + return null + objects.set(key, { bytes: Uint8Array.from(value), customMetadata: options?.customMetadata ?? {} }) + return describeObject(key) + }) + const bucket = { + put, + head: async (key: string) => describeObject(key), + get: async (key: string) => { + const object = describeObject(key) + const stored = objects.get(key) + return object && stored + ? { ...object, arrayBuffer: async () => Uint8Array.from(stored.bytes).buffer } + : null + }, + } as unknown as R2Bucket + return { + bucket, + put, + remove: (key: string) => objects.delete(key), + } +}