diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 9426b3ce3b..5a5a5b2bdc 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,18 +1,6 @@ name: Publish @bitgo-beta on: workflow_dispatch: - inputs: - recovery-mode: - description: | - Recover a stuck beta publish where sigstore/Rekor already logged - provenance for a package's currently-computed version/tarball - (registry write failed) but the package never landed on npm. When - enabled, the publish step automatically detects the Rekor - conflict per-package and bumps that package's version until it - finds one that publishes cleanly - no manual input needed. - type: boolean - required: false - default: false # push: # branches: # - master @@ -25,12 +13,18 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} env: - SOCKET_SECURITY_MODE: monitor # Options: monitor (non-blocking) or block (fails on vulnerabilities) + SOCKET_SECURITY_MODE: monitor # Options: monitor (non-blocking) or block (fails on vulnerabilities) jobs: publish: name: Publish Release runs-on: ubuntu-latest + # Normal runs finish well inside this. The worst case stacks the lerna-step + # prepack timeout (20 min), the global propagation wait (15 min), recovery + # bumps, and the fallback publish's own sibling wait (20 min) — this cap + # covers that with headroom while still failing out of a registry outage + # instead of holding the runner for hours. + timeout-minutes: 100 # Restricts @bitgo-beta/@bitgo-alpha publishing to protected branches via # a deployment branch policy on this environment, so the job cannot run # from arbitrary PR branches even though it is manually dispatched. @@ -58,7 +52,7 @@ jobs: - name: Audit Dependencies id: osv-scan continue-on-error: true - uses: google/osv-scanner-action/osv-scanner-action@9a498708959aeaef5ef730655706c5a1df1edbc2 # v2.3.8 + uses: google/osv-scanner-action/osv-scanner-action@9a498708959aeaef5ef730655706c5a1df1edbc2 # v2.3.8 with: scan-args: |- --config=osv-scanner.toml @@ -105,15 +99,23 @@ jobs: run: git commit -am "Auto updated ${{ env.preid }} branch" --no-verify || echo "No changes to commit" - name: Lerna Publish (siblings) - # In recovery mode a stuck package's Rekor conflict fails this step outright; let it - # continue so Verify Publish below still gets a chance to bump+retry that package. - continue-on-error: ${{ inputs.recovery-mode }} + # The lerna exit code is not a reliable failure signal: a Rekor TLOG 409 is + # usually a duplicate submission of provenance whose registry write already + # completed, and npm can report failure for versions it then publishes fine. + # Verify Publish is the registry-truth gate: it polls the exact-version + # endpoint and only republishes (bumping the version on staged conflicts) + # for packages that genuinely never landed. Set RECOVERY_MODE=false in that + # script's env to restore fail-fast behavior. + continue-on-error: true run: yarn lerna publish from-package --preid ${{ env.preid }} --dist-tag ${{ env.preid }} --force-publish --yes --loglevel silly env: NPM_CONFIG_PROVENANCE: true - name: Lerna Publish (bitgo) - continue-on-error: ${{ inputs.recovery-mode }} + # Non-fatal for the same reason as the siblings step; a bitgo failure here + # (e.g. its prepack sibling-visibility wait timing out) is retried by + # Verify Publish with BITGO_GENERATE_SHRINKWRAP enabled. + continue-on-error: true run: | bitgo_name=$(jq -r '.name' modules/bitgo/package.json) yarn lerna publish from-package --include-private "$bitgo_name" --preid ${{ env.preid }} --dist-tag ${{ env.preid }} --force-publish --yes --loglevel silly @@ -125,4 +127,4 @@ jobs: run: npx tsx ./scripts/verify-release.ts ${{ env.preid }} env: NPM_CONFIG_PROVENANCE: true - RECOVERY_MODE: ${{ inputs.recovery-mode }} + RECOVERY_MODE: true diff --git a/package.json b/package.json index 21efe36d4f..536a2f54db 100644 --- a/package.json +++ b/package.json @@ -323,6 +323,7 @@ "prepare-commit-msg": "node ./scripts/prepare-commit-msg.js", "test:prepare-release": "mocha --require tsx ./scripts/tests/prepareRelease/prepare-release-main.test.ts", "test:shrinkwrap": "mocha --require tsx ./scripts/tests/generate-bitgo-shrinkwrap.test.ts", + "test:verify-release": "mocha --require tsx ./scripts/tests/verify-release.test.ts", "test:scripts": "mocha './scripts/tests/**/*.test.js'" }, "dependencies": { diff --git a/scripts/generate-bitgo-shrinkwrap.ts b/scripts/generate-bitgo-shrinkwrap.ts index a2af1bd50c..b0bc76d054 100644 --- a/scripts/generate-bitgo-shrinkwrap.ts +++ b/scripts/generate-bitgo-shrinkwrap.ts @@ -251,8 +251,8 @@ export async function ensureWorkspaceSiblingVersionsAvailable( `Workspace sibling versions are still unavailable from npm after ${checks} checks over ${formatDuration( elapsedMs )}: ` + - `${missingPackages.join(', ')}. This is likely npm registry propagation delay; ` + - 'retry the release in recovery mode after these versions become visible.' + `${missingPackages.join(', ')}. This is likely npm registry propagation delay; the release ` + + 'pipeline automatically retries publishing this package with the shrinkwrap once these versions become visible.' ); } diff --git a/scripts/prepareRelease/distTags.ts b/scripts/prepareRelease/distTags.ts index 6ef5c9c44d..3873f60c12 100644 --- a/scripts/prepareRelease/distTags.ts +++ b/scripts/prepareRelease/distTags.ts @@ -1,4 +1,5 @@ import { readFileSync, writeFileSync, existsSync } from 'fs'; +import { setTimeout as setTimeoutPromise } from 'node:timers/promises'; import path from 'path'; import { inc } from 'semver'; import { LernaModule } from './getLernaModules'; @@ -21,24 +22,16 @@ export async function getDistTags(packageName: string): Promise { return response.json(); } -export type PackageVersionExists = ( - packageName: string, - version: string, -) => Promise; +export type PackageVersionExists = (packageName: string, version: string) => Promise; -export const packageVersionExists: PackageVersionExists = async ( - packageName, - version, -) => { +export const packageVersionExists: PackageVersionExists = async (packageName, version) => { const url = `https://registry.npmjs.org/${encodeURIComponent(packageName)}/${encodeURIComponent(version)}`; const response = await fetch(url); if (response.status === 404 || response.status === 410) { return false; } if (!response.ok) { - throw new Error( - `Failed ${url}: ${response.status} ${response.statusText} ${await response.text()}`, - ); + throw new Error(`Failed ${url}: ${response.status} ${response.statusText} ${await response.text()}`); } return true; }; @@ -47,15 +40,13 @@ export async function findFirstUnpublishedVersion( packageName: string, initialCandidate: string, preid: string, - versionExists: PackageVersionExists = packageVersionExists, + versionExists: PackageVersionExists = packageVersionExists ): Promise { let candidate = initialCandidate; while (await versionExists(packageName, candidate)) { const next = inc(candidate, 'prerelease', undefined, preid); if (!next) { - throw new Error( - `Failed to increment occupied version ${packageName}@${candidate}`, - ); + throw new Error(`Failed to increment occupied version ${packageName}@${candidate}`); } candidate = next; } @@ -118,3 +109,133 @@ export async function getDistTagsForModules(modules: LernaModule[]): Promise(modules.map((m, i) => [m, nameMap.get(names[i])])); } + +const versionWaitTimeoutEnvVar = 'BITGO_VERIFY_PUBLISH_WAIT_MINUTES'; +const defaultVersionWaitTimeoutMs = 15 * 60_000; // 15 minutes +const defaultVersionPollDelayMs = 30_000; // 30 seconds +const maxConsecutiveVersionCheckErrors = 5; + +/** + * Wall-clock budget for waiting until published versions become queryable on npm. + * Registry writes can complete minutes before they are visible, and npm can hold + * failed-write versions in a staged state for far longer, so callers must never + * treat an immediate 404 after a publish as final. + */ +export function getVersionWaitTimeoutMs(env: NodeJS.ProcessEnv = process.env): number { + const raw = env[versionWaitTimeoutEnvVar]; + if (raw === undefined || raw.trim() === '') return defaultVersionWaitTimeoutMs; + const minutes = Number(raw); + if (!Number.isFinite(minutes) || minutes < 0) { + throw new Error(`${versionWaitTimeoutEnvVar} must be zero or more minutes, got "${raw}".`); + } + return minutes * 60_000; +} + +export interface PackageVersionWaitOptions { + /** Total wall-clock budget for waiting. Defaults to getVersionWaitTimeoutMs(). Zero polls once and does not wait. */ + timeoutMs?: number; + /** Delay between polls. Defaults to 30s. */ + delayMs?: number; + /** Wall-clock source in milliseconds. Injectable for tests. */ + now?: () => number; + /** Injectable for tests. */ + sleep?: (delayMs: number) => Promise; + /** Injectable for tests. */ + versionExists?: PackageVersionExists; +} + +function formatWaitDuration(ms: number): string { + if (ms < 60_000) return `${Math.round(ms / 1000)}s`; + const minutes = ms / 60_000; + return `${Number.isInteger(minutes) ? minutes : minutes.toFixed(1)} min`; +} + +/** + * Polls the registry's exact-version endpoint until every requested version is + * visible or the time budget runs out, then returns the versions still missing. + * A request whose registry checks keep failing is degraded to "not visible" + * (returned in the missing list) rather than aborting the wait for all packages. + */ +export async function waitForPackageVersions( + requests: ReadonlyArray<{ name: string; version: string }>, + options: PackageVersionWaitOptions = {} +): Promise<{ name: string; version: string }[]> { + if (requests.length === 0) return []; + + const versionExists = options.versionExists ?? packageVersionExists; + const sleepFn = options.sleep ?? setTimeoutPromise; + const now = options.now ?? Date.now; + const timeoutMs = options.timeoutMs ?? getVersionWaitTimeoutMs(); + const delayMs = options.delayMs ?? defaultVersionPollDelayMs; + if (!(timeoutMs >= 0)) { + throw new Error(`Version wait timeout must be zero or more milliseconds, got ${timeoutMs}.`); + } + if (!(delayMs > 0)) { + throw new Error(`Version wait delay must be more than zero milliseconds, got ${delayMs}.`); + } + + const startedAt = now(); + let pending: { name: string; version: string; errorStreak: number }[] = requests.map((request) => ({ + ...request, + errorStreak: 0, + })); + const erroredOut: { name: string; version: string; errorStreak: number }[] = []; + let checks = 0; + + for (;;) { + checks += 1; + console.log( + `Checking npm for ${pending.length} expected package version(s) (check ${checks}, ${formatWaitDuration( + now() - startedAt + )} elapsed).` + ); + const nextPending: { name: string; version: string; errorStreak: number }[] = []; + await Promise.all( + pending.map(async (request) => { + try { + const exists = await versionExists(request.name, request.version); + // A round that resolved (even to not-visible) breaks the error streak, + // so only consecutive failures count toward the cap. + request.errorStreak = 0; + if (!exists) nextPending.push(request); + } catch (error) { + request.errorStreak += 1; + if (request.errorStreak > maxConsecutiveVersionCheckErrors) { + // One persistently-failing URL must not abort the wait for the other + // packages; degrade it to "not visible" and let the caller's recovery + // path deal with it. + console.warn( + `Registry check for ${request.name}@${request.version} failed ${request.errorStreak} rounds in a row; treating as not visible.` + ); + erroredOut.push(request); + } else { + console.warn(`Registry check failed for ${request.name}@${request.version}, will retry: ${error}`); + nextPending.push(request); + } + } + }) + ); + pending = nextPending; + if (pending.length === 0) { + if (erroredOut.length === 0 && checks > 1) { + console.log( + `All ${requests.length} expected package versions are visible on npm after ${formatWaitDuration( + now() - startedAt + )}.` + ); + } + return erroredOut.map(({ name, version }) => ({ name, version })); + } + const elapsedMs = now() - startedAt; + if (elapsedMs + delayMs > timeoutMs) { + const missing = pending.concat(erroredOut).map(({ name, version }) => ({ name, version })); + console.warn( + `Still missing from npm after ${checks} checks over ${formatWaitDuration(elapsedMs)}: ` + + missing.map(({ name, version }) => `${name}@${version}`).join(', ') + ); + return missing; + } + console.log(`Waiting ${delayMs}ms before the next npm version check.`); + await sleepFn(delayMs); + } +} diff --git a/scripts/tests/generate-bitgo-shrinkwrap.test.ts b/scripts/tests/generate-bitgo-shrinkwrap.test.ts index 578ce19362..c699bf9ce4 100644 --- a/scripts/tests/generate-bitgo-shrinkwrap.test.ts +++ b/scripts/tests/generate-bitgo-shrinkwrap.test.ts @@ -156,7 +156,7 @@ describe('generate-bitgo-shrinkwrap workspace sibling availability', function () assert.match(error.message, /still unavailable from npm after 4 checks over 1\.5 min/); assert.match(error.message, /@bitgo-beta\/sdk-core@1\.2\.3-beta\.4/); assert.match(error.message, /@bitgo-beta\/account-lib@2\.3\.4-beta\.5/); - assert.match(error.message, /recovery mode/); + assert.match(error.message, /automatically retries publishing this package with the shrinkwrap/); return true; } ); diff --git a/scripts/tests/verify-release.test.ts b/scripts/tests/verify-release.test.ts new file mode 100644 index 0000000000..49e9442b1b --- /dev/null +++ b/scripts/tests/verify-release.test.ts @@ -0,0 +1,797 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { describe, it } from 'mocha'; + +import { waitForPackageVersions, type PackageVersionWaitOptions } from '../../scripts/prepareRelease/distTags'; +import { + type PackageJson, + type WorkspaceModule, + isRecoverablePublishConflict, + getDistTagsWithRetry, + publishWithRecovery, + propagateVersionToDependents, + readPackageJson, + settleInDependencyOrder, + topoOrder, + verifyPackage, +} from '../../scripts/verify-release'; + +/** A clock that only moves when the code under test sleeps, so waits cost no real time. */ +function fakeClock() { + let time = 0; + return { + now: () => time, + sleep: async (delayMs: number): Promise => { + time += delayMs; + }, + }; +} + +/** Runs fn while capturing console.log lines; restores the original afterwards. */ +async function captureConsoleLog(fn: () => Promise): Promise<{ lines: string[]; result: T }> { + const lines: string[] = []; + const original = console.log; + console.log = (...args: unknown[]) => { + lines.push(args.join(' ')); + }; + try { + const result = await fn(); + return { lines, result }; + } finally { + console.log = original; + } +} + +/** Creates a temp dir with a package.json and returns read/write handles on it. */ +function makePackageDir(fields: Record): { + dir: string; + read: () => Record; + write: (json: Record) => void; +} { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'verify-release-test-')); + const write = (json: Record) => { + fs.writeFileSync(path.join(dir, 'package.json'), JSON.stringify(json, null, 2) + '\n'); + }; + write(fields); + return { + dir, + read: () => JSON.parse(fs.readFileSync(path.join(dir, 'package.json'), 'utf-8')), + write, + }; +} + +describe('waitForPackageVersions', function () { + const request = { name: '@bitgo-beta/sdk-core', version: '1.2.3-beta.4' }; + + function waitOptions(overrides: Partial): PackageVersionWaitOptions { + const clock = fakeClock(); + return { timeoutMs: 60_000, delayMs: 30_000, now: clock.now, sleep: clock.sleep, ...overrides }; + } + + it('returns nothing to do for an empty request list', async function () { + const missing = await waitForPackageVersions([], waitOptions({})); + assert.deepEqual(missing, []); + }); + + it('does not wait when every version is immediately visible', async function () { + const sleeps: number[] = []; + const checked: string[] = []; + const missing = await waitForPackageVersions( + [request], + waitOptions({ + sleep: async (delayMs) => { + sleeps.push(delayMs); + }, + versionExists: async (name, version) => { + checked.push(`${name}@${version}`); + return true; + }, + }) + ); + assert.deepEqual(missing, []); + assert.deepEqual(sleeps, []); + assert.deepEqual(checked, ['@bitgo-beta/sdk-core@1.2.3-beta.4']); + }); + + it('waits for a version that appears on a later check', async function () { + let checks = 0; + const missing = await waitForPackageVersions( + [request], + waitOptions({ + versionExists: async () => { + checks += 1; + return checks >= 3; + }, + }) + ); + assert.deepEqual(missing, []); + assert.equal(checks, 3); + }); + + it('returns the versions still missing when the time budget runs out', async function () { + const other = { name: '@bitgo-beta/account-lib', version: '2.3.4-beta.5' }; + const missing = await waitForPackageVersions( + [request, other], + waitOptions({ + versionExists: async (name) => name === other.name, + }) + ); + assert.deepEqual(missing, [request]); + }); + + it('degrades persistently failing registry checks to not-visible instead of aborting', async function () { + const missing = await waitForPackageVersions( + [request], + waitOptions({ + timeoutMs: 300_000, + versionExists: async () => { + throw new Error('502 Bad Gateway'); + }, + }) + ); + assert.deepEqual(missing, [request]); + }); + + it('treats a single failing round as transient and keeps polling', async function () { + let checks = 0; + const missing = await waitForPackageVersions( + [request], + waitOptions({ + versionExists: async () => { + checks += 1; + if (checks === 1) throw new Error('502 Bad Gateway'); + return true; + }, + }) + ); + assert.deepEqual(missing, []); + assert.equal(checks, 2); + }); + + it('does not drop a package from the wait for scattered (non-consecutive) failures', async function () { + // six errors, each separated by a round that resolved (even to not-visible): + // only consecutive failures may count toward the drop-out cap + const behavior: (boolean | 'error')[] = [ + 'error', + false, + 'error', + false, + 'error', + false, + 'error', + false, + 'error', + false, + 'error', + true, + ]; + let calls = 0; + const missing = await waitForPackageVersions( + [request], + waitOptions({ + timeoutMs: 600_000, + versionExists: async () => { + const b = behavior[calls]; + calls += 1; + if (b === 'error') throw new Error('502 Bad Gateway'); + return b; + }, + }) + ); + assert.deepEqual(missing, []); + assert.equal(calls, behavior.length); + }); +}); + +describe('getDistTagsWithRetry', function () { + it('retries with backoff and succeeds once the read recovers', async function () { + const sleeps: number[] = []; + let calls = 0; + const tags = await getDistTagsWithRetry( + '@bitgo-beta/pkg', + async () => { + calls += 1; + if (calls < 3) throw new Error('502 Bad Gateway'); + return { beta: '1.0.0-beta.1' }; + }, + 3, + 5000, + async (delayMs) => { + sleeps.push(delayMs); + } + ); + assert.deepEqual(tags, { beta: '1.0.0-beta.1' }); + assert.equal(calls, 3); + assert.deepEqual(sleeps, [5000, 5000]); + }); + + it('rethrows after exhausting the attempts', async function () { + const sleeps: number[] = []; + let calls = 0; + await assert.rejects( + () => + getDistTagsWithRetry( + '@bitgo-beta/pkg', + async () => { + calls += 1; + throw new Error('502 Bad Gateway'); + }, + 3, + 5000, + async (delayMs) => { + sleeps.push(delayMs); + } + ), + /502 Bad Gateway/ + ); + assert.equal(calls, 3); + assert.deepEqual(sleeps, [5000, 5000]); + }); +}); + +describe('isRecoverablePublishConflict', function () { + it('matches the sigstore transparency-log conflict', function () { + assert.equal( + isRecoverablePublishConflict( + 'npm error TLOG_CREATE_ENTRY_ERROR error creating tlog entry - (409) an equivalent entry already exists in the transparency log' + ), + true + ); + }); + + it('matches the staged-version and publish-over-published conflicts', function () { + assert.equal( + isRecoverablePublishConflict('npm error 403 You cannot publish over the previously staged version'), + true + ); + assert.equal( + isRecoverablePublishConflict('npm error You cannot publish over the previously published versions 1.0.0'), + true + ); + }); + + it('does not match unrelated failures', function () { + assert.equal(isRecoverablePublishConflict('npm error 404 Not Found - PUT /x'), false); + assert.equal(isRecoverablePublishConflict('network timeout'), false); + }); +}); + +describe('publishWithRecovery', function () { + const preid = 'beta'; + const conflictError = (message: string) => { + const e = new Error(message) as Error & { stdout?: string }; + e.stdout = message; + return e; + }; + const tlogConflict = conflictError( + 'npm error TLOG_CREATE_ENTRY_ERROR error creating tlog entry - (409) an equivalent entry already exists in the transparency log' + ); + + function makePackageDir(version: string): { dir: string; readVersion: () => string } { + const pkg = makePkg({ name: '@bitgo-beta/sdk-test-pkg', version, private: false }); + return { + dir: pkg.dir, + readVersion: () => { + const json = pkg.read(); + return typeof json.version === 'string' ? json.version : ''; + }, + }; + } + + function makePkg(fields: Record) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'verify-release-test-')); + fs.writeFileSync(path.join(dir, 'package.json'), JSON.stringify(fields, null, 2) + '\n'); + return { + dir, + read: () => JSON.parse(fs.readFileSync(path.join(dir, 'package.json'), 'utf-8')), + }; + } + + function recoveryOptions(overrides: Partial[3]> = {}) { + const clock = fakeClock(); + return { + runPublish: async () => '', + versionExists: async () => true, + sleep: clock.sleep, + now: clock.now, + conflictPollTimeoutMs: 60_000, + ...overrides, + }; + } + + it('returns success when the publish succeeds on the first attempt', async function () { + const pkg = makePackageDir('1.0.0-beta.1'); + let publishCalls = 0; + const result = await publishWithRecovery( + pkg.dir, + readPackageJson(pkg.dir), + preid, + recoveryOptions({ + runPublish: async () => { + publishCalls += 1; + return 'published'; + }, + }) + ); + assert.equal(result.exitCode, 0); + assert.equal(publishCalls, 1); + assert.equal(pkg.readVersion(), '1.0.0-beta.1'); + }); + + it('treats a conflict as success once the version shows up on npm', async function () { + const pkg = makePackageDir('1.0.0-beta.1'); + let publishCalls = 0; + const result = await publishWithRecovery( + pkg.dir, + readPackageJson(pkg.dir), + preid, + recoveryOptions({ + runPublish: async () => { + publishCalls += 1; + throw tlogConflict; + }, + versionExists: async () => true, + }) + ); + assert.equal(result.exitCode, 0); + assert.equal(publishCalls, 1); + assert.equal(pkg.readVersion(), '1.0.0-beta.1'); + }); + + it('bumps the version and retries when the version never becomes visible', async function () { + const pkg = makePackageDir('1.0.0-beta.1'); + const publishedVersions: string[] = []; + let publishCalls = 0; + const result = await publishWithRecovery( + pkg.dir, + readPackageJson(pkg.dir), + preid, + recoveryOptions({ + runPublish: async (_cwd, tag) => { + publishCalls += 1; + if (publishCalls === 1) { + throw tlogConflict; + } + publishedVersions.push(tag); + return 'published'; + }, + versionExists: async () => false, + }) + ); + assert.equal(result.exitCode, 0); + assert.equal(publishCalls, 2); + assert.deepEqual(publishedVersions, [preid]); + assert.equal(pkg.readVersion(), '1.0.0-beta.2'); + }); + + it('skips the poll for staged-version conflicts and bumps straight away', async function () { + const pkg = makePackageDir('1.0.0-beta.1'); + let publishCalls = 0; + let versionChecks = 0; + const stagedConflict = conflictError( + 'npm error 403 You cannot publish over the previously staged version of @bitgo-beta/sdk-test-pkg' + ); + const result = await publishWithRecovery( + pkg.dir, + readPackageJson(pkg.dir), + preid, + recoveryOptions({ + runPublish: async () => { + publishCalls += 1; + if (publishCalls === 1) { + throw stagedConflict; + } + return 'published'; + }, + versionExists: async () => { + versionChecks += 1; + return false; + }, + }) + ); + assert.equal(result.exitCode, 0); + assert.equal(publishCalls, 2); + assert.equal(versionChecks, 0, 'staged conflicts must skip the visibility poll'); + assert.equal(pkg.readVersion(), '1.0.0-beta.2'); + }); + + it('bumps immediately on later attempts without polling again', async function () { + const pkg = makePackageDir('1.0.0-beta.1'); + let publishCalls = 0; + let versionChecks = 0; + await assert.rejects( + () => + publishWithRecovery( + pkg.dir, + readPackageJson(pkg.dir), + preid, + recoveryOptions({ + runPublish: async () => { + publishCalls += 1; + throw tlogConflict; + }, + versionExists: async () => { + versionChecks += 1; + return false; + }, + }) + ), + /still hitting a publish conflict after 5 version bumps/ + ); + // only the first attempt polls (60s budget / 30s delay = 3 checks); a fresh + // bumped version cannot hit an equivalent-entry false negative, so later + // attempts bump straight away + assert.equal(versionChecks, 3); + assert.equal(publishCalls, 5); + }); + + it('gives up after the maximum number of bumped attempts', async function () { + const pkg = makePackageDir('1.0.0-beta.1'); + const clock = fakeClock(); + let publishCalls = 0; + await assert.rejects( + () => + publishWithRecovery( + pkg.dir, + readPackageJson(pkg.dir), + preid, + recoveryOptions({ + runPublish: async () => { + publishCalls += 1; + throw tlogConflict; + }, + versionExists: async () => false, + sleep: clock.sleep, + now: clock.now, + }) + ), + /still hitting a publish conflict after 5 version bumps/ + ); + assert.equal(publishCalls, 5); + assert.equal(pkg.readVersion(), '1.0.0-beta.5'); + }); + + it('throws immediately on non-conflict failures', async function () { + const pkg = makePackageDir('1.0.0-beta.1'); + let publishCalls = 0; + await assert.rejects( + () => + publishWithRecovery( + pkg.dir, + readPackageJson(pkg.dir), + preid, + recoveryOptions({ + runPublish: async () => { + publishCalls += 1; + throw conflictError('npm error 404 Not Found'); + }, + }) + ), + /404 Not Found/ + ); + assert.equal(publishCalls, 1); + assert.equal(pkg.readVersion(), '1.0.0-beta.1'); + }); +}); + +describe('topoOrder', function () { + function module( + dir: string, + name: string, + dependencies: Record, + privateField = false + ): WorkspaceModule { + return { + dir, + json: { name, version: '1.0.0-beta.1', private: privateField, dependencies }, + }; + } + + it('orders dependencies first with bitgo last', function () { + const modules = [ + module('/w/bitgo', '@bitgo-beta/bitgo', { + '@bitgo-beta/sdk-lib-a': '1.0.0-beta.1', + '@bitgo-beta/sdk-coin-b': '1.0.0-beta.1', + }), + module('/w/sdk-coin-b', '@bitgo-beta/sdk-coin-b', { '@bitgo-beta/sdk-lib-a': '1.0.0-beta.1' }), + module('/w/sdk-lib-a', '@bitgo-beta/sdk-lib-a', {}), + ]; + const order = topoOrder(modules).map((m) => m.json.name); + assert.deepEqual(order, ['@bitgo-beta/sdk-lib-a', '@bitgo-beta/sdk-coin-b', '@bitgo-beta/bitgo']); + }); + + it('excludes unpublished private modules but keeps bitgo', function () { + const modules = [ + module('/w/bitgo', '@bitgo-beta/bitgo', { '@bitgo-beta/sdk-lib-a': '1.0.0-beta.1' }, true), + module('/w/sdk-test', '@bitgo-beta/sdk-test', { '@bitgo-beta/sdk-lib-a': '1.0.0-beta.1' }, true), + module('/w/sdk-lib-a', '@bitgo-beta/sdk-lib-a', {}), + ]; + const order = topoOrder(modules).map((m) => m.json.name); + assert.deepEqual(order, ['@bitgo-beta/sdk-lib-a', '@bitgo-beta/bitgo']); + }); + + it('throws on a dependency cycle', function () { + const modules = [module('/w/a', '@x/a', { '@x/b': '1.0.0' }), module('/w/b', '@x/b', { '@x/a': '1.0.0' })]; + assert.throws(() => topoOrder(modules), /cycle/); + }); +}); + +describe('propagateVersionToDependents', function () { + it('updates prod and dev pins but reports only prod-pin changers', function () { + const prodDep = makePackageDir({ + name: '@bitgo-beta/sdk-coin-prod', + version: '1.0.0-beta.1', + dependencies: { '@bitgo-beta/sdk-lib-x': '1.0.0-beta.1' }, + }); + const devDep = makePackageDir({ + name: '@bitgo-beta/sdk-coin-dev', + version: '1.0.0-beta.1', + devDependencies: { '@bitgo-beta/sdk-lib-x': '1.0.0-beta.1' }, + }); + const unrelated = makePackageDir({ name: '@bitgo-beta/sdk-coin-y', version: '1.0.0-beta.1' }); + const modules: WorkspaceModule[] = [prodDep, devDep, unrelated].map((pkg) => ({ + dir: pkg.dir, + json: readPackageJson(pkg.dir), + })); + + const changed = propagateVersionToDependents(modules, '@bitgo-beta/sdk-lib-x', '1.0.0-beta.2'); + + assert.deepEqual(changed, ['@bitgo-beta/sdk-coin-prod']); + assert.equal(readPackageJson(prodDep.dir).dependencies?.['@bitgo-beta/sdk-lib-x'], '1.0.0-beta.2'); + assert.equal(readPackageJson(devDep.dir).devDependencies?.['@bitgo-beta/sdk-lib-x'], '1.0.0-beta.2'); + assert.equal(unrelated.read().dependencies, undefined); + }); + + it('does not mark a module dirty when only its dev pin changed', function () { + const libX = makePackageDir({ name: '@bitgo-beta/sdk-lib-x2', version: '1.0.0-beta.1' }); + const bothPins = makePackageDir({ + name: '@bitgo-beta/sdk-coin-both', + version: '1.0.0-beta.1', + dependencies: { '@bitgo-beta/sdk-lib-x2': '1.0.0-beta.2' }, + devDependencies: { '@bitgo-beta/sdk-lib-x2': '1.0.0-beta.1' }, + }); + const modules: WorkspaceModule[] = [libX, bothPins].map((pkg) => ({ + dir: pkg.dir, + json: readPackageJson(pkg.dir), + })); + + const changed = propagateVersionToDependents(modules, '@bitgo-beta/sdk-lib-x2', '1.0.0-beta.2'); + + // the prod pin already pointed at the new version, so only the dev pin was + // rewritten — the module must not be flagged for a redundant republish + assert.deepEqual(changed, []); + assert.equal(readPackageJson(bothPins.dir).devDependencies?.['@bitgo-beta/sdk-lib-x2'], '1.0.0-beta.2'); + }); +}); + +describe('verifyPackage', function () { + function makeTarget(version: string) { + return makePackageDir({ name: '@bitgo-beta/sdk-verify-target', version, private: false }); + } + + it('reports success without publishing when the version is on npm and pins are clean', async function () { + const pkg = makeTarget('1.0.0-beta.1'); + let publishCalls = 0; + const { lines, result } = await captureConsoleLog(() => + verifyPackage(pkg.dir, 'beta', false, { + versionExists: async () => true, + fetchDistTags: async () => ({ beta: '1.0.0-beta.1' }), + runPublish: async () => { + publishCalls += 1; + return 'published'; + }, + }) + ); + assert.equal(result, true); + assert.equal(publishCalls, 0); + assert.ok(lines.includes('@bitgo-beta/sdk-verify-target matches expected version 1.0.0-beta.1')); + }); + + it('bumps and republishes a published package whose pins went dirty', async function () { + const pkg = makeTarget('1.0.0-beta.1'); + let publishCalls = 0; + const { lines, result } = await captureConsoleLog(() => + verifyPackage(pkg.dir, 'beta', true, { + versionExists: async () => true, + fetchDistTags: async () => ({ beta: '1.0.0-beta.1' }), + runPublish: async () => { + publishCalls += 1; + return 'published'; + }, + }) + ); + assert.equal(result, true); + assert.equal(publishCalls, 1); + // canonical line must carry the FINAL (bumped) version for beta-tools + assert.ok(lines.includes('@bitgo-beta/sdk-verify-target matches expected version 1.0.0-beta.2')); + assert.equal(pkg.read().version, '1.0.0-beta.2'); + }); + + it('bumps a dirty package even when the visibility read fails or lags', async function () { + // regression: a failed or false visibility read used to skip the dirty + // bump, republishing the stale version itself — the conflict poll then + // saw it as published and the run went green shipping stale pins + const pkg = makeTarget('1.0.0-beta.1'); + let publishCalls = 0; + const { lines, result } = await captureConsoleLog(() => + verifyPackage(pkg.dir, 'beta', true, { + versionExists: async () => { + throw new Error('502 Bad Gateway'); + }, + fetchDistTags: async () => ({ beta: '1.0.0-beta.1' }), + runPublish: async () => { + publishCalls += 1; + return 'published'; + }, + }) + ); + assert.equal(result, true); + assert.equal(publishCalls, 1); + assert.ok(lines.includes('@bitgo-beta/sdk-verify-target matches expected version 1.0.0-beta.2')); + assert.equal(pkg.read().version, '1.0.0-beta.2'); + }); + + it('publishes a missing package with a diagnostic', async function () { + const pkg = makeTarget('1.0.0-beta.1'); + let publishCalls = 0; + const { lines, result } = await captureConsoleLog(() => + verifyPackage(pkg.dir, 'beta', false, { + versionExists: async () => false, + fetchDistTags: async () => ({ beta: '0.9.9-beta.1' }), + runPublish: async () => { + publishCalls += 1; + return 'published'; + }, + }) + ); + assert.equal(result, true); + assert.equal(publishCalls, 1); + assert.ok(lines.some((l) => l.includes('missing from npm'))); + assert.ok(lines.includes('@bitgo-beta/sdk-verify-target matches expected version 1.0.0-beta.1')); + }); + + it('skips unpublished private modules without any registry access', async function () { + const pkg = makePackageDir({ name: '@bitgo-beta/sdk-test', version: '1.0.0-beta.1', private: true }); + let versionChecks = 0; + const result = await verifyPackage(pkg.dir, 'beta', false, { + versionExists: async () => { + versionChecks += 1; + return false; + }, + }); + assert.equal(result, true); + assert.equal(versionChecks, 0); + }); +}); + +describe('settleInDependencyOrder', function () { + function makeChain(): { libB: string; coinA: string; bitgo: string } { + const libB = makePackageDir({ name: '@bitgo-beta/sdk-lib-b', version: '1.0.0-beta.1' }); + const coinA = makePackageDir({ + name: '@bitgo-beta/sdk-coin-a', + version: '1.0.0-beta.1', + dependencies: { '@bitgo-beta/sdk-lib-b': '1.0.0-beta.1' }, + }); + const bitgo = makePackageDir({ + name: '@bitgo-beta/bitgo', + version: '1.0.0-beta.1', + dependencies: { + '@bitgo-beta/sdk-lib-b': '1.0.0-beta.1', + '@bitgo-beta/sdk-coin-a': '1.0.0-beta.1', + }, + }); + return { libB: libB.dir, coinA: coinA.dir, bitgo: bitgo.dir }; + } + + it('settles a staged-conflict chain in one ordered pass', async function () { + const chain = makeChain(); + const modules: WorkspaceModule[] = [chain.bitgo, chain.coinA, chain.libB].map((dir) => ({ + dir, + json: readPackageJson(dir), + })); + const calls: Array<{ dir: string; pinsDirty: boolean }> = []; + const bumpTo = (dir: string, version: string) => { + const json = readPackageJson(dir); + json.version = version; + fs.writeFileSync(path.join(dir, 'package.json'), JSON.stringify(json, null, 2) + '\n'); + }; + + const anyFailed = await settleInDependencyOrder(modules, 'beta', { + verifyPackage: async (dir, _preid, pinsDirty) => { + calls.push({ dir, pinsDirty }); + // libB hits a staged-version conflict in its turn and recovers via a + // bump; the dependents are then dirty and republish with final pins + if (dir === chain.libB) { + bumpTo(dir, '1.0.0-beta.2'); + } else if (dir === chain.coinA) { + assert.equal(pinsDirty, true, 'coinA must be dirty after libB settled'); + bumpTo(dir, '1.0.0-beta.2'); + } else if (dir === chain.bitgo) { + assert.equal(pinsDirty, true, 'bitgo must be dirty after coinA settled'); + bumpTo(dir, '1.0.0-beta.2'); + } + return true; + }, + }); + + assert.equal(anyFailed, false); + // dependency-first order with bitgo last + assert.deepEqual( + calls.map((c) => c.dir), + [chain.libB, chain.coinA, chain.bitgo] + ); + assert.deepEqual( + calls.map((c) => c.pinsDirty), + [false, true, true] + ); + // every dependent's pins were propagated before its own turn + assert.equal(readPackageJson(chain.coinA).dependencies?.['@bitgo-beta/sdk-lib-b'], '1.0.0-beta.2'); + const bitgoDeps = readPackageJson(chain.bitgo).dependencies; + assert.equal(bitgoDeps?.['@bitgo-beta/sdk-lib-b'], '1.0.0-beta.2'); + assert.equal(bitgoDeps?.['@bitgo-beta/sdk-coin-a'], '1.0.0-beta.2'); + }); + + it('stops at the first failed package in fail-fast mode', async function () { + const chain = makeChain(); + const modules: WorkspaceModule[] = [chain.bitgo, chain.coinA, chain.libB].map((dir) => ({ + dir, + json: readPackageJson(dir), + })); + const calls: string[] = []; + const anyFailed = await settleInDependencyOrder(modules, 'beta', { + failFast: true, + verifyPackage: async (dir) => { + calls.push(dir); + return false; + }, + }); + assert.equal(anyFailed, true); + // only the deepest dependency was attempted before aborting + assert.deepEqual(calls, [chain.libB]); + }); + + it('blocks dependents of a failed package instead of publishing broken pins', async function () { + // npm is append-only: a red run must not publish a dependent pinning a + // version that never settled, or the broken artifact stays live after the run + const chain = makeChain(); + const modules: WorkspaceModule[] = [chain.bitgo, chain.coinA, chain.libB].map((dir) => ({ + dir, + json: readPackageJson(dir), + })); + const calls: string[] = []; + const anyFailed = await settleInDependencyOrder(modules, 'beta', { + verifyPackage: async (dir) => { + calls.push(dir); + // the deepest dependency fails to publish; its dependents must never be + // attempted, transitively up to bitgo + return dir !== chain.libB; + }, + }); + assert.equal(anyFailed, true); + assert.deepEqual(calls, [chain.libB]); + // nothing was published downstream, so no dependent pins an unsettled version + assert.equal(readPackageJson(chain.coinA).version, '1.0.0-beta.1'); + assert.equal(readPackageJson(chain.bitgo).version, '1.0.0-beta.1'); + }); + + it('blocks transitively through an intermediate dependent', async function () { + const chain = makeChain(); + const modules: WorkspaceModule[] = [chain.bitgo, chain.coinA, chain.libB].map((dir) => ({ + dir, + json: readPackageJson(dir), + })); + const calls: string[] = []; + const anyFailed = await settleInDependencyOrder(modules, 'beta', { + verifyPackage: async (dir) => { + calls.push(dir); + // coinA fails after libB succeeded; bitgo (depends on coinA) must be skipped + return dir !== chain.coinA; + }, + }); + assert.equal(anyFailed, true); + assert.deepEqual(calls, [chain.libB, chain.coinA]); + assert.equal(readPackageJson(chain.bitgo).version, '1.0.0-beta.1'); + }); +}); diff --git a/scripts/verify-release.ts b/scripts/verify-release.ts index 60a3f39d77..5227bd52d1 100644 --- a/scripts/verify-release.ts +++ b/scripts/verify-release.ts @@ -2,61 +2,184 @@ import assert from 'node:assert'; import execa from 'execa'; import { readFileSync, writeFileSync } from 'fs'; import path from 'path'; +import { setTimeout as setTimeoutPromise } from 'node:timers/promises'; import { inc } from 'semver'; -import { getLernaModules, getDistTags } from './prepareRelease'; +import { + getDistTags, + getLernaModules, + setDependencyVersion, + packageVersionExists, + waitForPackageVersions, + PackageVersionExists, +} from './prepareRelease'; + +export interface PackageJson { + dependencies?: Record; + devDependencies?: Record; + [key: string]: unknown; + name: string; + version: string; + // Some workspace packages (e.g. modules/sdk-test) declare "private": "true" as + // a string; npm treats any truthy value as private, so both forms are valid. + private?: boolean | string; +} + +function isPackageJson(value: unknown): value is PackageJson { + if (typeof value !== 'object' || value === null || Array.isArray(value)) return false; + if (!('name' in value) || typeof value.name !== 'string') return false; + if (!('version' in value) || typeof value.version !== 'string') return false; + if ('private' in value && typeof value.private !== 'boolean' && typeof value.private !== 'string') return false; + if ('dependencies' in value && (typeof value.dependencies !== 'object' || value.dependencies === null)) return false; + if ('devDependencies' in value && (typeof value.devDependencies !== 'object' || value.devDependencies === null)) + return false; + return true; +} + +export function readPackageJson(dir: string): PackageJson { + const parsed: unknown = JSON.parse(readFileSync(path.join(dir, 'package.json'), { encoding: 'utf-8' })); + if (!isPackageJson(parsed)) { + throw new Error(`${dir}/package.json is missing string "name"/"version" fields`); + } + return parsed; +} + +export interface WorkspaceModule { + dir: string; + json: PackageJson; +} + +/** + * Private modules are never published, so their versions never settle and + * dependents' pins on them cannot go stale — they are excluded from the + * release-verification ordering. bitgo is private but published, so it is + * always included. + */ +function isUnpublishedModule(module: { dir: string; json: PackageJson }): boolean { + return Boolean(module.json.private) && path.basename(module.dir) !== 'bitgo'; +} let lernaModuleLocations: string[] = []; -const RECOVERY_MODE = process.env.RECOVERY_MODE === 'true'; +// Bump-on-conflict is the default posture: a Rekor/staged conflict does not mean +// the registry write failed, and dist-tags lag minutes behind completed writes. +// Set RECOVERY_MODE=false to restore fail-fast behavior. +const RECOVERY_MODE = process.env.RECOVERY_MODE !== 'false'; const MAX_RECOVERY_ATTEMPTS = 5; +// After a recoverable publish conflict, poll this long for the version to appear +// on npm before bumping it — the write usually completed despite the client error. +const CONFLICT_POLL_TIMEOUT_MS = 5 * 60_000; +const VERSION_POLL_DELAY_MS = 30_000; async function getLernaModuleLocations(): Promise { const modules = await getLernaModules(); lernaModuleLocations = modules.map(({ location }) => location); } -function isRecoverablePublishConflict(output: string): boolean { +export function isRecoverablePublishConflict(output: string): boolean { return ( output.includes('TLOG_CREATE_ENTRY_ERROR') || output.includes('previously staged version') || + output.includes('publish over the previously published version') || (output.includes('409 Conflict') && output.includes('transparency log')) || (output.includes('(409)') && output.includes('transparency log')) ); } -// retries a stuck package's publish, bumping its version each time to sidestep an already-logged Rekor entry for that exact tarball digest -async function publishWithRecovery( +export interface PublishRecoveryOptions { + /** Publishes the package in `cwd` with the given dist-tag. Injectable for tests; defaults to `npm publish --provenance`. */ + runPublish?: (cwd: string, preid: string) => Promise; + /** Injectable for tests. */ + versionExists?: PackageVersionExists; + /** Injectable for tests. */ + sleep?: (delayMs: number) => Promise; + /** Injectable for tests. */ + now?: () => number; + /** Poll budget after a conflict before bumping. Injectable for tests. */ + conflictPollTimeoutMs?: number; +} + +// npm publish failures carry the CLI output on execa errors; join stdout/stderr +// for pattern matching. +function publishErrorOutput(e: unknown): string { + const parts: string[] = []; + if (typeof e === 'object' && e !== null) { + if ('stdout' in e && typeof e.stdout === 'string') parts.push(e.stdout); + if ('stderr' in e && typeof e.stderr === 'string') parts.push(e.stderr); + } + return parts.join('\n'); +} + +const defaultRunPublish = async (cwd: string, preid: string): Promise => { + const env: NodeJS.ProcessEnv = { ...process.env }; + // bitgo's prepack only generates npm-shrinkwrap.json when asked; the fallback + // publish must ship it exactly like the lerna pass does. Its prepack keeps + // the FULL default sibling wait: the settle pass may have bumped and + // republished siblings seconds before this publish, and the earlier global + // propagation wait only covered the original versions. When everything is + // already visible the prepack check passes in one round, so the budget is + // only spent when it is genuinely needed. + if (path.basename(cwd) === 'bitgo') { + env.BITGO_GENERATE_SHRINKWRAP = 'true'; + } + const result = await execa('npm', ['publish', '--tag', preid, '--provenance'], { cwd, env }); + return result.stdout; +}; + +// Recovers a stuck package after a publish conflict: polls the registry on the +// first attempt in case the write completed despite the client error, then +// bumps the version — a fresh version has no Rekor entry and no staged +// conflict, so it publishes cleanly. +export async function publishWithRecovery( cwd: string, - json: any, + json: PackageJson, preid: string, + options: PublishRecoveryOptions = {} ): Promise<{ stdout: string; exitCode: number }> { + const runPublish = options.runPublish ?? defaultRunPublish; + const versionExists = options.versionExists ?? packageVersionExists; + const sleepFn = options.sleep ?? setTimeoutPromise; + const now = options.now ?? Date.now; + const conflictPollTimeoutMs = options.conflictPollTimeoutMs ?? CONFLICT_POLL_TIMEOUT_MS; const maxAttempts = RECOVERY_MODE ? MAX_RECOVERY_ATTEMPTS : 1; for (let attempt = 1; attempt <= maxAttempts; attempt++) { try { - return await execa( - 'npm', - ['publish', '--tag', preid, '--provenance'], - { cwd }, - ); - } catch (e: any) { - const output = `${e.stdout ?? ''}\n${e.stderr ?? ''}`; + const stdout = await runPublish(cwd, preid); + return { stdout, exitCode: 0 }; + } catch (e) { + const output = publishErrorOutput(e); if (!RECOVERY_MODE || !isRecoverablePublishConflict(output)) { throw e; } + // Poll only on the first attempt: a bumped version is a fresh tarball, so + // a conflict on it cannot be an equivalent-entry false negative — bump + // immediately instead. A staged-version conflict also skips polling: npm + // holds the version, so polling can never make it visible. + if (attempt === 1 && !output.includes('previously staged version')) { + const missing = await waitForPackageVersions([{ name: json.name, version: json.version }], { + timeoutMs: conflictPollTimeoutMs, + delayMs: VERSION_POLL_DELAY_MS, + versionExists, + sleep: sleepFn, + now, + }); + if (missing.length === 0) { + console.log( + `${json.name}: ${json.version} is visible on npm after the publish conflict — treating the publish as complete.` + ); + return { stdout: output, exitCode: 0 }; + } + } if (attempt === maxAttempts) { - throw new Error( - `${json.name}: still hitting a publish conflict after ${maxAttempts} version bumps`, - ); + throw new Error(`${json.name}: still hitting a publish conflict after ${maxAttempts} version bumps`); } const next = inc(json.version, 'prerelease', undefined, preid); assert(typeof next === 'string', `Failed to increment version for ${json.name}`); json.version = next; - writeFileSync( - path.join(cwd, 'package.json'), - JSON.stringify(json, null, 2) + '\n', - ); + writeFileSync(path.join(cwd, 'package.json'), JSON.stringify(json, null, 2) + '\n'); console.warn( - `${json.name}: publish conflict, retrying with bumped version ${json.version} (attempt ${attempt + 1}/${maxAttempts})`, + `${json.name}: publish conflict, retrying with bumped version ${json.version} (attempt ${ + attempt + 1 + }/${maxAttempts})` ); } } @@ -64,54 +187,322 @@ async function publishWithRecovery( throw new Error(`${json.name}: exhausted publish attempts`); } -async function verifyPackage(dir: string, preid = 'beta'): Promise { +// Dist-tags are flaky at exactly the moments we care about (right after +// publishes, and for never-published names npm returns 401 instead of 404). A +// few retries before falling back keeps one bad read from failing the release. +export async function getDistTagsWithRetry( + packageName: string, + fetcher: (name: string) => Promise> = getDistTags, + attempts = 3, + delayMs = 5000, + sleep: (delayMs: number) => Promise = setTimeoutPromise +): Promise> { + for (let attempt = 1; attempt <= attempts; attempt++) { + try { + return await fetcher(packageName); + } catch (e) { + if (attempt === attempts) { + throw e; + } + console.warn(`Dist-tags fetch for ${packageName} failed (attempt ${attempt}/${attempts}), retrying...`); + await sleep(delayMs); + } + } + // unreachable: the loop above always returns or throws + throw new Error(`unreachable: dist-tags fetch loop for ${packageName}`); +} + +export interface VerifyPackageOptions { + versionExists?: PackageVersionExists; + fetchDistTags?: (name: string) => Promise>; + runPublish?: (cwd: string, preid: string) => Promise; +} + +// Verifies one package against registry truth and, when needed, (re)publishes +// it with the local — already final — pins: +// - on npm, pins unchanged: report success (with a note if the dist-tag lags) +// - on npm, pins changed since its publish (a dependency settled later in this +// run): npm cannot change a published manifest, so bump and republish +// - missing from npm: publish (conflict-tolerant recovery included) +// Every success path prints the canonical " matches expected version +// " line that beta-tools' bump-bitgo-beta parses from run logs. +export async function verifyPackage( + dir: string, + preid = 'beta', + pinsDirty = false, + options: VerifyPackageOptions = {} +): Promise { const cwd = dir; - const json = JSON.parse( - readFileSync(path.join(cwd, 'package.json'), { encoding: 'utf-8' }), - ); - // bitgo is private but still published via lerna --include-private. - if (json.private && path.basename(cwd) !== 'bitgo') { + const json = readPackageJson(cwd); + if (isUnpublishedModule({ dir, json })) { return true; } + const versionExists = options.versionExists ?? packageVersionExists; + const fetchDistTags = options.fetchDistTags ?? getDistTags; + const runPublish = options.runPublish ?? defaultRunPublish; - try { - const distTags = await getDistTags(json.name); - if (json.version !== distTags[preid]) { - console.log( - `${json.name} missing. Expected ${json.version}, latest is ${distTags[preid]}`, - ); - const { stdout, exitCode } = await publishWithRecovery(cwd, json, preid); + const reportSuccess = () => { + console.log(`${json.name} matches expected version ${json.version}`); + return true; + }; + const attemptPublish = async (): Promise => { + try { + const { stdout, exitCode } = await publishWithRecovery(cwd, json, preid, { runPublish }); console.log(stdout); - return exitCode === 0; - } else { - console.log(`${json.name} matches expected version ${json.version}`); + if (exitCode !== 0) { + return false; + } + return reportSuccess(); + } catch (e) { + console.error(`${json.name}: recovery publish failed: ${e instanceof Error ? e.message : e}`); + return false; } - return true; + }; + + if (pinsDirty) { + const next = inc(json.version, 'prerelease', undefined, preid); + assert(typeof next === 'string', `Failed to increment version for ${json.name}`); + json.version = next; + writeFileSync(path.join(cwd, 'package.json'), JSON.stringify(json, null, 2) + '\n'); + console.warn( + `${json.name}: published pins are stale (a dependency settled after its publish); ` + + `bumping to ${next} and republishing with the final pins.` + ); + // npm cannot change a published manifest, so a new version is the only + // fix — regardless of what a visibility read would say. A failed or + // lagging read must never make us republish the stale version itself: + // the republish would be rejected with publish-over-published, the + // conflict poll would then see the old version as visible, and the run + // would go green with the stale pins still shipped. + return attemptPublish(); + } + + // Registry truth: the exact-version endpoint. A version published moments + // ago can be absent from dist-tags while already queryable here. A read + // failure is treated as not-visible so the publish attempt still runs. + let visibleOnRegistry = false; + try { + visibleOnRegistry = await versionExists(json.name, json.version); } catch (e) { - console.warn(`Failed to fetch dist tags for ${json.name}`, e); - return false; + console.warn(`Registry check failed for ${json.name}@${json.version}; falling through to dist-tags`, e); + } + + if (visibleOnRegistry) { + // One non-fatal dist-tags read for the log: if the tag still lags, npm + // will catch up on its own — republishing cannot move it. + try { + const distTags = await fetchDistTags(json.name); + if (distTags[preid] !== json.version) { + console.log( + `${json.name}@${json.version} is on npm but the "${preid}" dist-tag still points at ` + + `${distTags[preid] ?? '(none)'}; npm propagation will catch up.` + ); + } + } catch (e) { + console.warn(`Dist-tags read for ${json.name} failed after publish; ignoring`, e); + } + return reportSuccess(); + } + + try { + const distTags = await getDistTagsWithRetry(json.name, fetchDistTags); + console.log( + `${json.name} missing from npm. Expected ${json.version}, dist-tag "${preid}" is ${distTags[preid] ?? '(none)'}` + ); + } catch (e) { + console.warn(`Failed to fetch dist tags for ${json.name}; proceeding with recovery publish`, e); } + return attemptPublish(); } -async function verify(preid?: string) { - await getLernaModuleLocations(); +/** + * Orders workspace modules dependencies-first (Kahn's algorithm) over + * production-dependency edges, so a package is only ever verified and published + * after every dependency's version has settled. bitgo depends on the whole + * workspace and nothing depends on it, so it lands last. Throws on a dependency + * cycle, which a publishable workspace must not contain. + */ +export function topoOrder(modules: ReadonlyArray): WorkspaceModule[] { + const targets = modules.filter((m) => !isUnpublishedModule(m)); + const byName: Record = {}; + for (const m of targets) { + byName[m.json.name] = m; + } + const targetNames = new Set(targets.map((m) => m.json.name)); + const remainingDeps: Record = {}; + const dependents: Record = {}; + for (const m of targets) { + const deps = Object.keys(m.json.dependencies ?? {}).filter((dep) => targetNames.has(dep)); + remainingDeps[m.json.name] = deps.length; + for (const dep of deps) { + (dependents[dep] ??= []).push(m.json.name); + } + } + const ready = targets.filter((m) => remainingDeps[m.json.name] === 0).map((m) => m.json.name); + const ordered: WorkspaceModule[] = []; + while (ready.length > 0) { + const name = ready.shift(); + if (name === undefined) break; + const module = byName[name]; + if (module === undefined) continue; + ordered.push(module); + for (const dependent of dependents[name] ?? []) { + remainingDeps[dependent] -= 1; + if (remainingDeps[dependent] === 0) { + ready.push(dependent); + } + } + } + if (ordered.length !== targets.length) { + throw new Error( + 'Workspace dependency cycle detected among publishable modules; cannot order release verification.' + ); + } + return ordered; +} + +/** + * Writes `name@version` into every module's dependency pins (prod and dev) and + * returns the names of modules whose PRODUCTION pin changed. Those modules' + * intended pins now diverge from whatever lerna published earlier in this run, + * so their published manifest is stale and they must be bumped and republished + * when their turn comes. dev-only pin changes are written for repo hygiene but + * never mark a module dirty: consumers do not install devDependencies. + */ +export function propagateVersionToDependents( + modules: ReadonlyArray, + name: string, + version: string +): string[] { + const changed: string[] = []; + for (const mod of modules) { + if (mod.json.name === name) continue; + const json = readPackageJson(mod.dir); + const before = JSON.stringify(json, null, 2) + '\n'; + const prodPinBefore = json.dependencies?.[name]; + setDependencyVersion(json, name, version); + const after = JSON.stringify(json, null, 2) + '\n'; + if (after !== before) { + writeFileSync(path.join(mod.dir, 'package.json'), after); + // Only a PRODUCTION pin that actually moved marks the module dirty — a + // dev-only pin update (e.g. the prod pin already pointed at the new + // version) must not trigger a redundant bump+republish. + if (prodPinBefore !== undefined && prodPinBefore !== version) { + changed.push(json.name); + } + } + } + return changed; +} + +export interface SettleOptions { + /** Verifies (and if needed, bump-) publishes one package. Injectable for tests. */ + verifyPackage: (dir: string, preid: string, pinsDirty: boolean) => Promise; + /** Injectable for tests. */ + propagate?: typeof propagateVersionToDependents; + /** Stop at the first failed package (non-recovery mode) instead of continuing. */ + failFast?: boolean; +} + +/** + * The heart of the release fix: verify (and publish where needed) every + * module in dependency order, so each package's pins are final before its + * turn and exactly one pass converges. When a module's version settles + * (staged-conflict recovery bump, or a stale-pin republish), the final version + * is propagated into every dependent's local pins and those dependents are + * marked dirty; a dependent that was already published then gets one + * bump+republish at its own turn. In dependency order a dependent is never + * processed before its dependencies settle, so dirtyPins can only ever mark + * modules that have not been processed yet. + * + * Failure safety: npm is append-only, so a failed publish must block every + * dependent's publish too — nothing ships pinning a version that never + * settled, even though the run is already red. The block is transitive through + * the same topological order. + * + * Returns true when any package failed to verify/publish. + */ +export async function settleInDependencyOrder( + modules: ReadonlyArray, + preid: string, + options: SettleOptions +): Promise { + const settleOrder = topoOrder(modules); + // Invariant: lerna published the LOCAL state, so at verify entry every + // published manifest matches its local package.json. dirtyPins therefore + // starts EMPTY — do not pre-populate it; only pin changes made during this + // run mark a module as needing a bump+republish. + const dirtyPins: Record = {}; + // Packages that failed to settle, or were skipped because a dependency of + // theirs failed. npm is append-only: even on an already-red run, publishing + // a dependent that pins an unsettled version would ship a permanently + // broken artifact and move the dist-tag to it, so failures must block every + // downstream publish. Topological order makes the block transitive for + // free: a skipped module is itself marked blocked before its dependents' + // turns arrive. + const blocked: Record = {}; + let anyFailed = false; - for (let i = 0; i < lernaModuleLocations.length; i++) { - const dir = lernaModuleLocations[i]; - if (!(await verifyPackage(dir, preid))) { - console.error('Failed to verify outstanding packages.'); - if (!RECOVERY_MODE) { - process.exitCode = 1; - return; + for (const mod of settleOrder) { + const before = readPackageJson(mod.dir); + const blockingDependency = Object.keys(before.dependencies ?? {}).find((dep) => blocked[dep] === true); + if (blockingDependency !== undefined) { + console.error( + `${before.name}: skipping publish — dependency ${blockingDependency} did not settle; ` + + 'not shipping a version that pins it.' + ); + blocked[before.name] = true; + anyFailed = true; + continue; + } + const ok = await options.verifyPackage(mod.dir, preid, dirtyPins[before.name] === true); + const after = readPackageJson(mod.dir); + if (ok && after.version !== before.version) { + for (const changed of (options.propagate ?? propagateVersionToDependents)(modules, after.name, after.version)) { + dirtyPins[changed] = true; + } + } + if (!ok) { + console.error(`Failed to verify ${after.name}.`); + blocked[after.name] = true; + if (options.failFast) { + return true; } - // keep bumping/retrying the rest of the packages instead of aborting on the first stuck one + // keep publishing the rest of the packages instead of aborting on the first stuck one anyFailed = true; } } + return anyFailed; +} + +async function verify(preid?: string) { + await getLernaModuleLocations(); + const modules: WorkspaceModule[] = lernaModuleLocations.map((dir) => ({ dir, json: readPackageJson(dir) })); + const expected = modules + .filter((m) => !isUnpublishedModule(m)) + .map((m) => ({ name: m.json.name, version: m.json.version })); + + // One bounded wait for the whole release: a successful publish can stay + // invisible on npm for minutes (occasionally much longer when npm stages a + // version), so wait for propagation once here instead of racing it + // package-by-package below. + const stillMissing = await waitForPackageVersions(expected); + if (stillMissing.length > 0) { + console.warn( + `${stillMissing.length} expected version(s) did not become visible within the propagation window; ` + + 'attempting recovery publish for those below.' + ); + } + const anyFailed = await settleInDependencyOrder(modules, preid ?? 'beta', { + verifyPackage: (dir, preidArg, pinsDirty) => verifyPackage(dir, preidArg, pinsDirty), + failFast: !RECOVERY_MODE, + }); if (anyFailed) { process.exitCode = 1; } } // e.g. for alpha releases: `npx tsx ./scripts/verify-release.ts alpha` -verify(process.argv.slice(2)[0]); +if (require.main === module) { + verify(process.argv.slice(2)[0]); +}