From 2df79d700f250c2bb5763794687722c9fd3363b8 Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Sat, 14 Nov 2026 09:16:08 -0600 Subject: [PATCH 1/5] ci: add a nightly run of the slow and long tests nightly.yml runs at 03:17 UTC what pull request CI skips: the integration suite without -short in three timed groups, the Docker suite with its concurrent tests as group 4, -race -count=3 on lock, journal, inode and dir, and every FuzzXxx for 25 minutes with its corpus cached per target. A failure fails the run; the Summary job lists the failed tests and uploads them as nightly-failures. Nothing posts to issues. docker-integration.yml becomes callable; pull requests keep groups 1-3 without the concurrent tests. ci.yml checks every workflow's GO_VERSION. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 7 + .github/workflows/docker-integration.yml | 26 +- .github/workflows/nightly.yml | 284 +++++++++++++++++++++ CLAUDE.md | 4 + tests/docker/scripts/run-tests.sh | 5 +- tests/scripts/failed-tests.sh | 32 +++ tests/scripts/integration-groups.sh | 4 +- tests/scripts/integration-timings-full.txt | 141 ++++++++++ 8 files changed, 495 insertions(+), 8 deletions(-) create mode 100644 .github/workflows/nightly.yml create mode 100755 tests/scripts/failed-tests.sh create mode 100644 tests/scripts/integration-timings-full.txt diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b0f4eaf2..b19f395c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -141,6 +141,13 @@ jobs: echo "::error file=go.mod::go.mod requires Go $gomod but CI uses Go $GO_VERSION. Update GO_VERSION in .github/workflows/ci.yml." exit 1 fi + for f in .github/workflows/*.yml; do + v=$(sed -n 's/^ GO_VERSION: "\(.*\)"$/\1/p' "$f") + if [ -n "$v" ] && [ "$v" != "$GO_VERSION" ]; then + echo "::error file=$f::GO_VERSION is $v here but $GO_VERSION in ci.yml; keep them the same." + exit 1 + fi + done - name: golangci-lint uses: golangci/golangci-lint-action@ba0d7d2ec06a0ea1cb5fa41b2e4a3ab91d21278a # v9.3.0 diff --git a/.github/workflows/docker-integration.yml b/.github/workflows/docker-integration.yml index 17686824..8fb604b6 100644 --- a/.github/workflows/docker-integration.yml +++ b/.github/workflows/docker-integration.yml @@ -10,6 +10,22 @@ on: branches: [main, master] # Allow manual trigger workflow_dispatch: + # nightly.yml calls this with the concurrent tests in group 4. Pushes and pull + # requests leave the inputs empty, which keeps groups 1 to 3 without them. + workflow_call: + inputs: + groups: + description: JSON list of run-tests.sh groups + type: string + default: "[1, 2, 3]" + run-concurrent: + description: Run the tests SKIP_CONCURRENT_TESTS skips + type: boolean + default: false + publish-report: + description: Publish the junit report as a check run + type: boolean + default: true permissions: contents: read @@ -23,7 +39,7 @@ jobs: fail-fast: false matrix: # run-tests.sh in_test_group decides which tests each group runs - group: [1, 2, 3] + group: ${{ fromJSON(inputs.groups || '[1, 2, 3]') }} permissions: contents: read checks: write @@ -53,10 +69,10 @@ jobs: - name: Run multi-node tests run: | - # Skip concurrent tests in CI - they hang due to resource constraints (see Task #5176) - # These tests pass locally and are covered by VM tests + # Pull requests skip the concurrent tests, which hung on runners (Task #5176); + # the nightly workflow runs them as group 4. # Export ensures docker-compose can interpolate the variable - export SKIP_CONCURRENT_TESTS="true" + export SKIP_CONCURRENT_TESTS="${{ inputs.run-concurrent && 'false' || 'true' }}" export TEST_GROUP="${{ matrix.group }}" # Start the test cluster detached (no --exit-code-from as it implies --abort-on-container-exit) @@ -135,7 +151,7 @@ jobs: - name: Publish Test Report uses: mikepenz/action-junit-report@6ef3fdcfcd5f0b5d14df3e659ef19b8c6820f93d # v6.6.1 - if: always() + if: always() && inputs.publish-report != false with: report_paths: 'tests/docker/results/junit.xml' check_name: Docker test report (group ${{ matrix.group }}) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml new file mode 100644 index 00000000..9ced2552 --- /dev/null +++ b/.github/workflows/nightly.yml @@ -0,0 +1,284 @@ +# Runs what pull request CI cannot: the integration suite without -short, the +# Docker suite with its concurrent tests, repeated race runs of the lock, +# journal, inode and dir packages, and every fuzz target. A failure fails the +# run; the Summary job lists the failed tests and uploads them as nightly-failures. +name: Nightly + +on: + schedule: + - cron: "17 3 * * *" + workflow_dispatch: + inputs: + fuzztime: + description: How long to fuzz each target + default: 25m + # A short run checks the wiring when the workflow or its scripts change. + pull_request: + branches: [main, master] + paths: + - .github/workflows/nightly.yml + - .github/workflows/docker-integration.yml + - tests/scripts/failed-tests.sh + - tests/scripts/integration-groups.sh + - tests/scripts/integration-timings-full.txt + +permissions: + contents: read + +concurrency: + group: nightly-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +env: + GO_VERSION: "1.26" + FUZZTIME: ${{ github.event_name == 'pull_request' && '1m' || inputs.fuzztime || '25m' }} + +jobs: + integration: + name: Integration, full (group ${{ matrix.group }}) + runs-on: ubuntu-latest + timeout-minutes: 40 + strategy: + fail-fast: false + matrix: + # tests/scripts/integration-groups.sh splits the tests by their run time without -short + group: [1, 2, 3] + env: + INTEGRATION_TIMINGS: tests/scripts/integration-timings-full.txt + steps: + - name: Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version: ${{ env.GO_VERSION }} + check-latest: true + cache: true + + - name: Install FUSE + run: | + sudo apt-get update + sudo apt-get install -y fuse3 libfuse3-dev + sudo modprobe fuse + which fusermount || which fusermount3 + + - name: Build + run: make build + + - name: Check test groups + run: tests/scripts/integration-groups.sh check ${{ strategy.job-total }} + + - name: Integration tests without -short + shell: bash + run: | + tests=$(tests/scripts/integration-groups.sh regex ${{ strategy.job-total }} ${{ matrix.group }}) + go test -v -timeout 30m -run "$tests" ./tests/integration/... 2>&1 | tee integration-${{ matrix.group }}.log + + - name: Upload test log + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: nightly-integration-${{ matrix.group }} + path: integration-${{ matrix.group }}.log + retention-days: 14 + + docker: + name: Docker + uses: ./.github/workflows/docker-integration.yml + permissions: + contents: read + checks: write + with: + groups: "[1, 2, 3, 4]" + run-concurrent: true + publish-report: false + + race: + name: Race x3 (lock, journal, inode, dir) + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - name: Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version: ${{ env.GO_VERSION }} + check-latest: true + cache: true + + # Test images go on tmpfs, as in ci.yml's unit tests. + - name: Race tests, three runs each + shell: bash + env: + TMPDIR: /dev/shm/gcfs-race + GOCLUSTERFS_TEST_DIR: /dev/shm/gcfs-race + run: | + mkdir -p "$TMPDIR" + go test -race -count=3 -timeout 50m ./internal/lock/ ./internal/journal/ ./internal/inode/ ./internal/dir/ 2>&1 | tee race.log + + - name: Upload test log + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: nightly-race + path: race.log + retention-days: 14 + + fuzz-targets: + name: Find fuzz targets + runs-on: ubuntu-latest + outputs: + targets: ${{ steps.list.outputs.targets }} + steps: + - name: Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version: ${{ env.GO_VERSION }} + check-latest: true + cache: true + + - name: List every FuzzXxx + id: list + shell: bash + run: | + targets=$(go test -list '^Fuzz' ./... | awk ' + /^Fuzz/ { names[++n] = $1; next } + /^ok[ \t]/ { + k = split($2, parts, "/") + for (i = 1; i <= n; i++) printf "{\"pkg\":\"%s\",\"target\":\"%s\",\"name\":\"%s-%s\"}\n", $2, names[i], parts[k], names[i] + n = 0 + }' | jq -s -c .) + echo "$targets" | jq -r '.[].name' + if [ "$targets" = "[]" ]; then + echo "::error::found no fuzz targets" + exit 1 + fi + echo "targets=$targets" >> "$GITHUB_OUTPUT" + + fuzz: + name: Fuzz ${{ matrix.name }} + needs: fuzz-targets + runs-on: ubuntu-latest + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.fuzz-targets.outputs.targets) }} + env: + TMPDIR: /dev/shm/gcfs-fuzz + GOCLUSTERFS_TEST_DIR: /dev/shm/gcfs-fuzz + CORPUS_KEY: fuzz-corpus-${{ matrix.pkg }}-${{ matrix.target }}- + steps: + - name: Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version: ${{ env.GO_VERSION }} + check-latest: true + cache: true + + - name: Restore the corpus from earlier nights + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: fuzz-corpus + key: ${{ env.CORPUS_KEY }}${{ github.run_id }}-${{ github.run_attempt }} + restore-keys: ${{ env.CORPUS_KEY }} + + - name: Fuzz for ${{ env.FUZZTIME }} + shell: bash + run: | + mkdir -p "$TMPDIR" fuzz-corpus + corpus="$(go env GOCACHE)/fuzz/${{ matrix.pkg }}/${{ matrix.target }}" + mkdir -p "$corpus" + cp -R fuzz-corpus/. "$corpus/" + echo "starting from $(find "$corpus" -type f | wc -l) corpus entries" + go test -run '^$' -fuzz '^${{ matrix.target }}$' -fuzztime "$FUZZTIME" "${{ matrix.pkg }}" 2>&1 | tee fuzz.log + + # A failed night keeps its corpus too; the input that failed goes to testdata, not the cache. + - name: Collect the corpus and any failing input + if: always() + shell: bash + run: | + corpus="$(go env GOCACHE)/fuzz/${{ matrix.pkg }}/${{ matrix.target }}" + rm -rf fuzz-corpus + mkdir -p fuzz-corpus failing-input + if [ -d "$corpus" ]; then cp -R "$corpus/." fuzz-corpus/; fi + echo "keeping $(find fuzz-corpus -type f | wc -l) corpus entries" + testdata="$(go list -f '{{.Dir}}' "${{ matrix.pkg }}")/testdata/fuzz/${{ matrix.target }}" + git status --porcelain --untracked-files=all -- "$testdata" | awk '{ print $2 }' | xargs -r cp -t failing-input/ + + - name: Save the corpus + if: always() && github.event_name != 'pull_request' + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: fuzz-corpus + key: ${{ env.CORPUS_KEY }}${{ github.run_id }}-${{ github.run_attempt }} + + - name: Upload the log and any failing input + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: nightly-fuzz-${{ matrix.name }} + path: | + fuzz.log + failing-input/ + retention-days: 14 + + summary: + name: Summary + if: always() + needs: [integration, docker, race, fuzz-targets, fuzz] + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Download logs + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + path: artifacts + + - name: List failed tests + shell: bash + env: + NEEDS: ${{ toJSON(needs) }} + run: | + shopt -s nullglob + { + for f in artifacts/nightly-integration-*/*.log; do tests/scripts/failed-tests.sh go integration "$f"; done + for f in artifacts/nightly-race/*.log; do tests/scripts/failed-tests.sh go race "$f"; done + for f in artifacts/nightly-fuzz-*/fuzz.log; do tests/scripts/failed-tests.sh go fuzz "$f"; done + for f in artifacts/docker-test-results-*/results.txt; do tests/scripts/failed-tests.sh docker docker "$f"; done + } | sort -u >nightly-failures.tsv + { + echo "## Nightly results" + echo + echo "| Job | Result |" + echo "| --- | --- |" + jq -r 'to_entries[] | "| \(.key) | \(.value.result) |"' <<<"$NEEDS" + echo + if [ -s nightly-failures.tsv ]; then + echo "### Failed tests" + echo + echo "| Suite | Test |" + echo "| --- | --- |" + awk -F'\t' '{ printf "| %s | `%s` |\n", $1, $2 }' nightly-failures.tsv + else + echo "The logs name no failed test." + fi + } >>"$GITHUB_STEP_SUMMARY" + + - name: Upload the failed test list + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: nightly-failures + path: nightly-failures.tsv + retention-days: 30 diff --git a/CLAUDE.md b/CLAUDE.md index cb80185c..5deeedc8 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -122,6 +122,10 @@ make test-parity # Validate test coverage across environments make test-parity-report # Detailed coverage report ``` +### Nightly Workflow + +`.github/workflows/nightly.yml` runs at 03:17 UTC what pull request CI skips: the integration suite without `-short` (grouped by `tests/scripts/integration-timings-full.txt`), the Docker suite with its concurrent tests (group 4 of `run-tests.sh`), `-race -count=3` on lock, journal, inode and dir, and every `FuzzXxx` for 25 minutes with its corpus cached between nights. New fuzz targets join automatically. A failure fails the run; the Summary job lists failed tests and uploads them as the `nightly-failures` artifact. It posts nothing to issues. + ### Test Parity System (IMPORTANT) All test scenarios are tracked in `tests/test-manifest.yaml`. This manifest is the **single source of truth** for test coverage. diff --git a/tests/docker/scripts/run-tests.sh b/tests/docker/scripts/run-tests.sh index ad382f51..3c8f64d9 100644 --- a/tests/docker/scripts/run-tests.sh +++ b/tests/docker/scripts/run-tests.sh @@ -268,7 +268,8 @@ md5_portion() { test_passed=0 test_failed=0 -# CI runs the suite as two parallel jobs. TEST_GROUP=1 or 2 runs one half; empty runs everything. +# CI runs groups 1 to 3 as parallel jobs; the nightly workflow adds group 4, the +# tests SKIP_CONCURRENT_TESTS skips. An empty TEST_GROUP runs everything. in_test_group() { [[ -z "$TEST_GROUP" ]] && return 0 local group=1 @@ -278,6 +279,8 @@ in_test_group() { # Takes about 14 minutes in CI, so it runs alone. "More Than 4096 Files") group=3 ;; + "Concurrent Writes"|"Concurrent File Creation"|"Concurrent Same-File Writes"*|"Mixed Size Writes"|"Overlapping Block Writes"|"Concurrent Write Stress"|"Reversed Write Order") + group=4 ;; esac [[ "$group" == "$TEST_GROUP" ]] } diff --git a/tests/scripts/failed-tests.sh b/tests/scripts/failed-tests.sh new file mode 100755 index 00000000..2834d985 --- /dev/null +++ b/tests/scripts/failed-tests.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Prints "" for each failed test in a log, for the nightly summary. +# +# failed-tests.sh go go test output, with or without -v, fuzzing included +# failed-tests.sh docker tests/docker results.txt +# +# A Go package that failed without a failed test (a build error, a panic in +# TestMain) prints as " (package failed)"; a timeout names the tests +# that were still running. +set -euo pipefail + +kind="$1" suite="$2" log="$3" +case "$kind" in +go) + awk -v suite="$suite" ' + /^[[:space:]]*--- FAIL: / { sub(/^[[:space:]]*--- FAIL: /, ""); print suite "\t" $1; tests++; next } + /^panic: test timed out/ { timedout = 1; next } + timedout && /^\t+running tests:/ { running = 1; next } + running && /^\t\t[^ ]+ \(/ { sub(/^\t\t/, ""); print suite "\t" $1 " (timed out)"; tests++; next } + running { running = 0 } + /^FAIL\t/ { pkgs[$2] = 1 } + END { if (!tests) for (p in pkgs) print suite "\t" p " (package failed)" } + ' "$log" | sort -u + ;; +docker) + awk -v suite="$suite" '/^FAIL: / { sub(/^FAIL: /, ""); print suite "\t" $0 }' "$log" | sort -u + ;; +*) + echo "usage: $0 go|docker " >&2 + exit 2 + ;; +esac diff --git a/tests/scripts/integration-groups.sh b/tests/scripts/integration-groups.sh index c64920cb..f5a4beeb 100755 --- a/tests/scripts/integration-groups.sh +++ b/tests/scripts/integration-groups.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # Splits the Go integration tests into CI groups of about equal run time, -# using integration-timings.txt. +# using integration-timings.txt, or the file INTEGRATION_TIMINGS names. # # integration-groups.sh regex -run regex for one group # integration-groups.sh check fail unless every test is in exactly one group @@ -8,7 +8,7 @@ set -euo pipefail cd "$(dirname "$0")/../.." pkg=./tests/integration -timings=tests/scripts/integration-timings.txt +timings="${INTEGRATION_TIMINGS:-tests/scripts/integration-timings.txt}" default_seconds=3 list_tests() { diff --git a/tests/scripts/integration-timings-full.txt b/tests/scripts/integration-timings-full.txt new file mode 100644 index 00000000..28627f0d --- /dev/null +++ b/tests/scripts/integration-timings-full.txt @@ -0,0 +1,141 @@ +# Seconds per integration test without -short, from one run of main 6ca8273 on a Linux host. +# nightly.yml groups the full suite with these through INTEGRATION_TIMINGS; tests not listed count as 3s. +TestAPIAdminOnLiveMount 0.24 +TestAPIServesHealthOnLoopback 0.22 +TestAPISnapshotLifecycle 0.41 +TestAPIStatsReportLiveValues 2.46 +TestAPIWritesDuringFileIO 99.50 +TestAPIWritesOffWithoutToken 0.22 +TestAppendFile 0.22 +TestBroadcastBidirectional 0.98 +TestBroadcastDirectoryInvalidation 0.97 +TestBroadcastInvalidatesPeerCache 1.49 +TestBroadcastInvalidationSpeed 0.98 +TestBroadcastMultipleUpdates 1.41 +TestChmodKeepsFileType 0.25 +TestCloneBasic 0.73 +TestCloneCoW 0.73 +TestCloneDataIntegrity 0.74 +TestCloneDestinationExists 0.22 +TestCloneFromSnapshot 0.74 +TestCloneInPlaceWrite 0.54 +TestCloneSourceCoW 0.73 +TestCommandsShareExitCodes 0.07 +TestConcurrentPwritesToFallocatedBlock 0.54 +TestCopyFile 0.22 +TestCreateDirectory 0.22 +TestCreateReadFile 0.22 +TestDashboardFlagOff 0.22 +TestDashboardOnLiveMount 0.22 +TestDataPersistence 0.45 +TestDeleteDirectory 0.22 +TestDeleteFile 0.22 +TestDeleteNonEmptyDirectory 0.24 +TestExtentTreeInterleavedAppends 0.67 +TestExtentTreeSnapshot 0.70 +TestExtentTreeSparseWrites 0.57 +TestExtentTreeTruncate 0.58 +TestFallocateUnwrittenBlocks 0.23 +TestFileAttributes 0.22 +TestFileInDirectory 0.23 +TestFileNotFound 0.21 +TestFileStatBlocks 0.23 +TestFilesystemStats 0.22 +TestFsckCleanAfterWorkload 12.37 +TestFsckReleasesDeadSlot 13.61 +TestFsckRepairsOlderSharing 14.83 +TestGfsctlReadsLiveMount 0.25 +TestHealthAndSignalShutdown 1.55 +TestJournalCrashDuringWrite 0.76 +TestJournalCrashRecoveryDirectory 0.55 +TestJournalCrashRecoveryFile 0.54 +TestJournalCrashRecoveryMultipleOps 0.55 +TestJournalDisabled 0.00 +TestJournalRemountOpensExisting 0.43 +TestLargeFile 0.26 +TestLargeVolume 9.65 +TestLongFilename 0.22 +TestManyFiles 0.45 +TestMkdirP 0.23 +TestMountExitsCleanlyDuringSnapshots 0.54 +TestMountFailsOnTakenBroadcastPortBeforeFormat 0.01 +TestMountPrintsVersion 0.01 +TestMountRefusesDamagedJournal 10.69 +TestMountRefusesLayoutV2 0.23 +TestMountRefusesLegacyLayout 0.01 +TestMountRefusesUnauthenticatedRemoteAPI 0.01 +TestMountRejectsBadAPISettingsBeforeOpeningDevice 0.05 +TestMountTokenSources 0.24 +TestMountUnmount 0.22 +TestMountWarnsWithoutBroadcastSecret 0.02 +TestMultiBlockFileStatBlocks 0.24 +TestMultiNodeAttrTimeoutZero 0.51 +TestMultiNodeConcurrentCreatesDoNotCollide 1.23 +TestMultiNodeConcurrentDirectoryOps 1.69 +TestMultiNodeConcurrentFileCreation 1.01 +TestMultiNodeConcurrentFirstWriteToSameBlock 3.09 +TestMultiNodeConcurrentSameFileWrites 1.50 +TestMultiNodeConcurrentSameFileWrites3Nodes 1.75 +TestMultiNodeConcurrentWrites 0.98 +TestMultiNodeCrashRecovery 3.47 +TestMultiNodeCreateWithSignals 0.77 +TestMultiNodeCreatesInOneDirectory 7.58 +TestMultiNodeDirIndexWithoutBroadcast 1.02 +TestMultiNodeExtentCacheCoherency 0.57 +TestMultiNodeInodeReuseStaleRead 0.60 +TestMultiNodeJournalRecovery 36.49 +TestMultiNodeMixedSizeWrites 1.74 +TestMultiNodeNeighborInodeWrites 7.78 +TestMultiNodeOpenRefreshesCachedSize 0.50 +TestMultiNodeOverlappingBlockWrites 1.75 +TestMultiNodeRecoveryIdempotency 43.71 +TestMultiNodeRecoveryWithLargeFile 37.48 +TestMultiNodeRecoveryWithMultipleFiles 36.49 +TestMultiNodeRestartKeepsPeerJournal 6.00 +TestMultiNodeReversedWriteOrder 1.54 +TestMultiNodeSequentialAccess 1.46 +TestMultiNodeStatfsFreeBlocks 0.55 +TestMultiNodeSubBlockWritesToOneBlock 2.86 +TestMultiNodeWriteStress 1.85 +TestMultiNodeWritesToOneFallocatedBlock 2.70 +TestNestedDirectories 0.24 +TestNetworkPartition 6.01 +TestNodeRestartReplaysOwnJournalAtMount 0.46 +TestOverwriteFile 0.22 +TestRename 0.22 +TestRollingRestart 3.81 +TestSFBDataIntegrity 0.27 +TestSFBMigrationAt64KB 0.23 +TestSFBMultipleSmallFiles 0.36 +TestSFBOverwrite 0.23 +TestSFBPersistence 0.45 +TestSFBSmallFileAllocation 0.24 +TestSFBTierSelection 0.24 +TestSkipRecoveryDiscardsOnlyNamedSlot 20.52 +TestSlowNodeKeepsLocks 17.52 +TestSnapshotCoW 0.23 +TestSnapshotConcurrentCreatesAcrossNodes 0.72 +TestSnapshotCreation 0.23 +TestSnapshotDataIntegrity 0.23 +TestSnapshotDelete 0.26 +TestSnapshotDeleteAfterOverwrite 0.30 +TestSnapshotDeleteAndReuse 0.27 +TestSnapshotInvalidFile 0.22 +TestSnapshotLargeFile 0.26 +TestSnapshotMiddleDeletion 0.87 +TestSnapshotMultiple 0.55 +TestSnapshotPersistence 0.97 +TestSnapshotPreservesData 0.59 +TestSnapshotsDuringWritesAcrossNodes 2.42 +TestSparseFileBlocks 0.22 +TestSpecialCharacters 0.23 +TestStalledNodesWritesStopAtWriteGate 25.58 +TestStatfsAvailableBlocks 0.24 +TestStatfsFreeBlocksAfterRemount 0.47 +TestStress 0.97 +TestStressRandomFailures 3.49 +TestSurvivorReleasesCrashedNodeLocks 5.48 +TestTreeReservationsDoNotLeakBlocks 1.05 +TestTruncateFile 0.22 +TestWaiterOutlastsLockTimeoutThroughFailover 35.50 +TestWriteInsideFallocatedRange 0.50 From 2d965b4f0d24ceb11cf2b6a2370b3758cd65fc95 Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Sat, 14 Nov 2026 09:18:24 -0600 Subject: [PATCH 2/5] ci: keep the Docker report on pull requests; set fuzz TMPDIR per step GitHub expressions compare null and false as equal, so inputs.publish-report != false skipped the report on pull requests and pushes. A skip-report input that defaults to false reads null as false. The fuzz job's TMPDIR moves to the fuzz step, after the directory exists, since setup-go runs go env before it. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/docker-integration.yml | 8 ++++---- .github/workflows/nightly.yml | 8 +++++--- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/.github/workflows/docker-integration.yml b/.github/workflows/docker-integration.yml index 8fb604b6..1b9bdbaa 100644 --- a/.github/workflows/docker-integration.yml +++ b/.github/workflows/docker-integration.yml @@ -22,10 +22,10 @@ on: description: Run the tests SKIP_CONCURRENT_TESTS skips type: boolean default: false - publish-report: - description: Publish the junit report as a check run + skip-report: + description: Do not publish the junit report as a check run type: boolean - default: true + default: false permissions: contents: read @@ -151,7 +151,7 @@ jobs: - name: Publish Test Report uses: mikepenz/action-junit-report@6ef3fdcfcd5f0b5d14df3e659ef19b8c6820f93d # v6.6.1 - if: always() && inputs.publish-report != false + if: always() && !inputs.skip-report with: report_paths: 'tests/docker/results/junit.xml' check_name: Docker test report (group ${{ matrix.group }}) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 9ced2552..fd8625ff 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -92,7 +92,7 @@ jobs: with: groups: "[1, 2, 3, 4]" run-concurrent: true - publish-report: false + skip-report: true race: name: Race x3 (lock, journal, inode, dir) @@ -171,8 +171,6 @@ jobs: matrix: include: ${{ fromJSON(needs.fuzz-targets.outputs.targets) }} env: - TMPDIR: /dev/shm/gcfs-fuzz - GOCLUSTERFS_TEST_DIR: /dev/shm/gcfs-fuzz CORPUS_KEY: fuzz-corpus-${{ matrix.pkg }}-${{ matrix.target }}- steps: - name: Checkout code @@ -192,8 +190,12 @@ jobs: key: ${{ env.CORPUS_KEY }}${{ github.run_id }}-${{ github.run_attempt }} restore-keys: ${{ env.CORPUS_KEY }} + # Test images go on tmpfs; TMPDIR is set only here, since setup-go's go env needs it to exist. - name: Fuzz for ${{ env.FUZZTIME }} shell: bash + env: + TMPDIR: /dev/shm/gcfs-fuzz + GOCLUSTERFS_TEST_DIR: /dev/shm/gcfs-fuzz run: | mkdir -p "$TMPDIR" fuzz-corpus corpus="$(go env GOCACHE)/fuzz/${{ matrix.pkg }}/${{ matrix.target }}" From 0d480bdf6029ec76de79608f68a690caf65aac7c Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Sat, 14 Nov 2026 09:35:51 -0600 Subject: [PATCH 3/5] ci: cap each fuzz process at 6 GiB of virtual memory An input that allocates without bound then kills its fuzz worker, which saves the input, instead of the runner OOM-killing the job first. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/nightly.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index fd8625ff..33c57de5 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -202,6 +202,8 @@ jobs: mkdir -p "$corpus" cp -R fuzz-corpus/. "$corpus/" echo "starting from $(find "$corpus" -type f | wc -l) corpus entries" + # 6 GiB per process: an input that allocates without bound kills its worker, which records it, not the runner. + ulimit -v 6291456 go test -run '^$' -fuzz '^${{ matrix.target }}$' -fuzztime "$FUZZTIME" "${{ matrix.pkg }}" 2>&1 | tee fuzz.log # A failed night keeps its corpus too; the input that failed goes to testdata, not the cache. From 2cdfe4442af496893629bf7067a15bbe28887c29 Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Sat, 14 Nov 2026 13:14:07 -0600 Subject: [PATCH 4/5] ci: nightly EIO sweep and per-target fuzz budgets The nightly runs TestOneFailedWriteKeepsAcknowledgedResults with GOCLUSTERFS_EIO_SWEEP=metadata under -race (19 min on a Linux host; full ran past 2 hours) and fails if the test did not run, since a -run that matches nothing passes. tests/scripts/fuzz-budgets.txt gives targets 10m or 45m instead of 25m, from 2h runs that show which ones stop finding coverage early and which still find it at 2h. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/nightly.yml | 73 +++++++++++++++++++++++++++++----- CLAUDE.md | 2 +- tests/scripts/fuzz-budgets.txt | 17 ++++++++ 3 files changed, 80 insertions(+), 12 deletions(-) create mode 100644 tests/scripts/fuzz-budgets.txt diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 33c57de5..cced95ee 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -1,7 +1,8 @@ # Runs what pull request CI cannot: the integration suite without -short, the # Docker suite with its concurrent tests, repeated race runs of the lock, -# journal, inode and dir packages, and every fuzz target. A failure fails the -# run; the Summary job lists the failed tests and uploads them as nightly-failures. +# journal, inode and dir packages, the wider EIO sweep, and every fuzz target. +# A failure fails the run; the Summary job lists the failed tests and uploads +# them as nightly-failures. name: Nightly on: @@ -10,8 +11,8 @@ on: workflow_dispatch: inputs: fuzztime: - description: How long to fuzz each target - default: 25m + description: Fuzz every target this long instead of its budget in tests/scripts/fuzz-budgets.txt + default: "" # A short run checks the wiring when the workflow or its scripts change. pull_request: branches: [main, master] @@ -19,6 +20,7 @@ on: - .github/workflows/nightly.yml - .github/workflows/docker-integration.yml - tests/scripts/failed-tests.sh + - tests/scripts/fuzz-budgets.txt - tests/scripts/integration-groups.sh - tests/scripts/integration-timings-full.txt @@ -31,7 +33,6 @@ concurrency: env: GO_VERSION: "1.26" - FUZZTIME: ${{ github.event_name == 'pull_request' && '1m' || inputs.fuzztime || '25m' }} jobs: integration: @@ -127,6 +128,50 @@ jobs: path: race.log retention-days: 14 + eio-sweep: + name: EIO sweep (${{ matrix.mode }}) + runs-on: ubuntu-latest + timeout-minutes: 80 + strategy: + fail-fast: false + matrix: + # GOCLUSTERFS_EIO_SWEEP in internal/goclusterfs/eio_sweep_test.go. Under -race, + # metadata took 19 min on a Linux host; full ran past 2 hours. + mode: [metadata] + steps: + - name: Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version: ${{ env.GO_VERSION }} + check-latest: true + cache: true + + # A -run that matches nothing passes, so the step also checks the test ran. + - name: Fail each chosen write once + shell: bash + env: + TMPDIR: /dev/shm/gcfs-eio + GOCLUSTERFS_TEST_DIR: /dev/shm/gcfs-eio + GOCLUSTERFS_EIO_SWEEP: ${{ matrix.mode }} + run: | + mkdir -p "$TMPDIR" + go test -race -count=1 -timeout 70m -v -run '^TestOneFailedWriteKeepsAcknowledgedResults$' ./internal/goclusterfs/ 2>&1 | tee eio-sweep.log + if ! grep -q '^--- PASS: TestOneFailedWriteKeepsAcknowledgedResults ' eio-sweep.log; then + echo "::error::TestOneFailedWriteKeepsAcknowledgedResults did not run" + exit 1 + fi + + - name: Upload test log + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: nightly-eio-sweep-${{ matrix.mode }} + path: eio-sweep.log + retention-days: 14 + fuzz-targets: name: Find fuzz targets runs-on: ubuntu-latest @@ -147,14 +192,18 @@ jobs: id: list shell: bash run: | - targets=$(go test -list '^Fuzz' ./... | awk ' + targets=$(go test -list '^Fuzz' ./... | awk -v def=25m ' + NR == FNR { if ($1 !~ /^#/ && NF == 2) budget[$1] = $2; next } /^Fuzz/ { names[++n] = $1; next } /^ok[ \t]/ { k = split($2, parts, "/") - for (i = 1; i <= n; i++) printf "{\"pkg\":\"%s\",\"target\":\"%s\",\"name\":\"%s-%s\"}\n", $2, names[i], parts[k], names[i] + for (i = 1; i <= n; i++) { + t = names[i] + printf "{\"pkg\":\"%s\",\"target\":\"%s\",\"name\":\"%s-%s\",\"fuzztime\":\"%s\"}\n", $2, t, parts[k], t, (t in budget ? budget[t] : def) + } n = 0 - }' | jq -s -c .) - echo "$targets" | jq -r '.[].name' + }' tests/scripts/fuzz-budgets.txt - | jq -s -c .) + echo "$targets" | jq -r '.[] | .name + " " + .fuzztime' if [ "$targets" = "[]" ]; then echo "::error::found no fuzz targets" exit 1 @@ -165,12 +214,13 @@ jobs: name: Fuzz ${{ matrix.name }} needs: fuzz-targets runs-on: ubuntu-latest - timeout-minutes: 45 + timeout-minutes: 60 strategy: fail-fast: false matrix: include: ${{ fromJSON(needs.fuzz-targets.outputs.targets) }} env: + FUZZTIME: ${{ github.event_name == 'pull_request' && '1m' || inputs.fuzztime || matrix.fuzztime }} CORPUS_KEY: fuzz-corpus-${{ matrix.pkg }}-${{ matrix.target }}- steps: - name: Checkout code @@ -239,7 +289,7 @@ jobs: summary: name: Summary if: always() - needs: [integration, docker, race, fuzz-targets, fuzz] + needs: [integration, docker, race, eio-sweep, fuzz-targets, fuzz] runs-on: ubuntu-latest steps: - name: Checkout code @@ -259,6 +309,7 @@ jobs: { for f in artifacts/nightly-integration-*/*.log; do tests/scripts/failed-tests.sh go integration "$f"; done for f in artifacts/nightly-race/*.log; do tests/scripts/failed-tests.sh go race "$f"; done + for f in artifacts/nightly-eio-sweep-*/*.log; do tests/scripts/failed-tests.sh go eio-sweep "$f"; done for f in artifacts/nightly-fuzz-*/fuzz.log; do tests/scripts/failed-tests.sh go fuzz "$f"; done for f in artifacts/docker-test-results-*/results.txt; do tests/scripts/failed-tests.sh docker docker "$f"; done } | sort -u >nightly-failures.tsv diff --git a/CLAUDE.md b/CLAUDE.md index 5deeedc8..fce09231 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -124,7 +124,7 @@ make test-parity-report # Detailed coverage report ### Nightly Workflow -`.github/workflows/nightly.yml` runs at 03:17 UTC what pull request CI skips: the integration suite without `-short` (grouped by `tests/scripts/integration-timings-full.txt`), the Docker suite with its concurrent tests (group 4 of `run-tests.sh`), `-race -count=3` on lock, journal, inode and dir, and every `FuzzXxx` for 25 minutes with its corpus cached between nights. New fuzz targets join automatically. A failure fails the run; the Summary job lists failed tests and uploads them as the `nightly-failures` artifact. It posts nothing to issues. +`.github/workflows/nightly.yml` runs at 03:17 UTC what pull request CI skips: the integration suite without `-short` (grouped by `tests/scripts/integration-timings-full.txt`), the Docker suite with its concurrent tests (group 4 of `run-tests.sh`), `-race -count=3` on lock, journal, inode and dir, the `GOCLUSTERFS_EIO_SWEEP=metadata` EIO sweep, and every `FuzzXxx` for its budget in `tests/scripts/fuzz-budgets.txt` (25 minutes if unlisted) with its corpus cached between nights. New fuzz targets join automatically. A failure fails the run; the Summary job lists failed tests and uploads them as the `nightly-failures` artifact. It posts nothing to issues. ### Test Parity System (IMPORTANT) diff --git a/tests/scripts/fuzz-budgets.txt b/tests/scripts/fuzz-budgets.txt new file mode 100644 index 00000000..2bfc8299 --- /dev/null +++ b/tests/scripts/fuzz-budgets.txt @@ -0,0 +1,17 @@ +# Fuzz time per night for targets that need other than nightly.yml's 25m default. +# From 2h runs on a 16-core host (2026-11-14): short for targets that stopped finding +# new coverage within minutes, long for those still finding it at 2h. The corpus is +# cached between nights, so each night continues where the last one stopped. +FuzzDeserializeInode 10m +FuzzJournalSuperblock 10m +FuzzDecodeDescriptor 10m +FuzzRegistrySlot 10m +FuzzReplay 45m +FuzzCheck 45m +FuzzLockRow 45m +FuzzRefcountSector 45m +FuzzWriteRequests 45m +FuzzFileOpsPaths 45m +FuzzParseRecords 45m +FuzzTxnRecord 45m +FuzzParseTreeNode 45m From 38fb4e93671d88deeb6837308405668f192cae84 Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Sat, 14 Nov 2026 13:16:24 -0600 Subject: [PATCH 5/5] ci: run the short EIO sample on pull requests A pull request that changes the nightly checks the sweep's wiring with the default sample instead of waiting on the 19-minute metadata sweep, as fuzzing runs for 1m there. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/nightly.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index cced95ee..dd9fd2d2 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -155,7 +155,8 @@ jobs: env: TMPDIR: /dev/shm/gcfs-eio GOCLUSTERFS_TEST_DIR: /dev/shm/gcfs-eio - GOCLUSTERFS_EIO_SWEEP: ${{ matrix.mode }} + # On pull requests any other value runs the short default sample, to check the wiring. + GOCLUSTERFS_EIO_SWEEP: ${{ github.event_name == 'pull_request' && 'sample' || matrix.mode }} run: | mkdir -p "$TMPDIR" go test -race -count=1 -timeout 70m -v -run '^TestOneFailedWriteKeepsAcknowledgedResults$' ./internal/goclusterfs/ 2>&1 | tee eio-sweep.log