diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b0f4eaf2..b19f395c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -141,6 +141,13 @@ jobs: echo "::error file=go.mod::go.mod requires Go $gomod but CI uses Go $GO_VERSION. Update GO_VERSION in .github/workflows/ci.yml." exit 1 fi + for f in .github/workflows/*.yml; do + v=$(sed -n 's/^ GO_VERSION: "\(.*\)"$/\1/p' "$f") + if [ -n "$v" ] && [ "$v" != "$GO_VERSION" ]; then + echo "::error file=$f::GO_VERSION is $v here but $GO_VERSION in ci.yml; keep them the same." + exit 1 + fi + done - name: golangci-lint uses: golangci/golangci-lint-action@ba0d7d2ec06a0ea1cb5fa41b2e4a3ab91d21278a # v9.3.0 diff --git a/.github/workflows/docker-integration.yml b/.github/workflows/docker-integration.yml index 17686824..1b9bdbaa 100644 --- a/.github/workflows/docker-integration.yml +++ b/.github/workflows/docker-integration.yml @@ -10,6 +10,22 @@ on: branches: [main, master] # Allow manual trigger workflow_dispatch: + # nightly.yml calls this with the concurrent tests in group 4. Pushes and pull + # requests leave the inputs empty, which keeps groups 1 to 3 without them. + workflow_call: + inputs: + groups: + description: JSON list of run-tests.sh groups + type: string + default: "[1, 2, 3]" + run-concurrent: + description: Run the tests SKIP_CONCURRENT_TESTS skips + type: boolean + default: false + skip-report: + description: Do not publish the junit report as a check run + type: boolean + default: false permissions: contents: read @@ -23,7 +39,7 @@ jobs: fail-fast: false matrix: # run-tests.sh in_test_group decides which tests each group runs - group: [1, 2, 3] + group: ${{ fromJSON(inputs.groups || '[1, 2, 3]') }} permissions: contents: read checks: write @@ -53,10 +69,10 @@ jobs: - name: Run multi-node tests run: | - # Skip concurrent tests in CI - they hang due to resource constraints (see Task #5176) - # These tests pass locally and are covered by VM tests + # Pull requests skip the concurrent tests, which hung on runners (Task #5176); + # the nightly workflow runs them as group 4. # Export ensures docker-compose can interpolate the variable - export SKIP_CONCURRENT_TESTS="true" + export SKIP_CONCURRENT_TESTS="${{ inputs.run-concurrent && 'false' || 'true' }}" export TEST_GROUP="${{ matrix.group }}" # Start the test cluster detached (no --exit-code-from as it implies --abort-on-container-exit) @@ -135,7 +151,7 @@ jobs: - name: Publish Test Report uses: mikepenz/action-junit-report@6ef3fdcfcd5f0b5d14df3e659ef19b8c6820f93d # v6.6.1 - if: always() + if: always() && !inputs.skip-report with: report_paths: 'tests/docker/results/junit.xml' check_name: Docker test report (group ${{ matrix.group }}) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml new file mode 100644 index 00000000..dd9fd2d2 --- /dev/null +++ b/.github/workflows/nightly.yml @@ -0,0 +1,340 @@ +# Runs what pull request CI cannot: the integration suite without -short, the +# Docker suite with its concurrent tests, repeated race runs of the lock, +# journal, inode and dir packages, the wider EIO sweep, and every fuzz target. +# A failure fails the run; the Summary job lists the failed tests and uploads +# them as nightly-failures. +name: Nightly + +on: + schedule: + - cron: "17 3 * * *" + workflow_dispatch: + inputs: + fuzztime: + description: Fuzz every target this long instead of its budget in tests/scripts/fuzz-budgets.txt + default: "" + # A short run checks the wiring when the workflow or its scripts change. + pull_request: + branches: [main, master] + paths: + - .github/workflows/nightly.yml + - .github/workflows/docker-integration.yml + - tests/scripts/failed-tests.sh + - tests/scripts/fuzz-budgets.txt + - tests/scripts/integration-groups.sh + - tests/scripts/integration-timings-full.txt + +permissions: + contents: read + +concurrency: + group: nightly-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +env: + GO_VERSION: "1.26" + +jobs: + integration: + name: Integration, full (group ${{ matrix.group }}) + runs-on: ubuntu-latest + timeout-minutes: 40 + strategy: + fail-fast: false + matrix: + # tests/scripts/integration-groups.sh splits the tests by their run time without -short + group: [1, 2, 3] + env: + INTEGRATION_TIMINGS: tests/scripts/integration-timings-full.txt + steps: + - name: Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version: ${{ env.GO_VERSION }} + check-latest: true + cache: true + + - name: Install FUSE + run: | + sudo apt-get update + sudo apt-get install -y fuse3 libfuse3-dev + sudo modprobe fuse + which fusermount || which fusermount3 + + - name: Build + run: make build + + - name: Check test groups + run: tests/scripts/integration-groups.sh check ${{ strategy.job-total }} + + - name: Integration tests without -short + shell: bash + run: | + tests=$(tests/scripts/integration-groups.sh regex ${{ strategy.job-total }} ${{ matrix.group }}) + go test -v -timeout 30m -run "$tests" ./tests/integration/... 2>&1 | tee integration-${{ matrix.group }}.log + + - name: Upload test log + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: nightly-integration-${{ matrix.group }} + path: integration-${{ matrix.group }}.log + retention-days: 14 + + docker: + name: Docker + uses: ./.github/workflows/docker-integration.yml + permissions: + contents: read + checks: write + with: + groups: "[1, 2, 3, 4]" + run-concurrent: true + skip-report: true + + race: + name: Race x3 (lock, journal, inode, dir) + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - name: Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version: ${{ env.GO_VERSION }} + check-latest: true + cache: true + + # Test images go on tmpfs, as in ci.yml's unit tests. + - name: Race tests, three runs each + shell: bash + env: + TMPDIR: /dev/shm/gcfs-race + GOCLUSTERFS_TEST_DIR: /dev/shm/gcfs-race + run: | + mkdir -p "$TMPDIR" + go test -race -count=3 -timeout 50m ./internal/lock/ ./internal/journal/ ./internal/inode/ ./internal/dir/ 2>&1 | tee race.log + + - name: Upload test log + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: nightly-race + path: race.log + retention-days: 14 + + eio-sweep: + name: EIO sweep (${{ matrix.mode }}) + runs-on: ubuntu-latest + timeout-minutes: 80 + strategy: + fail-fast: false + matrix: + # GOCLUSTERFS_EIO_SWEEP in internal/goclusterfs/eio_sweep_test.go. Under -race, + # metadata took 19 min on a Linux host; full ran past 2 hours. + mode: [metadata] + steps: + - name: Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version: ${{ env.GO_VERSION }} + check-latest: true + cache: true + + # A -run that matches nothing passes, so the step also checks the test ran. + - name: Fail each chosen write once + shell: bash + env: + TMPDIR: /dev/shm/gcfs-eio + GOCLUSTERFS_TEST_DIR: /dev/shm/gcfs-eio + # On pull requests any other value runs the short default sample, to check the wiring. + GOCLUSTERFS_EIO_SWEEP: ${{ github.event_name == 'pull_request' && 'sample' || matrix.mode }} + run: | + mkdir -p "$TMPDIR" + go test -race -count=1 -timeout 70m -v -run '^TestOneFailedWriteKeepsAcknowledgedResults$' ./internal/goclusterfs/ 2>&1 | tee eio-sweep.log + if ! grep -q '^--- PASS: TestOneFailedWriteKeepsAcknowledgedResults ' eio-sweep.log; then + echo "::error::TestOneFailedWriteKeepsAcknowledgedResults did not run" + exit 1 + fi + + - name: Upload test log + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: nightly-eio-sweep-${{ matrix.mode }} + path: eio-sweep.log + retention-days: 14 + + fuzz-targets: + name: Find fuzz targets + runs-on: ubuntu-latest + outputs: + targets: ${{ steps.list.outputs.targets }} + steps: + - name: Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version: ${{ env.GO_VERSION }} + check-latest: true + cache: true + + - name: List every FuzzXxx + id: list + shell: bash + run: | + targets=$(go test -list '^Fuzz' ./... | awk -v def=25m ' + NR == FNR { if ($1 !~ /^#/ && NF == 2) budget[$1] = $2; next } + /^Fuzz/ { names[++n] = $1; next } + /^ok[ \t]/ { + k = split($2, parts, "/") + for (i = 1; i <= n; i++) { + t = names[i] + printf "{\"pkg\":\"%s\",\"target\":\"%s\",\"name\":\"%s-%s\",\"fuzztime\":\"%s\"}\n", $2, t, parts[k], t, (t in budget ? budget[t] : def) + } + n = 0 + }' tests/scripts/fuzz-budgets.txt - | jq -s -c .) + echo "$targets" | jq -r '.[] | .name + " " + .fuzztime' + if [ "$targets" = "[]" ]; then + echo "::error::found no fuzz targets" + exit 1 + fi + echo "targets=$targets" >> "$GITHUB_OUTPUT" + + fuzz: + name: Fuzz ${{ matrix.name }} + needs: fuzz-targets + runs-on: ubuntu-latest + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.fuzz-targets.outputs.targets) }} + env: + FUZZTIME: ${{ github.event_name == 'pull_request' && '1m' || inputs.fuzztime || matrix.fuzztime }} + CORPUS_KEY: fuzz-corpus-${{ matrix.pkg }}-${{ matrix.target }}- + steps: + - name: Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version: ${{ env.GO_VERSION }} + check-latest: true + cache: true + + - name: Restore the corpus from earlier nights + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: fuzz-corpus + key: ${{ env.CORPUS_KEY }}${{ github.run_id }}-${{ github.run_attempt }} + restore-keys: ${{ env.CORPUS_KEY }} + + # Test images go on tmpfs; TMPDIR is set only here, since setup-go's go env needs it to exist. + - name: Fuzz for ${{ env.FUZZTIME }} + shell: bash + env: + TMPDIR: /dev/shm/gcfs-fuzz + GOCLUSTERFS_TEST_DIR: /dev/shm/gcfs-fuzz + run: | + mkdir -p "$TMPDIR" fuzz-corpus + corpus="$(go env GOCACHE)/fuzz/${{ matrix.pkg }}/${{ matrix.target }}" + mkdir -p "$corpus" + cp -R fuzz-corpus/. "$corpus/" + echo "starting from $(find "$corpus" -type f | wc -l) corpus entries" + # 6 GiB per process: an input that allocates without bound kills its worker, which records it, not the runner. + ulimit -v 6291456 + go test -run '^$' -fuzz '^${{ matrix.target }}$' -fuzztime "$FUZZTIME" "${{ matrix.pkg }}" 2>&1 | tee fuzz.log + + # A failed night keeps its corpus too; the input that failed goes to testdata, not the cache. + - name: Collect the corpus and any failing input + if: always() + shell: bash + run: | + corpus="$(go env GOCACHE)/fuzz/${{ matrix.pkg }}/${{ matrix.target }}" + rm -rf fuzz-corpus + mkdir -p fuzz-corpus failing-input + if [ -d "$corpus" ]; then cp -R "$corpus/." fuzz-corpus/; fi + echo "keeping $(find fuzz-corpus -type f | wc -l) corpus entries" + testdata="$(go list -f '{{.Dir}}' "${{ matrix.pkg }}")/testdata/fuzz/${{ matrix.target }}" + git status --porcelain --untracked-files=all -- "$testdata" | awk '{ print $2 }' | xargs -r cp -t failing-input/ + + - name: Save the corpus + if: always() && github.event_name != 'pull_request' + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: fuzz-corpus + key: ${{ env.CORPUS_KEY }}${{ github.run_id }}-${{ github.run_attempt }} + + - name: Upload the log and any failing input + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: nightly-fuzz-${{ matrix.name }} + path: | + fuzz.log + failing-input/ + retention-days: 14 + + summary: + name: Summary + if: always() + needs: [integration, docker, race, eio-sweep, fuzz-targets, fuzz] + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Download logs + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + path: artifacts + + - name: List failed tests + shell: bash + env: + NEEDS: ${{ toJSON(needs) }} + run: | + shopt -s nullglob + { + for f in artifacts/nightly-integration-*/*.log; do tests/scripts/failed-tests.sh go integration "$f"; done + for f in artifacts/nightly-race/*.log; do tests/scripts/failed-tests.sh go race "$f"; done + for f in artifacts/nightly-eio-sweep-*/*.log; do tests/scripts/failed-tests.sh go eio-sweep "$f"; done + for f in artifacts/nightly-fuzz-*/fuzz.log; do tests/scripts/failed-tests.sh go fuzz "$f"; done + for f in artifacts/docker-test-results-*/results.txt; do tests/scripts/failed-tests.sh docker docker "$f"; done + } | sort -u >nightly-failures.tsv + { + echo "## Nightly results" + echo + echo "| Job | Result |" + echo "| --- | --- |" + jq -r 'to_entries[] | "| \(.key) | \(.value.result) |"' <<<"$NEEDS" + echo + if [ -s nightly-failures.tsv ]; then + echo "### Failed tests" + echo + echo "| Suite | Test |" + echo "| --- | --- |" + awk -F'\t' '{ printf "| %s | `%s` |\n", $1, $2 }' nightly-failures.tsv + else + echo "The logs name no failed test." + fi + } >>"$GITHUB_STEP_SUMMARY" + + - name: Upload the failed test list + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: nightly-failures + path: nightly-failures.tsv + retention-days: 30 diff --git a/CLAUDE.md b/CLAUDE.md index 6b02fb56..5809bd70 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -122,6 +122,10 @@ make test-parity # Validate test coverage across environments make test-parity-report # Detailed coverage report ``` +### Nightly Workflow + +`.github/workflows/nightly.yml` runs at 03:17 UTC what pull request CI skips: the integration suite without `-short` (grouped by `tests/scripts/integration-timings-full.txt`), the Docker suite with its concurrent tests (group 4 of `run-tests.sh`), `-race -count=3` on lock, journal, inode and dir, the `GOCLUSTERFS_EIO_SWEEP=metadata` EIO sweep, and every `FuzzXxx` for its budget in `tests/scripts/fuzz-budgets.txt` (25 minutes if unlisted) with its corpus cached between nights. New fuzz targets join automatically. A failure fails the run; the Summary job lists failed tests and uploads them as the `nightly-failures` artifact. It posts nothing to issues. + ### Test Parity System (IMPORTANT) All test scenarios are tracked in `tests/test-manifest.yaml`. This manifest is the **single source of truth** for test coverage. diff --git a/tests/docker/scripts/run-tests.sh b/tests/docker/scripts/run-tests.sh index 7fd99d0a..44aea474 100644 --- a/tests/docker/scripts/run-tests.sh +++ b/tests/docker/scripts/run-tests.sh @@ -268,7 +268,8 @@ md5_portion() { test_passed=0 test_failed=0 -# CI runs the suite as two parallel jobs. TEST_GROUP=1 or 2 runs one half; empty runs everything. +# CI runs groups 1 to 3 as parallel jobs; the nightly workflow adds group 4, the +# tests SKIP_CONCURRENT_TESTS skips. An empty TEST_GROUP runs everything. in_test_group() { [[ -z "$TEST_GROUP" ]] && return 0 local group=1 @@ -278,6 +279,8 @@ in_test_group() { # Takes about 14 minutes in CI, so it runs alone. "More Than 4096 Files") group=3 ;; + "Concurrent Writes"|"Concurrent File Creation"|"Concurrent Same-File Writes"*|"Mixed Size Writes"|"Overlapping Block Writes"|"Concurrent Write Stress"|"Reversed Write Order") + group=4 ;; esac [[ "$group" == "$TEST_GROUP" ]] } diff --git a/tests/scripts/failed-tests.sh b/tests/scripts/failed-tests.sh new file mode 100755 index 00000000..2834d985 --- /dev/null +++ b/tests/scripts/failed-tests.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Prints "" for each failed test in a log, for the nightly summary. +# +# failed-tests.sh go go test output, with or without -v, fuzzing included +# failed-tests.sh docker tests/docker results.txt +# +# A Go package that failed without a failed test (a build error, a panic in +# TestMain) prints as " (package failed)"; a timeout names the tests +# that were still running. +set -euo pipefail + +kind="$1" suite="$2" log="$3" +case "$kind" in +go) + awk -v suite="$suite" ' + /^[[:space:]]*--- FAIL: / { sub(/^[[:space:]]*--- FAIL: /, ""); print suite "\t" $1; tests++; next } + /^panic: test timed out/ { timedout = 1; next } + timedout && /^\t+running tests:/ { running = 1; next } + running && /^\t\t[^ ]+ \(/ { sub(/^\t\t/, ""); print suite "\t" $1 " (timed out)"; tests++; next } + running { running = 0 } + /^FAIL\t/ { pkgs[$2] = 1 } + END { if (!tests) for (p in pkgs) print suite "\t" p " (package failed)" } + ' "$log" | sort -u + ;; +docker) + awk -v suite="$suite" '/^FAIL: / { sub(/^FAIL: /, ""); print suite "\t" $0 }' "$log" | sort -u + ;; +*) + echo "usage: $0 go|docker " >&2 + exit 2 + ;; +esac diff --git a/tests/scripts/fuzz-budgets.txt b/tests/scripts/fuzz-budgets.txt new file mode 100644 index 00000000..2bfc8299 --- /dev/null +++ b/tests/scripts/fuzz-budgets.txt @@ -0,0 +1,17 @@ +# Fuzz time per night for targets that need other than nightly.yml's 25m default. +# From 2h runs on a 16-core host (2026-11-14): short for targets that stopped finding +# new coverage within minutes, long for those still finding it at 2h. The corpus is +# cached between nights, so each night continues where the last one stopped. +FuzzDeserializeInode 10m +FuzzJournalSuperblock 10m +FuzzDecodeDescriptor 10m +FuzzRegistrySlot 10m +FuzzReplay 45m +FuzzCheck 45m +FuzzLockRow 45m +FuzzRefcountSector 45m +FuzzWriteRequests 45m +FuzzFileOpsPaths 45m +FuzzParseRecords 45m +FuzzTxnRecord 45m +FuzzParseTreeNode 45m diff --git a/tests/scripts/integration-groups.sh b/tests/scripts/integration-groups.sh index c64920cb..f5a4beeb 100755 --- a/tests/scripts/integration-groups.sh +++ b/tests/scripts/integration-groups.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # Splits the Go integration tests into CI groups of about equal run time, -# using integration-timings.txt. +# using integration-timings.txt, or the file INTEGRATION_TIMINGS names. # # integration-groups.sh regex -run regex for one group # integration-groups.sh check fail unless every test is in exactly one group @@ -8,7 +8,7 @@ set -euo pipefail cd "$(dirname "$0")/../.." pkg=./tests/integration -timings=tests/scripts/integration-timings.txt +timings="${INTEGRATION_TIMINGS:-tests/scripts/integration-timings.txt}" default_seconds=3 list_tests() { diff --git a/tests/scripts/integration-timings-full.txt b/tests/scripts/integration-timings-full.txt new file mode 100644 index 00000000..28627f0d --- /dev/null +++ b/tests/scripts/integration-timings-full.txt @@ -0,0 +1,141 @@ +# Seconds per integration test without -short, from one run of main 6ca8273 on a Linux host. +# nightly.yml groups the full suite with these through INTEGRATION_TIMINGS; tests not listed count as 3s. +TestAPIAdminOnLiveMount 0.24 +TestAPIServesHealthOnLoopback 0.22 +TestAPISnapshotLifecycle 0.41 +TestAPIStatsReportLiveValues 2.46 +TestAPIWritesDuringFileIO 99.50 +TestAPIWritesOffWithoutToken 0.22 +TestAppendFile 0.22 +TestBroadcastBidirectional 0.98 +TestBroadcastDirectoryInvalidation 0.97 +TestBroadcastInvalidatesPeerCache 1.49 +TestBroadcastInvalidationSpeed 0.98 +TestBroadcastMultipleUpdates 1.41 +TestChmodKeepsFileType 0.25 +TestCloneBasic 0.73 +TestCloneCoW 0.73 +TestCloneDataIntegrity 0.74 +TestCloneDestinationExists 0.22 +TestCloneFromSnapshot 0.74 +TestCloneInPlaceWrite 0.54 +TestCloneSourceCoW 0.73 +TestCommandsShareExitCodes 0.07 +TestConcurrentPwritesToFallocatedBlock 0.54 +TestCopyFile 0.22 +TestCreateDirectory 0.22 +TestCreateReadFile 0.22 +TestDashboardFlagOff 0.22 +TestDashboardOnLiveMount 0.22 +TestDataPersistence 0.45 +TestDeleteDirectory 0.22 +TestDeleteFile 0.22 +TestDeleteNonEmptyDirectory 0.24 +TestExtentTreeInterleavedAppends 0.67 +TestExtentTreeSnapshot 0.70 +TestExtentTreeSparseWrites 0.57 +TestExtentTreeTruncate 0.58 +TestFallocateUnwrittenBlocks 0.23 +TestFileAttributes 0.22 +TestFileInDirectory 0.23 +TestFileNotFound 0.21 +TestFileStatBlocks 0.23 +TestFilesystemStats 0.22 +TestFsckCleanAfterWorkload 12.37 +TestFsckReleasesDeadSlot 13.61 +TestFsckRepairsOlderSharing 14.83 +TestGfsctlReadsLiveMount 0.25 +TestHealthAndSignalShutdown 1.55 +TestJournalCrashDuringWrite 0.76 +TestJournalCrashRecoveryDirectory 0.55 +TestJournalCrashRecoveryFile 0.54 +TestJournalCrashRecoveryMultipleOps 0.55 +TestJournalDisabled 0.00 +TestJournalRemountOpensExisting 0.43 +TestLargeFile 0.26 +TestLargeVolume 9.65 +TestLongFilename 0.22 +TestManyFiles 0.45 +TestMkdirP 0.23 +TestMountExitsCleanlyDuringSnapshots 0.54 +TestMountFailsOnTakenBroadcastPortBeforeFormat 0.01 +TestMountPrintsVersion 0.01 +TestMountRefusesDamagedJournal 10.69 +TestMountRefusesLayoutV2 0.23 +TestMountRefusesLegacyLayout 0.01 +TestMountRefusesUnauthenticatedRemoteAPI 0.01 +TestMountRejectsBadAPISettingsBeforeOpeningDevice 0.05 +TestMountTokenSources 0.24 +TestMountUnmount 0.22 +TestMountWarnsWithoutBroadcastSecret 0.02 +TestMultiBlockFileStatBlocks 0.24 +TestMultiNodeAttrTimeoutZero 0.51 +TestMultiNodeConcurrentCreatesDoNotCollide 1.23 +TestMultiNodeConcurrentDirectoryOps 1.69 +TestMultiNodeConcurrentFileCreation 1.01 +TestMultiNodeConcurrentFirstWriteToSameBlock 3.09 +TestMultiNodeConcurrentSameFileWrites 1.50 +TestMultiNodeConcurrentSameFileWrites3Nodes 1.75 +TestMultiNodeConcurrentWrites 0.98 +TestMultiNodeCrashRecovery 3.47 +TestMultiNodeCreateWithSignals 0.77 +TestMultiNodeCreatesInOneDirectory 7.58 +TestMultiNodeDirIndexWithoutBroadcast 1.02 +TestMultiNodeExtentCacheCoherency 0.57 +TestMultiNodeInodeReuseStaleRead 0.60 +TestMultiNodeJournalRecovery 36.49 +TestMultiNodeMixedSizeWrites 1.74 +TestMultiNodeNeighborInodeWrites 7.78 +TestMultiNodeOpenRefreshesCachedSize 0.50 +TestMultiNodeOverlappingBlockWrites 1.75 +TestMultiNodeRecoveryIdempotency 43.71 +TestMultiNodeRecoveryWithLargeFile 37.48 +TestMultiNodeRecoveryWithMultipleFiles 36.49 +TestMultiNodeRestartKeepsPeerJournal 6.00 +TestMultiNodeReversedWriteOrder 1.54 +TestMultiNodeSequentialAccess 1.46 +TestMultiNodeStatfsFreeBlocks 0.55 +TestMultiNodeSubBlockWritesToOneBlock 2.86 +TestMultiNodeWriteStress 1.85 +TestMultiNodeWritesToOneFallocatedBlock 2.70 +TestNestedDirectories 0.24 +TestNetworkPartition 6.01 +TestNodeRestartReplaysOwnJournalAtMount 0.46 +TestOverwriteFile 0.22 +TestRename 0.22 +TestRollingRestart 3.81 +TestSFBDataIntegrity 0.27 +TestSFBMigrationAt64KB 0.23 +TestSFBMultipleSmallFiles 0.36 +TestSFBOverwrite 0.23 +TestSFBPersistence 0.45 +TestSFBSmallFileAllocation 0.24 +TestSFBTierSelection 0.24 +TestSkipRecoveryDiscardsOnlyNamedSlot 20.52 +TestSlowNodeKeepsLocks 17.52 +TestSnapshotCoW 0.23 +TestSnapshotConcurrentCreatesAcrossNodes 0.72 +TestSnapshotCreation 0.23 +TestSnapshotDataIntegrity 0.23 +TestSnapshotDelete 0.26 +TestSnapshotDeleteAfterOverwrite 0.30 +TestSnapshotDeleteAndReuse 0.27 +TestSnapshotInvalidFile 0.22 +TestSnapshotLargeFile 0.26 +TestSnapshotMiddleDeletion 0.87 +TestSnapshotMultiple 0.55 +TestSnapshotPersistence 0.97 +TestSnapshotPreservesData 0.59 +TestSnapshotsDuringWritesAcrossNodes 2.42 +TestSparseFileBlocks 0.22 +TestSpecialCharacters 0.23 +TestStalledNodesWritesStopAtWriteGate 25.58 +TestStatfsAvailableBlocks 0.24 +TestStatfsFreeBlocksAfterRemount 0.47 +TestStress 0.97 +TestStressRandomFailures 3.49 +TestSurvivorReleasesCrashedNodeLocks 5.48 +TestTreeReservationsDoNotLeakBlocks 1.05 +TestTruncateFile 0.22 +TestWaiterOutlastsLockTimeoutThroughFailover 35.50 +TestWriteInsideFallocatedRange 0.50