Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 8 additions & 13 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -56,8 +56,8 @@ jobs:
strategy:
fail-fast: false
matrix:
# tests/scripts/unit-groups.sh splits the packages by recorded run time
group: [1, 2, 3]
# tests/scripts/unit-groups.sh splits the tests by recorded run time
group: [1, 2, 3, 4]
steps:
- name: Checkout code
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
Expand All @@ -83,6 +83,8 @@ jobs:
mkdir -p "$TMPDIR"
dd if=/dev/zero of="$TMPDIR/odirect-check" bs=4096 count=1 oflag=direct
pkgs=$(tests/scripts/unit-groups.sh packages ${{ strategy.job-total }} ${{ matrix.group }})
UNIT_RUN=$(tests/scripts/unit-groups.sh run ${{ strategy.job-total }} ${{ matrix.group }})
export UNIT_RUN
make test-unit UNIT_PKGS="$pkgs" UNIT_TIMEOUT=8m COVER_PROFILE=coverage-${{ matrix.group }}.out 2>&1 | tee unit-test-${{ matrix.group }}.log
timeout-minutes: 10

Expand All @@ -101,24 +103,17 @@ jobs:
runs-on: ubuntu-latest
needs: unit-tests
steps:
- name: Checkout code
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

- name: Download coverage
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
with:
pattern: unit-tests-*
merge-multiple: true

# The groups test disjoint packages, so their profiles concatenate.
- name: Coverage summary
run: |
{
echo "## Unit test coverage"
echo
echo "**Total: $(awk 'FNR > 1 { n += $2; if ($3 > 0) c += $2 } END { printf "%.1f%%", 100 * c / n }' coverage-*.out)**"
echo
echo "| Package | Coverage |"
echo "| --- | --- |"
awk '$1 == "ok" && $4 == "coverage:" {print "| " $2 " | " $5 " |"}' unit-test-*.log | sort
} >> "$GITHUB_STEP_SUMMARY"
run: tests/scripts/coverage-summary.sh coverage-*.out >> "$GITHUB_STEP_SUMMARY"

lint:
name: Lint
Expand Down
3 changes: 2 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -63,9 +63,10 @@ dashboard:
# Unit tests (fast, no FUSE needed)
test: test-unit

# UNIT_RUN in the environment limits the tests to a -run regex; CI uses it for split packages
test-unit:
@echo "Running unit tests with race detector..."
$(GOTEST) -v -race -timeout $(UNIT_TIMEOUT) -covermode=atomic -coverprofile=$(COVER_PROFILE) $(UNIT_PKGS)
$(GOTEST) -v -race -timeout $(UNIT_TIMEOUT) -run "$${UNIT_RUN:-.}" -covermode=atomic -coverprofile=$(COVER_PROFILE) $(UNIT_PKGS)

# Full integration suite, including what CI's -short skips (requires FUSE and built binary)
test-integration: check-fuse build
Expand Down
27 changes: 27 additions & 0 deletions tests/scripts/coverage-summary.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
#!/usr/bin/env bash
# Prints a markdown coverage summary from Go cover profiles. A package that CI splits
# by test appears in several profiles, so each block counts once, covered if any
# profile ran it.
#
# coverage-summary.sh <profile>...
set -euo pipefail

awk '
FNR == 1 { next }
{ stmts[$1] = $2; hits[$1] += $3 }
END {
for (b in stmts) {
pkg = b
sub(/\/[^\/]*:.*/, "", pkg)
n[pkg] += stmts[b]
total += stmts[b]
if (hits[b] > 0) {
c[pkg] += stmts[b]
covered += stmts[b]
}
}
printf "## Unit test coverage\n\n**Total: %.1f%%**\n\n| Package | Coverage |\n| --- | --- |\n", total ? 100 * covered / total : 0
fflush()
for (p in n) printf "| %s | %.1f%% |\n", p, 100 * c[p] / n[p] | "sort"
close("sort")
}' "$@"
82 changes: 70 additions & 12 deletions tests/scripts/unit-groups.sh
Original file line number Diff line number Diff line change
@@ -1,54 +1,112 @@
#!/usr/bin/env bash
# Splits the unit test packages into CI groups of about equal run time,
# using unit-timings.txt.
# Splits the unit tests into CI groups of about equal run time, using unit-timings.txt.
# A package with a shard count after its time there gets that many groups to itself,
# split by test name and balanced with unit-test-timings.txt; the other packages share
# the remaining groups.
#
# unit-groups.sh packages <groups> <group> packages for one group, space separated
# unit-groups.sh check <groups> fail unless every package is in exactly one group
# unit-groups.sh run <groups> <group> -run regex for one group; . runs whole packages
# unit-groups.sh check <groups> fail unless every package, and every test of a
# split package, is in exactly one group
set -euo pipefail

cd "$(dirname "$0")/../.."
timings=tests/scripts/unit-timings.txt
test_timings=tests/scripts/unit-test-timings.txt
default_seconds=5
default_test_seconds=1

list_packages() {
local out
out=$(go list ./internal/... ./tests/soak ./tests/integration/history)
echo "$out"
}

# Longest package first, each into the group with the least time so far.
assign() {
# "package shards" for each package to split by test.
split_packages() {
awk '$1 !~ /^#/ && NF >= 3 { print $1, $3 }' "$timings"
}

list_tests() {
local out
out=$(go test -list "${2:-.}" "$1")
grep -E '^(Test|Example|Fuzz)' <<<"$out" || true
}

# Longest first, each into the shard with the least time so far.
balance() {
sort -k1,1nr -k2,2 |
awk -v n="$1" -v first="$2" '{ m = 1; for (i = 2; i <= n; i++) if (load[i] < load[m]) m = i; load[m] += $1; print first + m - 1, $2 }'
}

# One line per assignment: "<group> <package>" or "<group> <package> <test>".
plan() {
local groups=$1 next=1 pkg shards
local -a split=()
while read -r pkg shards; do
split+=("$pkg")
list_tests "$pkg" |
awk -v def="$default_test_seconds" -v pkg="$pkg" 'NR == FNR { if ($1 == pkg) w[$2] = $3; next } { print ($1 in w ? w[$1] : def), $1 }' "$test_timings" - |
balance "$shards" "$next" |
awk -v pkg="$pkg" '{ print $1, pkg, $2 }'
next=$((next + shards))
done < <(split_packages)
if ((next > groups)); then
echo "unit-groups.sh: split packages need $((next - 1)) groups, leaving none of $groups for the rest" >&2
exit 1
fi
list_packages |
grep -vxF -f <(printf '%s\n' "${split[@]+"${split[@]}"}") |
awk -v def="$default_seconds" 'NR == FNR { if ($1 !~ /^#/) w[$1] = $2; next } { print ($1 in w ? w[$1] : def), $1 }' "$timings" - |
sort -k1,1nr -k2,2 |
awk -v n="$1" '{ m = 1; for (i = 2; i <= n; i++) if (load[i] < load[m]) m = i; load[m] += $1; print m, $2 }'
balance $((groups - next + 1)) "$next"
}

packages() {
assign "$1" | awk -v g="$2" '$1 == g { print $2 }'
plan "$1" | awk -v g="$2" '$1 == g && !seen[$2]++ { print $2 }'
}

# The -run regex for group $2 of the plan in $1.
regex() {
awk -v g="$2" '$1 == g && NF == 3 { names = names sep $3; sep = "|" } END { print (names == "" ? "." : "^(" names ")$") }' <<<"$1"
}

run() {
regex "$(plan "$1")" "$2"
}

check() {
local all got
local groups=$1 p all got pkg shards
p=$(plan "$groups")
all=$(list_packages | sort)
if [[ -z "$all" ]]; then
echo "found no unit test packages"
exit 1
fi
got=$(for g in $(seq 1 "$1"); do packages "$1" "$g"; done | sort)
got=$(awk 'NF == 2 { print $2 } NF == 3 && !seen[$2]++ { print $2 }' <<<"$p" | sort)
if [[ "$all" != "$got" ]]; then
echo "unit test groups miss or repeat packages (< missing, > repeated):"
diff <(echo "$all") <(echo "$got") || true
exit 1
fi
echo "each of $(echo "$all" | wc -l | tr -d ' ') unit test packages is in exactly one of $1 groups"
echo "each of $(wc -l <<<"$all" | tr -d ' ') unit test packages is in exactly one of $groups groups"
while read -r pkg shards; do
all=$(list_tests "$pkg" | sort)
got=$(for g in $(awk -v pkg="$pkg" '$2 == pkg { print $1 }' <<<"$p" | sort -u); do list_tests "$pkg" "$(regex "$p" "$g")"; done | sort)
if [[ "$all" != "$got" ]]; then
echo "the $shards groups of $pkg miss or repeat tests (< missing, > repeated):"
diff <(echo "$all") <(echo "$got") || true
exit 1
fi
echo "each of the $(wc -l <<<"$all" | tr -d ' ') tests of $pkg is in exactly one of its $shards groups"
done < <(split_packages)
}

case "${1:-}" in
packages) packages "$2" "$3" | paste -sd ' ' - ;;
run) run "$2" "$3" ;;
check) check "$2" ;;
*)
echo "usage: $0 packages <groups> <group> | check <groups>" >&2
echo "usage: $0 packages <groups> <group> | run <groups> <group> | check <groups>" >&2
exit 2
;;
esac
Loading
Loading