From 076e194a178a0ef6e37d56cdc6b44189e9b49cdf Mon Sep 17 00:00:00 2001 From: tannevaled Date: Tue, 3 Nov 2026 09:25:20 +0100 Subject: [PATCH 1/2] specdelta: a gate whose honest exit nobody could reach `-dir` took one directory. The move gate refuses a run whose surface reaches outside it and advised "Widen -dir" -- advice that could not be followed, for the move this repository is actually making: OPENED language/variables_spec.rb:366 ... OPENED core/binding/implicit_parameters_spec.rb:3 ... The 3.4.1 -> 4.0.9 surface spans core/ AND language/, and the only widening available from `core` was the whole corpus -- library/, optional/, command_line/, security/ and hours of them. So the only reachable exit was -partial, and every real answer about the move would have come out stamped PARTIAL. A derogation that is the normal road is not a derogation, and a gate whose honest exit nobody can reach is not a gate. -dir now names a SET, comma-separated, and the refusal names the exact -dir that would cover the surface instead of saying "widen": specdelta: the move 3.4.1 -> 4.0.9 touches 1 file(s) that -dir core does not measure: language/variables_spec.rb A number from this run would not be a number about the move. Either -dir core,language which measures the whole surface, or -partial to say so on purpose -- the report is then stamped PARTIAL. Top-level suites and no deeper: 50 exact directories would be unreadable and would go stale with the corpus, while `core,language` is both the honest surface and something a reader can retype. The suggestion is computed from the census, so it reaches core/ and language/ and NOT library/. Two things the set makes possible to get wrong, both tested: - a file reached through two named directories (`-dir core,core/set`) is measured ONCE; counted twice it is counted on both sides, and the totals then move with how the caller spelled -dir rather than with the engines; - `-dir core,` does not quietly become the whole corpus. Refs #799 Co-Authored-By: Claude Opus 5 --- .../rubyspec/cmd/specdelta/main.go | 139 ++++++++++++++---- .../rubyspec/cmd/specdelta/main_test.go | 98 ++++++++++++ 2 files changed, 207 insertions(+), 30 deletions(-) diff --git a/scripts/conformance/rubyspec/cmd/specdelta/main.go b/scripts/conformance/rubyspec/cmd/specdelta/main.go index 8a67e894..228b39f1 100644 --- a/scripts/conformance/rubyspec/cmd/specdelta/main.go +++ b/scripts/conformance/rubyspec/cmd/specdelta/main.go @@ -117,7 +117,13 @@ func main1(args []string, out, errOut io.Writer) int { fs := flag.NewFlagSet("specdelta", flag.ContinueOnError) fs.SetOutput(errOut) root := fs.String("root", ".", "the ruby/spec checkout") - dir := fs.String("dir", "", "a directory under -root to walk (recursively), e.g. core/set") + // ⛔ A SET, comma-separated. The move gate below refuses a surface -dir does + // not cover and advises widening it; with one directory that advice was not + // followable, because the 3.4.1→4.0.9 surface spans core/ AND language/ and + // the only widening from `core` was the whole corpus — library/, optional/, + // command_line/ and hours of them. The only reachable exit was -partial, so + // the derogation was the normal road and the gate stamped every real answer. + dir := fs.String("dir", "", "directories under -root to walk (recursively), comma-separated, e.g. core/set or core,language") timeout := fs.Duration("timeout", 60*time.Second, "per-file limit for one engine") quiet := fs.Bool("quiet", false, "print only the files that moved") // ⛔ One run per engine cannot tell a FLAKE from a difference, and the first @@ -140,11 +146,12 @@ func main1(args []string, out, errOut io.Writer) int { fmt.Fprintln(errOut, "specdelta: name exactly two interpreters — the control first, the treatment second") return 2 } - if *dir == "" { + dirs := splitDirs(*dir) + if len(dirs) == 0 { fmt.Fprintln(errOut, "specdelta: -dir is required") return 2 } - files, err := specFiles(*root, *dir) + files, err := specFiles(*root, dirs) if err != nil { fmt.Fprintf(errOut, "specdelta: %v\n", err) return 2 @@ -152,7 +159,7 @@ func main1(args []string, out, errOut io.Writer) int { if len(files) == 0 { // ⛔ An empty population is a refusal, not a clean result: a mistyped // -dir would otherwise print "0 files, no regressions" and pass. - fmt.Fprintf(errOut, "specdelta: no _spec.rb files under %s\n", filepath.Join(*root, *dir)) + fmt.Fprintf(errOut, "specdelta: no _spec.rb files under %s\n", strings.Join(dirs, ", ")) return 2 } @@ -163,7 +170,7 @@ func main1(args []string, out, errOut io.Writer) int { // THE MOVE GATE, before a single engine is started: a run whose surface is // already known to be incomplete must not cost an hour first. - mv, err := declaredMove(*move, *root, *dir, files, errOut, *partial) + mv, err := declaredMove(*move, *root, dirs, files, errOut, *partial) if err != nil { return 2 } @@ -409,7 +416,7 @@ func (d declared) partialNote() string { // readable table saying nothing about the move, because the three files the // move hits hardest are under core/binding. There is no way to see that in the // results; it is only visible in the corpus, before anything runs. -func declaredMove(spec, root, dir string, files []string, errOut io.Writer, allowPartial bool) (declared, error) { +func declaredMove(spec, root string, dirs []string, files []string, errOut io.Writer, allowPartial bool) (declared, error) { if spec == "" { fmt.Fprint(errOut, `specdelta: -move is required. @@ -472,7 +479,7 @@ side, 37 failures on the other, and a pass delta of zero either way. return d, nil } sort.Strings(outside) - fmt.Fprintf(errOut, "specdelta: the move %s → %s touches %d file(s) that -dir %s does not measure:\n", from, to, len(outside), dir) + fmt.Fprintf(errOut, "specdelta: the move %s → %s touches %d file(s) that -dir %s does not measure:\n", from, to, len(outside), strings.Join(dirs, ",")) for i, f := range outside { if i == 12 { fmt.Fprintf(errOut, " … and %d more\n", len(outside)-i) @@ -481,8 +488,14 @@ side, 37 failures on the other, and a pass delta of zero either way. fmt.Fprintf(errOut, " %s\n", f) } if !allowPartial { - fmt.Fprintln(errOut, " A number from this run would not be a number about the move. Widen -dir, or") - fmt.Fprintln(errOut, " pass -partial to say so on purpose — the report is then stamped PARTIAL.") + // ⛔ Name the -dir that WOULD cover it. "Widen -dir" was advice the + // reader could not act on while -dir held one directory: widening + // `core` meant the whole corpus. An exit nobody can reach makes the + // derogation the normal road, which is the opposite of a gate. + fmt.Fprintln(errOut, " A number from this run would not be a number about the move. Either") + fmt.Fprintf(errOut, " -dir %s\n", strings.Join(coveringDirs(dirs, d.move.Touched()), ",")) + fmt.Fprintln(errOut, " which measures the whole surface, or -partial to say so on purpose —") + fmt.Fprintln(errOut, " the report is then stamped PARTIAL.") return declared{}, errRefused } d.partial = true @@ -639,39 +652,105 @@ func abs(n int) int { return n } -// specFiles lists the *_spec.rb files under root/dir, as paths relative to root -// — which is how the shim expects to be handed them. -func specFiles(root, dir string) ([]string, error) { - base := filepath.Join(root, dir) - info, err := os.Stat(base) - if err != nil { - return nil, err - } - if !info.IsDir() { - return nil, fmt.Errorf("%s is not a directory", base) +// splitDirs reads the comma-separated -dir into a set, in the order given and +// without repeats. An empty element is dropped rather than read as the root: +// `-dir core,` must not quietly become the whole corpus. +func splitDirs(s string) []string { + var out []string + seen := map[string]bool{} + for _, part := range strings.Split(s, ",") { + d := filepath.ToSlash(strings.TrimSpace(part)) + if d == "" || seen[d] { + continue + } + seen[d] = true + out = append(out, d) } + return out +} + +// specFiles lists the *_spec.rb files under each of root/dirs, as paths +// relative to root — which is how the shim expects to be handed them. +// +// ⛔ The result is DEDUPLICATED. `-dir core,core/set` names core/set twice, and +// a file run twice would be counted twice on both sides: a total that moves +// with how the caller spelled the set, not with the engines. +func specFiles(root string, dirs []string) ([]string, error) { + seen := map[string]bool{} var out []string - err = filepath.Walk(base, func(p string, fi os.FileInfo, err error) error { + for _, dir := range dirs { + base := filepath.Join(root, dir) + info, err := os.Stat(base) if err != nil { - return err + return nil, err } - if fi.IsDir() || !strings.HasSuffix(p, "_spec.rb") { - return nil + if !info.IsDir() { + return nil, fmt.Errorf("%s is not a directory", base) } - rel, err := filepath.Rel(root, p) + err = filepath.Walk(base, func(p string, fi os.FileInfo, err error) error { + if err != nil { + return err + } + if fi.IsDir() || !strings.HasSuffix(p, "_spec.rb") { + return nil + } + rel, err := filepath.Rel(root, p) + if err != nil { + return err + } + rel = filepath.ToSlash(rel) + if seen[rel] { + return nil + } + seen[rel] = true + out = append(out, rel) + return nil + }) if err != nil { - return err + return nil, err } - out = append(out, rel) - return nil - }) - if err != nil { - return nil, err } sort.Strings(out) return out, nil } +// coveringDirs is the -dir the refusal advises: the set already asked for, plus +// the top-level suite of every file the move touches that it does not reach. +// +// Top level and no deeper on purpose. A list of the 50 exact directories would +// be unreadable and would go stale the moment the corpus moves; `core,language` +// is both the honest surface and something a reader can retype. +func coveringDirs(dirs, touched []string) []string { + out := append([]string(nil), dirs...) + have := map[string]bool{} + for _, d := range out { + have[d] = true + } + for _, f := range touched { + top := filepath.ToSlash(f) + if i := strings.Index(top, "/"); i >= 0 { + top = top[:i] + } + if covered(have, filepath.ToSlash(f)) || have[top] { + continue + } + have[top] = true + out = append(out, top) + } + sort.Strings(out) + return out +} + +// covered says whether one of the named directories already contains f. +func covered(dirs map[string]bool, f string) bool { + for d := range dirs { + if f == d || strings.HasPrefix(f, d+"/") { + return true + } + } + return false +} + // runRepeatedly runs one file under one engine `times` over, and returns the // LOWEST-scoring result plus whether the engine disagreed with itself. // diff --git a/scripts/conformance/rubyspec/cmd/specdelta/main_test.go b/scripts/conformance/rubyspec/cmd/specdelta/main_test.go index 3c9c84c0..051abfc2 100644 --- a/scripts/conformance/rubyspec/cmd/specdelta/main_test.go +++ b/scripts/conformance/rubyspec/cmd/specdelta/main_test.go @@ -738,6 +738,104 @@ func TestAnExampleThatWentRedInsideAnImprovedFileIsNamed(t *testing.T) { } } +// ⛔ The surface of the real 3.4.1→4.0.9 move spans core/ AND language/, and +// -dir took ONE directory. The refusal advised "widen -dir", which from `core` +// meant the whole corpus — library/, optional/, command_line/ and hours of +// them. So the only reachable exit was -partial, and every real answer came out +// stamped. A gate whose honest exit nobody can reach is not a gate. +func TestTheSurfaceSpanningTwoSuitesHasAnExitThatIsNotADerogation(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("the fake engines are POSIX shell scripts") + } + root := t.TempDir() + guarded := "describe \"x\" do\n ruby_version_is \"4.0\" do\n it \"exists\" do\n end\n end\nend\n" + write := func(p, body string) { + t.Helper() + if err := os.MkdirAll(filepath.Join(root, filepath.Dir(p)), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, filepath.FromSlash(p)), []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + write("core/set/divide_spec.rb", guarded) + write("language/variables_spec.rb", guarded) + // Outside the surface the move touches, and in the real corpus the costly + // half: the advised -dir must not reach it. + write("library/erb/erb_spec.rb", "#\n") + engine := fakeEngine(t, root, "e", "echo 'RBGO_RESULT pass=1 fail=0 error=0 skip=0'\n") + move := []string{"-root", root, "-move", "3.4.1:4.0.9"} + + // -dir core misses language/, so it is refused — and the refusal NAMES the + // -dir that would cover the surface instead of telling the reader to widen. + var out, errOut bytes.Buffer + if code := main1(append(append([]string{}, move...), "-dir", "core", engine, engine), &out, &errOut); code != 2 { + t.Fatalf("exit = %d, want 2; err %q", code, errOut.String()) + } + if !strings.Contains(errOut.String(), "-dir core,language") { + t.Errorf("the refusal does not name the -dir that would cover the surface:\n%s", errOut.String()) + } + // And it does not drag in the suite the move never touches. + if strings.Contains(errOut.String(), "library") { + t.Errorf("the advised -dir reaches outside the surface:\n%s", errOut.String()) + } + + // Retyping exactly what it advised is accepted, unstamped. This is the + // control: without it the refusal above would only prove the gate fires. + out.Reset() + errOut.Reset() + if code := main1(append(append([]string{}, move...), "-dir", "core,language", engine, engine), &out, &errOut); code != 0 { + t.Fatalf("the advised -dir: exit = %d, want 0; err %q", code, errOut.String()) + } + if strings.Contains(out.String(), "PARTIAL") { + t.Errorf("a run covering the whole surface was stamped PARTIAL:\n%s", out.String()) + } + if !strings.Contains(out.String(), "2 file(s):") { + t.Errorf("the measured set is not the two suites:\n%s", out.String()) + } +} + +// ⛔ A file reached through two named directories must be measured ONCE. Counted +// twice it is counted on both sides, so the totals move with how the caller +// spelled -dir rather than with the engines. +func TestADirectoryNamedTwiceDoesNotCountItsFilesTwice(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("the fake engines are POSIX shell scripts") + } + root := t.TempDir() + for _, p := range []string{"core/set/divide_spec.rb", "language/variables_spec.rb"} { + if err := os.MkdirAll(filepath.Join(root, filepath.Dir(p)), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, filepath.FromSlash(p)), []byte("#\n"), 0o644); err != nil { + t.Fatal(err) + } + } + engine := fakeEngine(t, root, "e", "echo 'RBGO_RESULT pass=2 fail=0 error=0 skip=0'\n") + + for _, tc := range []struct { + name, dir, want string + }{ + // core and core/set reach the same file; it is measured once. + {"the same file through a directory and its parent", "core,core/set,core", "1 file(s): control 2 passing"}, + {"the set spelled once", "core,language", "2 file(s): control 4 passing"}, + // An empty element must not be read as the root: `-dir core,` would + // otherwise quietly sweep the whole corpus, which is the opposite of + // what the trailing comma suggests. + {"a trailing comma is not the root", "core,", "1 file(s): control 2 passing"}, + } { + t.Run(tc.name, func(t *testing.T) { + var out, errOut bytes.Buffer + if code := main1([]string{"-root", root, "-move", "none", "-dir", tc.dir, engine, engine}, &out, &errOut); code != 0 { + t.Fatalf("exit = %d, want 0; err %q", code, errOut.String()) + } + if !strings.Contains(out.String(), tc.want) { + t.Errorf("-dir %q: want %q in\n%s", tc.dir, tc.want, out.String()) + } + }) + } +} + func fakeEngine(t *testing.T, dir, name, body string) string { t.Helper() p := filepath.Join(dir, name) From 51a72ddc4a45007c804ff97b52883fded9e6d4e7 Mon Sep 17 00:00:00 2001 From: tannevaled Date: Tue, 3 Nov 2026 09:28:58 +0100 Subject: [PATCH 2/2] README: -dir names a set, and the refusal names the right one Co-Authored-By: Claude Opus 5 --- scripts/conformance/rubyspec/README.md | 39 ++++++++++++++++++++++++-- 1 file changed, 37 insertions(+), 2 deletions(-) diff --git a/scripts/conformance/rubyspec/README.md b/scripts/conformance/rubyspec/README.md index e271e206..2920f71d 100644 --- a/scripts/conformance/rubyspec/README.md +++ b/scripts/conformance/rubyspec/README.md @@ -287,8 +287,10 @@ specdelta: the move 3.4.1 → 4.0.9 touches 49 file(s) that -dir core/set does n core/binding/implicit_parameter_get_spec.rb core/binding/implicit_parameters_spec.rb … - A number from this run would not be a number about the move. Widen -dir, or - pass -partial to say so on purpose — the report is then stamped PARTIAL. + A number from this run would not be a number about the move. Either + -dir core,language + which measures the whole surface, or -partial to say so on purpose — + the report is then stamped PARTIAL. ``` The refusal comes **first** because a `core/` sweep is an hour, and an hour @@ -298,6 +300,39 @@ here` and is **never dropped by `-quiet`** — a delta of zero on such a file is the signature of the defect, not of calm — and the summary reports the move's share apart from the engines'. +### ⛔ `-dir` names a set, because the refusal's advice has to be followable + +That refusal first read *"Widen `-dir`"*, and `-dir` took **one** directory. +The surface of the move this repository is actually making spans **two** suites +— `core/` and `language/` — so from `-dir core` the only widening available was +the whole corpus: `library/`, `optional/`, `command_line/`, `security/`, and +hours of sweep over files the move never touches. + +That left `-partial` as the only reachable exit, which would have stamped the +headline figure for #799 `PARTIAL` not because the surface was incomplete but +because the flag could not express a complete one. **A derogation that is the +normal road is not a derogation**, and a gate whose honest exit nobody can +reach is not a gate. + +So `-dir` is comma-separated, and the refusal computes the `-dir` that *would* +cover the surface from the census rather than telling the reader to widen: + +```sh +specdelta -root ~/ruby-spec-corpus -move 3.4.1:4.0.9 -dir core,language \ + -repeat 3 -quiet ./rbgo-341 ./rbgo-409 +``` + +Top-level suites and no deeper, on purpose: the 50 exact directories would be +unreadable and would go stale the moment the corpus moves, while +`core,language` is both the honest surface and something a reader can retype. +It reaches `core/` and `language/` and **not** `library/` — a suggestion that +over-reached would re-create the problem it fixes. + +A file reached through two named directories (`-dir core,core/set`) is measured +**once**. Counted twice it would be counted on both sides, which makes the +totals move with how the caller spelled `-dir` rather than with the engines. +And `-dir core,` does not quietly become the whole corpus. + **⛔ Provenance, and it is a repeat offence.** On 2026-11-01 `pass=118 fail=1` was published for [#858](https://github.com/go-embedded-ruby/ruby/pull/858) from a binary built **before that branch existed**; it carried the previous branch's