diff --git a/internal/vm/marshal_codec.go b/internal/vm/marshal_codec.go index 7f865c00..7d334435 100644 --- a/internal/vm/marshal_codec.go +++ b/internal/vm/marshal_codec.go @@ -253,8 +253,49 @@ func (d *mDumper) writeHash(h *object.Hash) { } } +// encodingASCIICompatible reports whether an encoding name denotes an +// ASCII-compatible encoding — one whose ASCII bytes stand for ASCII characters. +// The wide Unicode transformation formats (UTF-16/UTF-32, in any endianness) are +// the exceptions Marshal cares about: a Regexp over such a source keeps that +// encoding even when every byte happens to be ASCII. +func encodingASCIICompatible(name string) bool { + u := strings.ToUpper(name) + return !strings.HasPrefix(u, "UTF-16") && !strings.HasPrefix(u, "UTF-32") +} + +// regexpMarshalFixed reports whether the Regexp is dumped with the +// ARG_ENCODING_FIXED (0x10) option bit set. MRI sets it when FIXEDENCODING was +// requested or the source is otherwise tied to a concrete encoding: a byte +// outside ASCII, or an ASCII-incompatible source encoding (UTF-16/UTF-32) even +// with ASCII-only bytes. A NOENCODING (/n) Regexp is not fixed. +func regexpMarshalFixed(r *Regexp) bool { + if r.fixedEnc { + return true + } + if r.noEnc { + return false + } + if r.srcEnc != "" && !encodingASCIICompatible(r.srcEnc) { + return true + } + return !marshalIsASCII(r.source) +} + +// regexpMarshalEnc returns the encoding name Marshal records for the Regexp, +// which is the encoding of the compiled Regexp (#encoding). When the Regexp is +// fixed to a concrete source encoding, that encoding is recorded even for an +// ASCII-only source (so a Windows-1251 or UTF-16 Regexp keeps its encoding); +// otherwise #encoding's own rule applies (US-ASCII for an ASCII-only source, the +// source encoding for a binary or wide one). An "ASCII-8BIT" result is dumped +// bare, with no encoding ivar. +func regexpMarshalEnc(r *Regexp) string { + if regexpMarshalFixed(r) && r.srcEnc != "" { + return r.srcEnc + } + return r.encodingName() +} + func (d *mDumper) writeRegexp(r *Regexp) { - nonASCII := !marshalIsASCII(r.source) opt := 0 if strings.ContainsRune(r.flags, 'i') { opt |= reIgnoreCase @@ -265,18 +306,35 @@ func (d *mDumper) writeRegexp(r *Regexp) { if strings.ContainsRune(r.flags, 'm') { opt |= reMultiline } - if nonASCII { - opt |= 16 // ARG_ENCODING_FIXED, set for a non-ASCII (UTF-8) source + if regexpMarshalFixed(r) { + opt |= reFixedEncoding + } + if r.noEnc { + opt |= reNoEncoding + } + enc := regexpMarshalEnc(r) + if enc == "ASCII-8BIT" { + // A binary (ASCII-8BIT) Regexp carries no encoding ivar, so it is emitted + // bare — no 'I' wrapper. + d.buf = append(d.buf, '/') + d.writeBytes(r.source) + d.buf = append(d.buf, byte(opt)) + return } d.buf = append(d.buf, 'I', '/') d.writeBytes(r.source) d.buf = append(d.buf, byte(opt)) d.writeLong(1) - d.writeSymbol("E") - if nonASCII { + switch enc { + case "UTF-8": + d.writeSymbol("E") d.buf = append(d.buf, 'T') - } else { + case "US-ASCII": + d.writeSymbol("E") d.buf = append(d.buf, 'F') + default: + d.writeSymbol("encoding") + d.writeValue(object.NewStringBytesEnc([]byte(enc), "ASCII-8BIT")) } } @@ -386,6 +444,20 @@ func (d *mDumper) writeObject(o *RObject) { } return } + // A Data (Ruby 3.2+ immutable value object, minted by Data.define) marshals + // with the same 'S' container as a Struct — the class name followed by each + // member name/value pair — because MRI stores a Data's members the same way it + // stores a Struct's. The real class name is used (never a #name override). + if dd := dataDefOf(o.class); dd != nil { + d.buf = append(d.buf, 'S') + d.writeSymbol(o.class.name) + d.writeLong(len(dd.names)) + for i, name := range dd.names { + d.writeSymbol(name) + d.writeValue(o.structVals[i]) + } + return + } // An instance of a user subclass of a built-in value type (Array/Hash/ // String/Range/...) dumps as 'C' — the class name followed by the wrapped // built-in value. Any instance variables are carried by an outer 'I' wrapper, @@ -792,7 +864,20 @@ func (r *mReader) applyIvar(base object.Value, name string, val object.Value) { } } case *Regexp: - // :E only refines the source encoding; the compiled Regexp needs nothing. + // The encoding ivar records the Regexp's source encoding, which #encoding + // reports (and a re-dump reproduces); the compiled matcher is unaffected. + switch name { + case "E": + if val.Truthy() { + b.srcEnc = "UTF-8" + } else { + b.srcEnc = "US-ASCII" + } + case "encoding": + if s, ok := val.(*object.String); ok { + b.srcEnc = s.Str() + } + } default: setIvar(base, name, val) } @@ -899,22 +984,30 @@ func (r *mReader) readRange() object.Value { func (r *mReader) readStruct() object.Value { className := r.readSymbol() cls := r.vm.marshalClass(className) - sd := structDefOf(cls) - if sd == nil { + // The 'S' container carries both Struct and Data (Ruby 3.2+ immutable value) + // instances, whose members are stored the same way; memberDefNames resolves + // either layout. A Data loads as a frozen instance, matching MRI's immutability + // (independent of the freeze: kwarg). + names, ok := memberDefNames(cls) + if !ok { raise("TypeError", "%s is not a Struct", className) } - o := &RObject{class: cls, ivars: map[string]object.Value{}, structVals: make([]object.Value, len(sd.names))} + isData := dataDefOf(cls) != nil + o := &RObject{class: cls, ivars: map[string]object.Value{}, structVals: make([]object.Value, len(names))} r.register(o) n := r.long() for i := 0; i < n; i++ { name := r.readSymbol() val := r.readValue() - for idx, m := range sd.names { + for idx, m := range names { if m == name { o.structVals[idx] = val } } } + if isData { + o.frozen = true + } return o } @@ -992,6 +1085,14 @@ func (r *mReader) readRegexp(_ bool) object.Value { flags += "x" } re := r.vm.compileRegexp(source, flags) + if rx, ok := re.(*Regexp); ok { + // A non-ASCII source is already treated as fixed-encoding by its content, + // exactly as a source literal is, so recording the FIXED bit for it would + // make a loaded /café/ unequal to the literal /café/ (whose fixedEnc field + // is false). Only an ASCII-only source needs the bit carried explicitly. + rx.fixedEnc = opt&reFixedEncoding != 0 && marshalIsASCII(source) + rx.noEnc = opt&reNoEncoding != 0 + } return r.register(re) } diff --git a/internal/vm/marshal_residuals_test.go b/internal/vm/marshal_residuals_test.go new file mode 100644 index 00000000..776a9de5 --- /dev/null +++ b/internal/vm/marshal_residuals_test.go @@ -0,0 +1,90 @@ +package vm_test + +import ( + "strings" + "testing" +) + +// TestMarshalResiduals covers the Marshal conformance residuals fixed on top of +// the earlier passes: Data (Ruby 3.2+ immutable value objects) dump/load via the +// 'S' container, and the Regexp encoding model (the option byte's +// FIXEDENCODING / NOENCODING bits and the recorded source encoding). Every +// byte-exact expectation was captured from MRI Ruby 4.0 +// (`ruby -e 'p Marshal.dump(x).bytes'`). +func TestMarshalResiduals(t *testing.T) { + cases := []struct{ src, want string }{ + // --- Data: 'S' container, member name/value pairs ----------------- + {`M = Data.define(:amount, :unit); p Marshal.dump(M.new(100, "km".b)).bytes`, + "[4, 8, 83, 58, 6, 77, 7, 58, 11, 97, 109, 111, 117, 110, 116, 105, 105, 58, 9, 117, 110, 105, 116, 34, 7, 107, 109]\n"}, + // A Data round-trips to a frozen instance with its members restored. + {`MB = Data.define(:a, :b); v = Marshal.load(Marshal.dump(MB.new(1, 2))); p [v.a, v.b]`, "[1, 2]\n"}, + {`MF = Data.define(:a); p Marshal.load(Marshal.dump(MF.new(1))).frozen?`, "true\n"}, + // A plain subclass of a Data.define class shares the parent's members. + {`MP = Data.define(:a); class MPSub < MP; end; v = Marshal.load(Marshal.dump(MPSub.new(9))); p [v.class.name, v.a]`, + "[\"MPSub\", 9]\n"}, + // The real class name is used, never a #name override. + {`MN = Data.define(:a); def MN.name; "Nope"; end; d = Marshal.dump(MN.new(1)); p [d.include?("MN"), d.include?("Nope")]`, "[true, false]\n"}, + + // --- Regexp: encoding + option byte ------------------------------- + // Binary (ASCII-8BIT) source: dumped bare, no encoding ivar, FIXED bit set. + {`p Marshal.dump(Regexp.new("".b, Regexp::FIXEDENCODING)).bytes`, "[4, 8, 47, 0, 16]\n"}, + // US-ASCII source, FIXEDENCODING: E => false, FIXED bit set. + {`p Marshal.dump(Regexp.new("a".encode("us-ascii"), Regexp::FIXEDENCODING)).bytes`, + "[4, 8, 73, 47, 6, 97, 16, 6, 58, 6, 69, 70]\n"}, + // US-ASCII source, not fixed: E => false, no FIXED bit. + {`p Marshal.dump(Regexp.new("a".encode("us-ascii"))).bytes`, + "[4, 8, 73, 47, 6, 97, 0, 6, 58, 6, 69, 70]\n"}, + // Windows-1251 (ASCII-compatible) source, FIXEDENCODING: named :encoding ivar. + {`p Marshal.dump(Regexp.new("a".encode("windows-1251"), Regexp::FIXEDENCODING)).bytes`, + "[4, 8, 73, 47, 6, 97, 16, 6, 58, 13, 101, 110, 99, 111, 100, 105, 110, 103, 34, 17, 87, 105, 110, 100, 111, 119, 115, 45, 49, 50, 53, 49]\n"}, + // UTF-8 source, FIXEDENCODING: E => true. + {`p Marshal.dump(Regexp.new("".dup.force_encoding("utf-8"), Regexp::FIXEDENCODING)).bytes`, + "[4, 8, 73, 47, 0, 16, 6, 58, 6, 69, 84]\n"}, + // UTF-16LE (ASCII-incompatible) source: :encoding ivar, FIXED bit. + {`p Marshal.dump(Regexp.new("a".encode("utf-16le"), Regexp::FIXEDENCODING)).bytes`, + "[4, 8, 73, 47, 7, 97, 0, 16, 6, 58, 13, 101, 110, 99, 111, 100, 105, 110, 103, 34, 13, 85, 84, 70, 45, 49, 54, 76, 69]\n"}, + // UTF-32LE source, NOT fixed by the caller: an ASCII-incompatible source + // is fixed anyway, so FIXED is set and the encoding is recorded. + {`p Marshal.dump(Regexp.new("a".encode("utf-32le"))).bytes`, + "[4, 8, 73, 47, 9, 97, 0, 0, 0, 16, 6, 58, 13, 101, 110, 99, 111, 100, 105, 110, 103, 34, 13, 85, 84, 70, 45, 51, 50, 76, 69]\n"}, + // A UTF-8 literal with a non-ASCII byte is implicitly fixed: FIXED bit, E => true. + {`p Marshal.dump(Regexp.new("café".dup.force_encoding("utf-8"))).bytes`, + "[4, 8, 73, 47, 10, 99, 97, 102, 195, 169, 16, 6, 58, 6, 69, 84]\n"}, + // An ASCII-only literal: US-ASCII, no FIXED bit. + {`p Marshal.dump(/\A.\Z/).bytes`, "[4, 8, 73, 47, 10, 92, 65, 46, 92, 90, 0, 6, 58, 6, 69, 70]\n"}, + // Flag letters map to the low option bits (i|m = 5); still US-ASCII. + {`p Marshal.dump(//im).bytes`, "[4, 8, 73, 47, 0, 5, 6, 58, 6, 69, 70]\n"}, + // NOENCODING over an ASCII-only source: encoding stays US-ASCII (E => false), + // only the NONE option bit (32) is set — not FIXED. + {`p Marshal.dump(Regexp.new("a", Regexp::NOENCODING)).bytes`, + "[4, 8, 73, 47, 6, 97, 32, 6, 58, 6, 69, 70]\n"}, + + // --- Regexp reader: options + encoding survive a round-trip ------- + // dump -> load -> dump is byte-stable for each encoding shape, which + // exercises the reader restoring the FIXED / NONE option bits and the + // source encoding from the E / :encoding ivar. + {`re = Regexp.new("a".encode("us-ascii"), Regexp::FIXEDENCODING); p Marshal.dump(Marshal.load(Marshal.dump(re))).bytes == Marshal.dump(re).bytes`, "true\n"}, + {`re = Regexp.new("café".dup.force_encoding("utf-8"), Regexp::FIXEDENCODING); p Marshal.dump(Marshal.load(Marshal.dump(re))).bytes == Marshal.dump(re).bytes`, "true\n"}, + {`re = Regexp.new("a".encode("windows-1251"), Regexp::FIXEDENCODING); p Marshal.dump(Marshal.load(Marshal.dump(re))).bytes == Marshal.dump(re).bytes`, "true\n"}, + {`re = Regexp.new("a".encode("utf-16le"), Regexp::FIXEDENCODING); p Marshal.dump(Marshal.load(Marshal.dump(re))).bytes == Marshal.dump(re).bytes`, "true\n"}, + {`re = Regexp.new("a".encode("utf-32le")); p Marshal.dump(Marshal.load(Marshal.dump(re))).bytes == Marshal.dump(re).bytes`, "true\n"}, + {`re = Regexp.new("a", Regexp::NOENCODING); p Marshal.dump(Marshal.load(Marshal.dump(re))).bytes == Marshal.dump(re).bytes`, "true\n"}, + {`re = /\A.\Z/; p Marshal.dump(Marshal.load(Marshal.dump(re))).bytes == Marshal.dump(re).bytes`, "true\n"}, + } + for _, c := range cases { + if got := eval(t, c.src); got != c.want { + t.Errorf("src=%q got=%q want=%q", c.src, got, c.want) + } + } + + errs := []struct{ src, want string }{ + // An 'S' container naming a class with no member layout is rejected, the + // same as it is for a non-Struct (here a plain object class). + {`class NotAData; end; Marshal.load("\x04\bS:\rNotAData\x00")`, "is not a Struct"}, + } + for _, c := range errs { + if err := runErr(t, c.src); err == nil || !strings.Contains(err.Error(), c.want) { + t.Errorf("src=%q got=%v want error containing %q", c.src, err, c.want) + } + } +}