Advance Marshal conformance: Data dump/load and Regexp encoding

internal/vm/marshal_codec.go

@@ -253,8 +253,49 @@ func (d *mDumper) writeHash(h *object.Hash) {
 	}
 }
 
+// encodingASCIICompatible reports whether an encoding name denotes an
+// ASCII-compatible encoding — one whose ASCII bytes stand for ASCII characters.
+// The wide Unicode transformation formats (UTF-16/UTF-32, in any endianness) are
+// the exceptions Marshal cares about: a Regexp over such a source keeps that
+// encoding even when every byte happens to be ASCII.
+func encodingASCIICompatible(name string) bool {
+	u := strings.ToUpper(name)
+	return !strings.HasPrefix(u, "UTF-16") && !strings.HasPrefix(u, "UTF-32")
+}
+
+// regexpMarshalFixed reports whether the Regexp is dumped with the
+// ARG_ENCODING_FIXED (0x10) option bit set. MRI sets it when FIXEDENCODING was
+// requested or the source is otherwise tied to a concrete encoding: a byte
+// outside ASCII, or an ASCII-incompatible source encoding (UTF-16/UTF-32) even
+// with ASCII-only bytes. A NOENCODING (/n) Regexp is not fixed.
+func regexpMarshalFixed(r *Regexp) bool {
+	if r.fixedEnc {
+		return true
+	}
+	if r.noEnc {
+		return false
+	}
+	if r.srcEnc != "" && !encodingASCIICompatible(r.srcEnc) {
+		return true
+	}
+	return !marshalIsASCII(r.source)
+}
+
+// regexpMarshalEnc returns the encoding name Marshal records for the Regexp,
+// which is the encoding of the compiled Regexp (#encoding). When the Regexp is
+// fixed to a concrete source encoding, that encoding is recorded even for an
+// ASCII-only source (so a Windows-1251 or UTF-16 Regexp keeps its encoding);
+// otherwise #encoding's own rule applies (US-ASCII for an ASCII-only source, the
+// source encoding for a binary or wide one). An "ASCII-8BIT" result is dumped
+// bare, with no encoding ivar.
+func regexpMarshalEnc(r *Regexp) string {
+	if regexpMarshalFixed(r) && r.srcEnc != "" {
+		return r.srcEnc
+	}
+	return r.encodingName()
+}
+
 func (d *mDumper) writeRegexp(r *Regexp) {
-	nonASCII := !marshalIsASCII(r.source)
 	opt := 0
 	if strings.ContainsRune(r.flags, 'i') {
 		opt |= reIgnoreCase
@@ -265,18 +306,35 @@ func (d *mDumper) writeRegexp(r *Regexp) {
 	if strings.ContainsRune(r.flags, 'm') {
 		opt |= reMultiline
 	}
-	if nonASCII {
-		opt |= 16 // ARG_ENCODING_FIXED, set for a non-ASCII (UTF-8) source
+	if regexpMarshalFixed(r) {
+		opt |= reFixedEncoding
+	}
+	if r.noEnc {
+		opt |= reNoEncoding
+	}
+	enc := regexpMarshalEnc(r)
+	if enc == "ASCII-8BIT" {
+		// A binary (ASCII-8BIT) Regexp carries no encoding ivar, so it is emitted
+		// bare — no 'I' wrapper.
+		d.buf = append(d.buf, '/')
+		d.writeBytes(r.source)
+		d.buf = append(d.buf, byte(opt))
+		return
 	}
 	d.buf = append(d.buf, 'I', '/')
 	d.writeBytes(r.source)
 	d.buf = append(d.buf, byte(opt))
 	d.writeLong(1)
-	d.writeSymbol("E")
-	if nonASCII {
+	switch enc {
+	case "UTF-8":
+		d.writeSymbol("E")
 		d.buf = append(d.buf, 'T')
-	} else {
+	case "US-ASCII":
+		d.writeSymbol("E")
 		d.buf = append(d.buf, 'F')
+	default:
+		d.writeSymbol("encoding")
+		d.writeValue(object.NewStringBytesEnc([]byte(enc), "ASCII-8BIT"))
 	}
 }
 
@@ -386,6 +444,20 @@ func (d *mDumper) writeObject(o *RObject) {
 		}
 		return
 	}
+	// A Data (Ruby 3.2+ immutable value object, minted by Data.define) marshals
+	// with the same 'S' container as a Struct — the class name followed by each
+	// member name/value pair — because MRI stores a Data's members the same way it
+	// stores a Struct's. The real class name is used (never a #name override).
+	if dd := dataDefOf(o.class); dd != nil {
+		d.buf = append(d.buf, 'S')
+		d.writeSymbol(o.class.name)
+		d.writeLong(len(dd.names))
+		for i, name := range dd.names {
+			d.writeSymbol(name)
+			d.writeValue(o.structVals[i])
+		}
+		return
+	}
 	// An instance of a user subclass of a built-in value type (Array/Hash/
 	// String/Range/...) dumps as 'C' — the class name followed by the wrapped
 	// built-in value. Any instance variables are carried by an outer 'I' wrapper,
@@ -792,7 +864,20 @@ func (r *mReader) applyIvar(base object.Value, name string, val object.Value) {
 			}
 		}
 	case *Regexp:
-		// :E only refines the source encoding; the compiled Regexp needs nothing.
+		// The encoding ivar records the Regexp's source encoding, which #encoding
+		// reports (and a re-dump reproduces); the compiled matcher is unaffected.
+		switch name {
+		case "E":
+			if val.Truthy() {
+				b.srcEnc = "UTF-8"
+			} else {
+				b.srcEnc = "US-ASCII"
+			}
+		case "encoding":
+			if s, ok := val.(*object.String); ok {
+				b.srcEnc = s.Str()
+			}
+		}
 	default:
 		setIvar(base, name, val)
 	}
@@ -899,22 +984,30 @@ func (r *mReader) readRange() object.Value {
 func (r *mReader) readStruct() object.Value {
 	className := r.readSymbol()
 	cls := r.vm.marshalClass(className)
-	sd := structDefOf(cls)
-	if sd == nil {
+	// The 'S' container carries both Struct and Data (Ruby 3.2+ immutable value)
+	// instances, whose members are stored the same way; memberDefNames resolves
+	// either layout. A Data loads as a frozen instance, matching MRI's immutability
+	// (independent of the freeze: kwarg).
+	names, ok := memberDefNames(cls)
+	if !ok {
 		raise("TypeError", "%s is not a Struct", className)
 	}
-	o := &RObject{class: cls, ivars: map[string]object.Value{}, structVals: make([]object.Value, len(sd.names))}
+	isData := dataDefOf(cls) != nil
+	o := &RObject{class: cls, ivars: map[string]object.Value{}, structVals: make([]object.Value, len(names))}
 	r.register(o)
 	n := r.long()
 	for i := 0; i < n; i++ {
 		name := r.readSymbol()
 		val := r.readValue()
-		for idx, m := range sd.names {
+		for idx, m := range names {
 			if m == name {
 				o.structVals[idx] = val
 			}
 		}
 	}
+	if isData {
+		o.frozen = true
+	}
 	return o
 }
 
@@ -992,6 +1085,14 @@ func (r *mReader) readRegexp(_ bool) object.Value {
 		flags += "x"
 	}
 	re := r.vm.compileRegexp(source, flags)
+	if rx, ok := re.(*Regexp); ok {
+		// A non-ASCII source is already treated as fixed-encoding by its content,
+		// exactly as a source literal is, so recording the FIXED bit for it would
+		// make a loaded /café/ unequal to the literal /café/ (whose fixedEnc field
+		// is false). Only an ASCII-only source needs the bit carried explicitly.
+		rx.fixedEnc = opt&reFixedEncoding != 0 && marshalIsASCII(source)
+		rx.noEnc = opt&reNoEncoding != 0
+	}
 	return r.register(re)
 }
 

internal/vm/marshal_residuals_test.go

@@ -0,0 +1,90 @@
+package vm_test
+
+import (
+	"strings"
+	"testing"
+)
+
+// TestMarshalResiduals covers the Marshal conformance residuals fixed on top of
+// the earlier passes: Data (Ruby 3.2+ immutable value objects) dump/load via the
+// 'S' container, and the Regexp encoding model (the option byte's
+// FIXEDENCODING / NOENCODING bits and the recorded source encoding). Every
+// byte-exact expectation was captured from MRI Ruby 4.0
+// (`ruby -e 'p Marshal.dump(x).bytes'`).
+func TestMarshalResiduals(t *testing.T) {
+	cases := []struct{ src, want string }{
+		// --- Data: 'S' container, member name/value pairs -----------------
+		{`M = Data.define(:amount, :unit); p Marshal.dump(M.new(100, "km".b)).bytes`,
+			"[4, 8, 83, 58, 6, 77, 7, 58, 11, 97, 109, 111, 117, 110, 116, 105, 105, 58, 9, 117, 110, 105, 116, 34, 7, 107, 109]\n"},
+		// A Data round-trips to a frozen instance with its members restored.
+		{`MB = Data.define(:a, :b); v = Marshal.load(Marshal.dump(MB.new(1, 2))); p [v.a, v.b]`, "[1, 2]\n"},
+		{`MF = Data.define(:a); p Marshal.load(Marshal.dump(MF.new(1))).frozen?`, "true\n"},
+		// A plain subclass of a Data.define class shares the parent's members.
+		{`MP = Data.define(:a); class MPSub < MP; end; v = Marshal.load(Marshal.dump(MPSub.new(9))); p [v.class.name, v.a]`,
+			"[\"MPSub\", 9]\n"},
+		// The real class name is used, never a #name override.
+		{`MN = Data.define(:a); def MN.name; "Nope"; end; d = Marshal.dump(MN.new(1)); p [d.include?("MN"), d.include?("Nope")]`, "[true, false]\n"},
+
+		// --- Regexp: encoding + option byte -------------------------------
+		// Binary (ASCII-8BIT) source: dumped bare, no encoding ivar, FIXED bit set.
+		{`p Marshal.dump(Regexp.new("".b, Regexp::FIXEDENCODING)).bytes`, "[4, 8, 47, 0, 16]\n"},
+		// US-ASCII source, FIXEDENCODING: E => false, FIXED bit set.
+		{`p Marshal.dump(Regexp.new("a".encode("us-ascii"), Regexp::FIXEDENCODING)).bytes`,
+			"[4, 8, 73, 47, 6, 97, 16, 6, 58, 6, 69, 70]\n"},
+		// US-ASCII source, not fixed: E => false, no FIXED bit.
+		{`p Marshal.dump(Regexp.new("a".encode("us-ascii"))).bytes`,
+			"[4, 8, 73, 47, 6, 97, 0, 6, 58, 6, 69, 70]\n"},
+		// Windows-1251 (ASCII-compatible) source, FIXEDENCODING: named :encoding ivar.
+		{`p Marshal.dump(Regexp.new("a".encode("windows-1251"), Regexp::FIXEDENCODING)).bytes`,
+			"[4, 8, 73, 47, 6, 97, 16, 6, 58, 13, 101, 110, 99, 111, 100, 105, 110, 103, 34, 17, 87, 105, 110, 100, 111, 119, 115, 45, 49, 50, 53, 49]\n"},
+		// UTF-8 source, FIXEDENCODING: E => true.
+		{`p Marshal.dump(Regexp.new("".dup.force_encoding("utf-8"), Regexp::FIXEDENCODING)).bytes`,
+			"[4, 8, 73, 47, 0, 16, 6, 58, 6, 69, 84]\n"},
+		// UTF-16LE (ASCII-incompatible) source: :encoding ivar, FIXED bit.
+		{`p Marshal.dump(Regexp.new("a".encode("utf-16le"), Regexp::FIXEDENCODING)).bytes`,
+			"[4, 8, 73, 47, 7, 97, 0, 16, 6, 58, 13, 101, 110, 99, 111, 100, 105, 110, 103, 34, 13, 85, 84, 70, 45, 49, 54, 76, 69]\n"},
+		// UTF-32LE source, NOT fixed by the caller: an ASCII-incompatible source
+		// is fixed anyway, so FIXED is set and the encoding is recorded.
+		{`p Marshal.dump(Regexp.new("a".encode("utf-32le"))).bytes`,
+			"[4, 8, 73, 47, 9, 97, 0, 0, 0, 16, 6, 58, 13, 101, 110, 99, 111, 100, 105, 110, 103, 34, 13, 85, 84, 70, 45, 51, 50, 76, 69]\n"},
+		// A UTF-8 literal with a non-ASCII byte is implicitly fixed: FIXED bit, E => true.
+		{`p Marshal.dump(Regexp.new("café".dup.force_encoding("utf-8"))).bytes`,
+			"[4, 8, 73, 47, 10, 99, 97, 102, 195, 169, 16, 6, 58, 6, 69, 84]\n"},
+		// An ASCII-only literal: US-ASCII, no FIXED bit.
+		{`p Marshal.dump(/\A.\Z/).bytes`, "[4, 8, 73, 47, 10, 92, 65, 46, 92, 90, 0, 6, 58, 6, 69, 70]\n"},
+		// Flag letters map to the low option bits (i|m = 5); still US-ASCII.
+		{`p Marshal.dump(//im).bytes`, "[4, 8, 73, 47, 0, 5, 6, 58, 6, 69, 70]\n"},
+		// NOENCODING over an ASCII-only source: encoding stays US-ASCII (E => false),
+		// only the NONE option bit (32) is set — not FIXED.
+		{`p Marshal.dump(Regexp.new("a", Regexp::NOENCODING)).bytes`,
+			"[4, 8, 73, 47, 6, 97, 32, 6, 58, 6, 69, 70]\n"},
+
+		// --- Regexp reader: options + encoding survive a round-trip -------
+		// dump -> load -> dump is byte-stable for each encoding shape, which
+		// exercises the reader restoring the FIXED / NONE option bits and the
+		// source encoding from the E / :encoding ivar.
+		{`re = Regexp.new("a".encode("us-ascii"), Regexp::FIXEDENCODING); p Marshal.dump(Marshal.load(Marshal.dump(re))).bytes == Marshal.dump(re).bytes`, "true\n"},
+		{`re = Regexp.new("café".dup.force_encoding("utf-8"), Regexp::FIXEDENCODING); p Marshal.dump(Marshal.load(Marshal.dump(re))).bytes == Marshal.dump(re).bytes`, "true\n"},
+		{`re = Regexp.new("a".encode("windows-1251"), Regexp::FIXEDENCODING); p Marshal.dump(Marshal.load(Marshal.dump(re))).bytes == Marshal.dump(re).bytes`, "true\n"},
+		{`re = Regexp.new("a".encode("utf-16le"), Regexp::FIXEDENCODING); p Marshal.dump(Marshal.load(Marshal.dump(re))).bytes == Marshal.dump(re).bytes`, "true\n"},
+		{`re = Regexp.new("a".encode("utf-32le")); p Marshal.dump(Marshal.load(Marshal.dump(re))).bytes == Marshal.dump(re).bytes`, "true\n"},
+		{`re = Regexp.new("a", Regexp::NOENCODING); p Marshal.dump(Marshal.load(Marshal.dump(re))).bytes == Marshal.dump(re).bytes`, "true\n"},
+		{`re = /\A.\Z/; p Marshal.dump(Marshal.load(Marshal.dump(re))).bytes == Marshal.dump(re).bytes`, "true\n"},
+	}
+	for _, c := range cases {
+		if got := eval(t, c.src); got != c.want {
+			t.Errorf("src=%q got=%q want=%q", c.src, got, c.want)
+		}
+	}
+
+	errs := []struct{ src, want string }{
+		// An 'S' container naming a class with no member layout is rejected, the
+		// same as it is for a non-Struct (here a plain object class).
+		{`class NotAData; end; Marshal.load("\x04\bS:\rNotAData\x00")`, "is not a Struct"},
+	}
+	for _, c := range errs {
+		if err := runErr(t, c.src); err == nil || !strings.Contains(err.Error(), c.want) {
+			t.Errorf("src=%q got=%v want error containing %q", c.src, err, c.want)
+		}
+	}
+}