diff --git a/internal/vm/converter_impl.go b/internal/vm/converter_impl.go index c86e3ca4..443b5627 100644 --- a/internal/vm/converter_impl.go +++ b/internal/vm/converter_impl.go @@ -330,10 +330,10 @@ func (vm *VM) converterConvert(c *converterObj, v object.Value) object.Value { case "invalid_byte_sequence": vm.setConvError(c, "invalid_byte_sequence", errBytes, readAgain, false) c.readAgain = append([]byte(nil), readAgain...) - raise("Encoding::InvalidByteSequenceError", "%s", c.invalidMessage(errBytes, readAgain)) + vm.raiseEconvError(c.lastErrExc) case "undefined_conversion": vm.setConvError(c, "undefined_conversion", errBytes, nil, true) - raise("Encoding::UndefinedConversionError", "%s", c.undefMessage(errBytes)) + vm.raiseEconvError(c.lastErrExc) case "incomplete_input": // Hold the incomplete tail; #finish reports it. #convert itself succeeds. c.pendingIncomplete = errBytes @@ -355,8 +355,7 @@ func (vm *VM) converterFinish(c *converterObj) object.Value { errBytes := c.pendingIncomplete c.pendingIncomplete = nil vm.setConvError(c, "incomplete_input", errBytes, nil, false) - raise("Encoding::InvalidByteSequenceError", "incomplete %s on %s", - binStr(errBytes).Inspect(), c.src) + vm.raiseEconvError(c.lastErrExc) } c.finished = true return object.NewStringBytesEnc(nil, c.dst) @@ -561,6 +560,12 @@ func (vm *VM) setConvError(c *converterObj, status string, errBytes, readAgain [ case "undefined_conversion": c.lastErrExc = vm.buildException("Encoding::UndefinedConversionError", c.undefMessage(errBytes)) } + // make_econv_exception (transcode.c) does not only pick the class and the + // message: it attaches the error attributes in the same breath, so + // #source_encoding and friends can never disagree with #primitive_errinfo. A + // status that is not one of the three failures leaves setEconvErrorAttrs with + // nothing to attach and it returns without touching the exception. + vm.setEconvErrorAttrs(c.lastErrExc, status, c.errBytes, c.errReadAgain, c.errSrcEnc, c.errDstEnc) } // errinfo builds the #primitive_errinfo tuple [status, src_enc, dst_enc, diff --git a/internal/vm/encoding.go b/internal/vm/encoding.go index 69c23b3f..21b5f5af 100644 --- a/internal/vm/encoding.go +++ b/internal/vm/encoding.go @@ -431,4 +431,85 @@ func (vm *VM) registerEncodingErrors() { vm.cEncoding.consts[n] = c vm.consts["Encoding::"+n] = c } + undef := vm.consts["Encoding::UndefinedConversionError"].(*RClass) + invalid := vm.consts["Encoding::InvalidByteSequenceError"].(*RClass) + // transcode.c: every accessor is a plain rb_attr_get of the ivar + // make_econv_exception stored, so an exception built any other way (a bare + // .new) reports nil rather than raising. + for _, cls := range []*RClass{undef, invalid} { + for _, name := range []string{ + "source_encoding", "source_encoding_name", + "destination_encoding", "destination_encoding_name", + } { + cls.define(name, econvAttrReader("@"+name)) + } + } + undef.define("error_char", econvAttrReader("@error_char")) + invalid.define("error_bytes", econvAttrReader("@error_bytes")) + invalid.define("readagain_bytes", econvAttrReader("@readagain_bytes")) + invalid.define("incomplete_input?", econvAttrReader("@incomplete_input")) +} + +// raiseEconvError raises the very exception object the converter recorded as its +// #last_error. rb_econv_check_error (transcode.c) passes make_econv_exception's +// result straight to rb_exc_raise, so the raised object and #last_error are one +// object carrying one set of attributes; rebuilding a second exception from the +// message alone would raise something that answers nil to every accessor. +func (vm *VM) raiseEconvError(exc object.Value) object.Value { + panic(vm.excError(exc)) +} + +// econvAttrReader is the ecerr_* accessor shape from transcode.c: each of +// #source_encoding, #error_char, #incomplete_input? and their siblings is +// rb_attr_get of one ivar, so an unset one reads as nil (the spec for +// `Encoding::InvalidByteSequenceError.new.incomplete_input?` asserts exactly +// that) instead of raising NameError. +func econvAttrReader(ivar string) NativeFn { + return func(_ *VM, self object.Value, _ []object.Value, _ *Proc) object.Value { + return getIvar(self, ivar) + } +} + +// setEconvErrorAttrs attaches the transcoding-error attributes MRI's +// make_econv_exception (transcode.c) stores on the exception it builds, from the +// converter's last_error state: srcEnc/dstEnc are the encodings of the *hop that +// failed* (not the converter's own endpoints), errBytes the discarded bytes and +// readAgain the bytes to be read again. +// +// - undefined_conversion sets @error_char: the error bytes tagged with the +// failing hop's source encoding, so it is the one character that could not be +// converted (rb_enc_associate_index on the source encoding index). +// - invalid_byte_sequence / incomplete_input set @error_bytes and +// @readagain_bytes as BINARY strings — rb_str_new leaves them ASCII-8BIT — +// with @readagain_bytes nil when readagain_len is 0, plus @incomplete_input. +// +// Both then fall through to set_encs, which records the names unconditionally and +// the Encoding objects only when the name resolves to a registered encoding. +func (vm *VM) setEconvErrorAttrs(exc object.Value, status string, errBytes, readAgain []byte, srcEnc, dstEnc string) { + switch status { + case "undefined_conversion": + ec := object.NewStringBytes(append([]byte(nil), errBytes...)) + if _, ok := vm.findEncoding(srcEnc); ok { + ec.Enc = srcEnc + } + setIvar(exc, "@error_char", ec) + case "invalid_byte_sequence", "incomplete_input": + setIvar(exc, "@error_bytes", binStr(errBytes)) + if len(readAgain) > 0 { + setIvar(exc, "@readagain_bytes", binStr(readAgain)) + } else { + setIvar(exc, "@readagain_bytes", object.NilV) + } + setIvar(exc, "@incomplete_input", object.Bool(status == "incomplete_input")) + default: + return + } + setIvar(exc, "@source_encoding_name", object.NewString(srcEnc)) + setIvar(exc, "@destination_encoding_name", object.NewString(dstEnc)) + if e, ok := vm.findEncoding(srcEnc); ok { + setIvar(exc, "@source_encoding", e) + } + if e, ok := vm.findEncoding(dstEnc); ok { + setIvar(exc, "@destination_encoding", e) + } } diff --git a/internal/vm/encoding_wave34_internal_test.go b/internal/vm/encoding_wave34_internal_test.go new file mode 100644 index 00000000..c3d3b9a4 --- /dev/null +++ b/internal/vm/encoding_wave34_internal_test.go @@ -0,0 +1,333 @@ +// Copyright (c) the go-embedded-ruby/ruby authors +// +// SPDX-License-Identifier: BSD-3-Clause + +// Wave-34 encoding conformance tests. +// +// TestEconvErrorAttributes, TestEncodeSameEncoding, +// TestValidInEncodingUsesTheScanner and TestUnicodeNormalizeEncodingDispatch are +// WITNESSES: each was run against the unfixed product (the behaviour reverted while +// the new symbols stayed, so the mutation still compiled) and each failed. The +// remaining tests in this file cover helpers that only exist with the fix, so they +// are GUARDS against future drift rather than witnesses of it. + +package vm + +import ( + "io" + "strings" + "testing" + + "github.com/go-embedded-ruby/ruby/internal/object" +) + +// TestEconvErrorAttributes covers the transcoding-error attributes MRI's +// make_econv_exception (transcode.c) attaches to the exception it builds, and the +// fact that rb_econv_check_error raises that very object — so the rescued +// exception and Encoding::Converter#last_error are one object with one set of +// attributes. +// +// A/B: every case here fails before the fix with NoMethodError, because the +// accessors did not exist and #convert raised a second, attribute-less exception. +func TestEconvErrorAttributes(t *testing.T) { + cases := []struct{ src, want string }{ + // undefined_conversion: @error_char is the failing hop's bytes tagged with + // that hop's SOURCE encoding, and the names/objects are the hop's, not the + // converter's endpoints. + {`ec = Encoding::Converter.new("utf-8", "ascii") +begin; ec.convert("\u{8765}"); rescue Encoding::UndefinedConversionError => e + p [e.source_encoding, e.source_encoding_name, e.destination_encoding, e.destination_encoding_name] + p [e.error_char, e.error_char.encoding] +end`, `[#, "UTF-8", #, "US-ASCII"] +["蝥", #]`}, + // A multi-hop path reports the hop that failed (ISO-8859-1 -> UTF-8 -> EUC-JP + // fails on the UTF-8 -> EUC-JP leg), which is the case the spec's comment + // singles out. + {`ec = Encoding::Converter.new("ISO-8859-1", "EUC-JP") +begin; ec.convert("\xA0"); rescue Encoding::UndefinedConversionError => e + p [e.source_encoding_name, e.destination_encoding_name, e.error_char.size] +end`, `["UTF-8", "EUC-JP", 1]`}, + // invalid_byte_sequence: @error_bytes and @readagain_bytes are BINARY, and + // @incomplete_input is false. + {`ec = Encoding::Converter.new("utf-8", "iso-8859-1") +begin; ec.convert("\xf1abcd"); rescue Encoding::InvalidByteSequenceError => e + p [e.error_bytes, e.error_bytes.encoding, e.readagain_bytes, e.readagain_bytes.encoding] + p [e.incomplete_input?, e.source_encoding_name, e.destination_encoding_name] +end`, `["\xF1", #, "a", #] +[false, "UTF-8", "ISO-8859-1"]`}, + // incomplete_input reaches the same builder through #finish, where there are + // no read-again bytes, so @readagain_bytes is nil rather than an empty String. + {`ec = Encoding::Converter.new("EUC-JP", "ISO-8859-1") +ec.convert("abc\xA1") +begin; ec.finish; rescue Encoding::InvalidByteSequenceError => e + p [e.incomplete_input?, e.error_bytes, e.readagain_bytes] +end`, `[true, "\xA1", nil]`}, + // The raised object IS #last_error (rb_econv_check_error passes + // make_econv_exception's result straight to rb_exc_raise). + {`ec = Encoding::Converter.new("utf-8", "ascii") +begin; ec.convert("\u{8765}"); rescue => e; p e.equal?(ec.last_error); end`, `true`}, + // Each accessor is rb_attr_get of one ivar, so an exception built any other + // way answers nil instead of raising. + {`e = Encoding::InvalidByteSequenceError.new +p [e.incomplete_input?, e.error_bytes, e.readagain_bytes, e.source_encoding, e.destination_encoding_name]`, + `[nil, nil, nil, nil, nil]`}, + // UndefinedConversionError has no #error_bytes and InvalidByteSequenceError no + // #error_char: transcode.c defines each only on its own class. + {`p Encoding::UndefinedConversionError.new.respond_to?(:error_bytes) +p Encoding::InvalidByteSequenceError.new.respond_to?(:error_char) +p Encoding::UndefinedConversionError.new.respond_to?(:error_char)`, `false +false +true`}, + } + for _, c := range cases { + if got := strings.TrimRight(eval(t, c.src), "\n"); got != c.want { + t.Errorf("src:\n%s\ngot:\n%s\nwant:\n%s", c.src, got, c.want) + } + } +} + +// TestSetEconvErrorAttrsUnknownStatus covers setEconvErrorAttrs's guard directly: a +// status that is not one of the three transcoding failures attaches nothing, so the +// encoding names are not stamped onto an exception that has no error to describe. +// make_econv_exception returns Qnil in that case rather than building anything. +func TestSetEconvErrorAttrsUnknownStatus(t *testing.T) { + vm := New(io.Discard) + exc := vm.buildException("RuntimeError", "not a transcoding error") + vm.setEconvErrorAttrs(exc, "source_buffer_empty", []byte("x"), nil, "UTF-8", "EUC-JP") + if got := getIvar(exc, "@source_encoding_name"); got != object.NilV { + t.Errorf("@source_encoding_name = %v, want nil for a non-error status", got.Inspect()) + } +} + +// TestSetEconvErrorAttrsUnregisteredEncoding covers the set_encs branch that MRI +// guards with `if (0 <= idx)`: an encoding NAME is always recorded, but the Encoding +// OBJECT only when the name resolves to a registered encoding. +func TestSetEconvErrorAttrsUnregisteredEncoding(t *testing.T) { + vm := New(io.Discard) + vm.registerEncodingErrors() + exc := vm.buildException("Encoding::UndefinedConversionError", "synthetic") + vm.setEconvErrorAttrs(exc, "undefined_conversion", []byte("x"), nil, "X-NOT-AN-ENCODING", "Y-NOR-THIS") + if got := getIvar(exc, "@source_encoding_name").ToS(); got != "X-NOT-AN-ENCODING" { + t.Errorf("@source_encoding_name = %q, want the name as given", got) + } + if got := getIvar(exc, "@source_encoding"); got != object.NilV { + t.Errorf("@source_encoding = %v, want nil for an unregistered name", got.Inspect()) + } + if got := getIvar(exc, "@destination_encoding"); got != object.NilV { + t.Errorf("@destination_encoding = %v, want nil for an unregistered name", got.Inspect()) + } + // The error char keeps the receiver's default tag when the source encoding is + // not one rbgo knows, rather than being tagged with a name nothing can resolve. + if ec := getIvar(exc, "@error_char"); ec.ToS() != "x" { + t.Errorf("@error_char = %q, want %q", ec.ToS(), "x") + } +} + +// TestCanonicalEncName covers the registry resolution the transcoding path needs +// because rbgo tags a String with an encoding NAME where MRI stores an index. +func TestCanonicalEncName(t *testing.T) { + vm := New(io.Discard) + cases := map[string]string{ + "iso-8859-9": "ISO-8859-9", + "UTF-8": "UTF-8", + "binary": "ASCII-8BIT", + "eucJP": "EUC-JP", + "X-NOT-AN-ENCODING": "X-NOT-AN-ENCODING", // left as written + } + for in, want := range cases { + if got := vm.canonicalEncName(in); got != want { + t.Errorf("canonicalEncName(%q) = %q, want %q", in, got, want) + } + } +} + +// TestTranscodeOptsHasDecorator covers each decorator str_transcode0 gates its +// same-encoding short-circuits on. +func TestTranscodeOptsHasDecorator(t *testing.T) { + if (transcodeOpts{}).hasDecorator() { + t.Error("plain options report a decorator") + } + for name, o := range map[string]transcodeOpts{ + "cr_newline": {crNewline: true}, + "crlf_newline": {crlfNewline: true}, + "universal_newline": {universalNewline: true}, + "xml": {xml: "text"}, + } { + if !o.hasDecorator() { + t.Errorf("%s: hasDecorator() = false", name) + } + } +} + +// TestEncodeSameEncoding covers the str_transcode0 short-circuits: with matching +// source and destination encodings MRI never looks up a converter, so an explicit +// `invalid:` is a scrub in that one encoding and everything else passes the bytes +// through — even for an encoding rbgo cannot transcode. +// +// A/B: before the fix the first two cases raised +// Encoding::ConverterNotFoundError(Emacs-Mule to Emacs-Mule). +func TestEncodeSameEncoding(t *testing.T) { + cases := []struct{ src, want string }{ + {`p [0x80].pack("C").force_encoding("Emacs-Mule").encode(invalid: :replace)`, `"?"`}, + {`p [0x80].pack("C").force_encoding("Emacs-Mule").encode(invalid: :replace, replace: "!")`, `"!"`}, + // Without `invalid:` the bytes are returned unchanged, ill-formed and all. + {`p [0x80].pack("C").force_encoding("Emacs-Mule").encode.bytes`, `[128]`}, + {`p "abc".force_encoding("Emacs-Mule").encode.encoding`, `#`}, + // A decorator suppresses the short-circuit, so the conversion still runs. + {`p "a\nb".encode("UTF-8", crlf_newline: true).bytes`, `[97, 13, 10, 98]`}, + // An encoding WITH a scanner scrubs on the same path. + {"p \"a\\xE2\".dup.force_encoding(\"UTF-8\").encode(invalid: :replace)", "\"a\uFFFD\""}, + } + for _, c := range cases { + if got := strings.TrimRight(eval(t, c.src), "\n"); got != c.want { + t.Errorf("%s\ngot %s want %s", c.src, got, c.want) + } + } +} + +// TestScrubBytesInWithoutScanner covers scrubBytesIn's early return for an encoding +// rbgo has no character automaton for: the bytes are handed back untouched, which is +// how String#scrub already treats such an encoding. +func TestScrubBytesInWithoutScanner(t *testing.T) { + in := []byte{0x80, 0x41} + got := scrubBytesIn(in, "Big5", transcodeOpts{}) + if string(got) != string(in) { + t.Errorf("scrubBytesIn = %q, want the input unchanged", got) + } +} + +// TestEmacsMuleAutomaton covers the character automaton transcribed from the +// `trans` table of enc/emacs_mule.c, state by state. The three secondary leads take +// different extension ranges, which is the part the grammar comment alone does not +// give. +func TestEmacsMuleAutomaton(t *testing.T) { + cases := []struct { + bytes []byte + n int + valid bool + why string + }{ + {[]byte{0x41}, 1, true, "ASCII is a one-byte character"}, + {[]byte{0x80}, 1, false, "0x80 begins no character"}, + {[]byte{0x9E}, 1, false, "0x9E is neither a lead nor a character"}, + {[]byte{0xA0}, 1, false, "a C byte cannot start a character"}, + {[]byte{0x8F, 0xA0}, 2, true, "PRIMARY_CHAR_1: 0x81..0x8F + one C byte"}, + {[]byte{0x8F, 0x41}, 1, false, "a byte below 0xA0 cannot be C1"}, + {[]byte{0x8F}, 1, false, "input ends mid-character"}, + {[]byte{0x90, 0xA0, 0xA0}, 3, true, "PRIMARY_CHAR_2: 0x90..0x99 + two C bytes"}, + {[]byte{0x90, 0xA0, 0x41}, 2, false, "the maximal ill-formed subpart is what was consumed"}, + {[]byte{0x90, 0x41}, 1, false, "C1 must be a C byte"}, + {[]byte{0x9A, 0xE0, 0xA0}, 3, true, "0x9A/0x9B take an extension in 0xE0..0xEF"}, + {[]byte{0x9A, 0xF0, 0xA0}, 1, false, "0xF0 is out of 0x9A's extension range"}, + {[]byte{0x9C, 0xF0, 0xA0, 0xA0}, 4, true, "0x9C takes an extension in 0xF0..0xF4 and two C bytes"}, + {[]byte{0x9C, 0xF5, 0xA0, 0xA0}, 1, false, "0xF5 is out of 0x9C's extension range"}, + {[]byte{0x9D, 0xF5, 0xA0, 0xA0}, 4, true, "0x9D takes an extension in 0xF5..0xFE"}, + {[]byte{0x9D, 0xFF, 0xA0, 0xA0}, 1, false, "0xFF is out of 0x9D's extension range"}, + {[]byte{0x9C, 0xF0, 0xA0}, 3, false, "a secondary character cut off by the end of input"}, + } + for _, c := range cases { + n, valid := scanEmacsMuleToken(c.bytes) + if n != c.n || valid != c.valid { + t.Errorf("scanEmacsMuleToken(% X) = (%d, %v), want (%d, %v) — %s", + c.bytes, n, valid, c.n, c.valid, c.why) + } + } +} + +// TestValidInEncodingUsesTheScanner covers validInEncoding agreeing with the +// scrubber: rb_enc_str_coderange asks the encoding's own automaton whether the +// string is BROKEN, so the two cannot hold separate opinions. +// +// A/B: before the fix Emacs-Mule had no automaton here and every byte string was +// reported valid. +func TestValidInEncodingUsesTheScanner(t *testing.T) { + cases := []struct { + src string + ok bool + }{ + {`p "\x8F\xA0".dup.force_encoding("Emacs-Mule").valid_encoding?`, true}, + {`p "\x8FA".dup.force_encoding("Emacs-Mule").valid_encoding?`, false}, + {`p "\x9C\xF0\xA0".dup.force_encoding("Emacs-Mule").valid_encoding?`, false}, + {`p "abc".force_encoding("Emacs-Mule").valid_encoding?`, true}, + } + for _, c := range cases { + want := "false\n" + if c.ok { + want = "true\n" + } + if got := eval(t, c.src); got != want { + t.Errorf("%s: got %q want %q", c.src, got, want) + } + } + // An encoding with neither an automaton nor an x/text codec is treated as always + // valid, which is the pre-existing fallback and is asserted here as a guard. + if !validInEncoding([]byte{0xFF}, "X-NOT-AN-ENCODING") { + t.Error("an encoding with no scanner and no codec should read as valid") + } +} + +// TestUnicodeNormalizeEncodingDispatch covers the encoding dispatch at the head of +// UnicodeNormalize.normalize and .normalized? +// (lib/unicode_normalize/normalize.rb). The encoding decides before the bytes are +// looked at, which is why a non-Unicode encoding reports the incompatibility rather +// than "invalid byte sequence in UTF-8". +// +// A/B: before the fix the first three cases returned a value (normalizing the bytes +// as if they were UTF-8) instead of raising. +func TestUnicodeNormalizeEncodingDispatch(t *testing.T) { + cases := []struct{ src, want string }{ + {`begin; [0xE0].pack("C").force_encoding("ISO-8859-1").unicode_normalize(:nfd) +rescue => e; p [e.class, e.message]; end`, + `[Encoding::CompatibilityError, "Unicode Normalization not appropriate for ISO-8859-1"]`}, + {`begin; [0xE0].pack("C").force_encoding("ISO-8859-1").unicode_normalize! +rescue => e; p e.class; end`, `Encoding::CompatibilityError`}, + {`begin; "abc".dup.force_encoding("ISO-8859-1").unicode_normalized? +rescue => e; p e.class; end`, `Encoding::CompatibilityError`}, + // US-ASCII is normalized in every form, and is returned untouched — the bytes + // are never examined, so a high byte does not raise. + {`s = [0xE0].pack("C").force_encoding("US-ASCII") +p [s.unicode_normalized?, s.unicode_normalize.bytes, s.unicode_normalize.encoding]`, + `[true, [224], #]`}, + {`s = +"abc"; s.force_encoding("US-ASCII"); s.unicode_normalize!; p [s, s.encoding]`, + `["abc", #]`}, + // The other Unicode encodings round-trip through UTF-8 and come back in the + // receiver's encoding. + {`s = "ẛ̣".encode("UTF-16LE") +n = s.unicode_normalize(:nfc) +p [n.encoding, n.encode("UTF-8") == "ẛ̣".unicode_normalize(:nfc)]`, + `[#, true]`}, + {`p "ẛ̣".encode("UTF-32BE").unicode_normalized?(:nfd)`, `false`}, + {`s = "ẛ̣".encode("UTF-16BE").dup +s.unicode_normalize!(:nfd) +p [s.encoding, s.encode("UTF-8") == "ẛ̣".unicode_normalize(:nfd)]`, + `[#, true]`}, + // The UTF-8 branch still reports an ill-formed receiver as MRI's gsub does. + {`begin; "a\xE2".dup.unicode_normalize; rescue => e; p [e.class, e.message]; end`, + `[ArgumentError, "invalid byte sequence in UTF-8"]`}, + // GB18030 is in UNICODE_ENCODINGS even though it is not a UTF, so it is + // normalizable rather than incompatible. + {`p "abc".dup.force_encoding("GB18030").unicode_normalized?`, `true`}, + } + for _, c := range cases { + if got := strings.TrimRight(eval(t, c.src), "\n"); got != c.want { + t.Errorf("src:\n%s\ngot:\n%s\nwant:\n%s", c.src, got, c.want) + } + } +} + +// TestUnicodeNormalizeVia covers the UNICODE_ENCODINGS membership test on both +// sides, including the aliases (UCS-2BE, UCS-4BE) that reach it under their +// canonical names. +func TestUnicodeNormalizeVia(t *testing.T) { + vm := New(io.Discard) + for _, name := range []string{"UTF-16BE", "UTF-16LE", "UTF-32BE", "UTF-32LE", "GB18030", "UCS-2BE", "UCS-4BE"} { + if !unicodeNormalizeVia(vm.canonicalEncName(name)) { + t.Errorf("%s should normalize through UTF-8", name) + } + } + for _, name := range []string{"ISO-8859-1", "Shift_JIS", "Big5", "EUC-JP", "ASCII-8BIT"} { + if unicodeNormalizeVia(name) { + t.Errorf("%s should not normalize through UTF-8", name) + } + } +} diff --git a/internal/vm/string_encode.go b/internal/vm/string_encode.go index f31b1929..b3b5245b 100644 --- a/internal/vm/string_encode.go +++ b/internal/vm/string_encode.go @@ -498,27 +498,47 @@ func encodeUTF32(u string, be bool) []byte { // raising — #encode reports an unusable destination as a missing converter, not as // the unknown-encoding ArgumentError that Encoding.find would raise. func (vm *VM) encodeEncName(v object.Value) string { - resolve := func(s string) string { - if e, ok := vm.findEncoding(s); ok { - return e.name - } - return s - } switch e := v.(type) { case *encodingObj: return e.name case *object.String: - return resolve(e.Str()) + return vm.canonicalEncName(e.Str()) } if vm.respondsToDynamic(v, "to_str") { if s, ok := vm.send(v, "to_str", nil, nil).(*object.String); ok { - return resolve(s.Str()) + return vm.canonicalEncName(s.Str()) } } raise("TypeError", "no implicit conversion of %s into String", classNameOf(v)) return "" } +// canonicalEncName resolves an encoding name to the registry's canonical spelling +// ("iso-8859-9" -> "ISO-8859-9", "binary" -> "ASCII-8BIT"), leaving a name the +// registry does not know as written. +// +// MRI needs no such step because a String carries an encoding INDEX, never a name: +// rb_enc_associate_index (encoding.c) resolves the spelling once — at the point a +// `# encoding:` magic comment or #force_encoding is honoured — and every later +// comparison is index equality. rbgo tags a String with the name it was given, so +// a tag has to be resolved before it can key a codec table; otherwise +// `# encoding: iso-8859-9` produces literals that #encoding reports correctly (it +// goes through the registry) yet no converter can be found for. +func (vm *VM) canonicalEncName(name string) string { + if e, ok := vm.findEncoding(name); ok { + return e.name + } + return name +} + +// hasDecorator reports whether any newline or xml decorator is requested. MRI +// gates the same-encoding short-circuits of str_transcode0 (transcode.c) on +// exactly this set: a decorator rewrites the text, so the conversion has to run +// even when the encodings match. +func (o transcodeOpts) hasDecorator() bool { + return o.crNewline || o.crlfNewline || o.universalNewline || o.xml != "" +} + // encAsciiCompat reports whether the registered encoding `name` is // ASCII-compatible; an unregistered name is treated as not compatible. func (vm *VM) encAsciiCompat(name string) bool { @@ -539,8 +559,8 @@ func (vm *VM) stringEncode(s *object.String, args []object.Value) *object.String positional = args } - to := s.EncName() - from := s.EncName() + to := vm.canonicalEncName(s.EncName()) + from := to if len(positional) >= 1 { to = vm.encodeEncName(positional[0]) } else if vm.defInternalEnc != nil { @@ -557,6 +577,21 @@ func (vm *VM) stringEncode(s *object.String, args []object.Value) *object.String repl = defaultReplacement(to) } + // str_transcode0 (transcode.c) short-circuits BEFORE any code converter is + // looked up when the source and destination encodings are the same and no + // decorator is in play: with an explicit `invalid:` the result is + // rb_enc_str_scrub in that one encoding, and otherwise the bytes are returned + // unchanged. So `#encode(invalid: :replace)` cleans up a string in an encoding + // rbgo has no converter for (Emacs-Mule, say) instead of raising + // ConverterNotFoundError — no converter is involved at all. + if from == to && !opts.hasDecorator() { + out := append([]byte(nil), s.Bytes()...) + if opts.invalidReplace { + out = scrubBytesIn(out, to, opts) + } + return object.NewStringBytesEnc(out, to) + } + // A destination rbgo cannot transcode into (a dummy encoding like Emacs-Mule, // or an unregistered name) has no converter. MRI still succeeds when the whole // string is 7-bit ASCII and both encodings are ASCII-compatible — the bytes are diff --git a/internal/vm/string_encoding_ops.go b/internal/vm/string_encoding_ops.go index a2e0818e..58a5716e 100644 --- a/internal/vm/string_encoding_ops.go +++ b/internal/vm/string_encoding_ops.go @@ -43,6 +43,20 @@ func validInEncoding(b []byte, enc string) bool { case "UTF-32BE": return validUTF32(b, true) } + // An encoding with a scrub scanner has a character automaton, and + // rb_enc_str_coderange (string.c) asks that same automaton whether the string is + // BROKEN: validity and scrubbing cannot be allowed to disagree about what a + // well-formed character is, so both read the one scanner. + if scan, ok := scrubScannerFor(enc); ok { + for i := 0; i < len(b); { + n, valid := scan(b[i:]) + if !valid { + return false + } + i += n + } + return true + } if _, ok := xtextEncodings[enc]; ok { return xtextValid(b, enc) } @@ -132,10 +146,118 @@ func scrubScannerFor(enc string) (scrubScan, bool) { return func(b []byte) (int, bool) { return scanUTF32Token(b, false) }, true case "UTF-32BE": return func(b []byte) (int, bool) { return scanUTF32Token(b, true) }, true + case "Emacs-Mule": + return scanEmacsMuleToken, true } return nil, false } +// emacsMuleNext is one step of the Emacs-Mule character automaton transcribed from +// the `trans` table in enc/emacs_mule.c. It returns the next state for byte c, or +// emacsMuleAccept when the character is complete and emacsMuleFail when the byte +// cannot continue it. The grammar the table encodes is stated in that file: +// +// CHARACTER := ASCII_CHAR | MULTIBYTE_CHAR +// PRIMARY_CHAR_1 := LEADING_CODE_PRI C1 (0x81..0x8F + one C byte) +// PRIMARY_CHAR_2 := LEADING_CODE_PRI C1 C2 (0x90..0x99 + two C bytes) +// SECONDARY_CHAR := LEADING_CODE_SEC LEADING_CODE_EXT C1 [C2] +// C1, C2, LEADING_CODE_EXT := 0xA0..0xFF +// +// The three secondary leads take DIFFERENT extension ranges (0x9A/0x9B accept +// 0xE0..0xEF, 0x9C accepts 0xF0..0xF4 and 0x9D accepts 0xF5..0xFE), which is why +// the automaton is transcribed rather than derived from the grammar comment: the +// EncLen_EmacsMule table is only an upper bound on a character's length, and MRI +// judges validity with the automaton (precise_mbc_enc_len), not with EncLen. +// +// The table's S3 is not transcribed: no S0 transition reaches it, so it can never +// be entered for any input. +const ( + emacsMuleAccept = -1 + emacsMuleFail = -2 +) + +func emacsMuleNext(state int, c byte) int { + switch state { + case 0: // S0: the first byte of a character + switch { + case c <= 0x7F: + return emacsMuleAccept + case c >= 0x81 && c <= 0x8F: + return 1 + case c >= 0x90 && c <= 0x99: + return 2 + case c == 0x9A || c == 0x9B: + return 4 + case c == 0x9C: + return 5 + case c == 0x9D: + return 6 + } + case 1: // S1: the last C byte + if c >= 0xA0 { + return emacsMuleAccept + } + case 2: // S2: a C byte with one more to follow + if c >= 0xA0 { + return 1 + } + case 4: // S4: the extension byte after 0x9A / 0x9B + if c >= 0xE0 && c <= 0xEF { + return 1 + } + case 5: // S5: the extension byte after 0x9C + if c >= 0xF0 && c <= 0xF4 { + return 2 + } + case 6: // S6: the extension byte after 0x9D + if c >= 0xF5 && c <= 0xFE { + return 2 + } + } + return emacsMuleFail +} + +// scanEmacsMuleToken is the scrub scanner for Emacs-Mule: it runs the automaton +// over the head of b. An ill-formed head reports the maximal subpart consumed +// before the automaton failed (at least one byte, so the walk always advances), +// and a character cut off by the end of the input is ill-formed too. +func scanEmacsMuleToken(b []byte) (int, bool) { + state := 0 + for i := 0; i < len(b); i++ { + switch next := emacsMuleNext(state, b[i]); next { + case emacsMuleAccept: + return i + 1, true + case emacsMuleFail: + if i == 0 { + return 1, false + } + return i, false // the bytes already consumed are the ill-formed subpart + default: + state = next + } + } + return len(b), false // the input ended mid-character +} + +// scrubBytesIn is rb_enc_str_scrub over raw bytes: every ill-formed token in +// encoding enc becomes the transcoding options' `replace:` string, or the +// encoding's own default replacement when none was given. String#encode's +// same-encoding path reaches for it because str_transcode0 (transcode.c) calls +// rb_enc_str_scrub there rather than building a converter. An encoding rbgo has no +// scanner for is left untouched, which is how #scrub already treats it. +func scrubBytesIn(b []byte, enc string, opts transcodeOpts) []byte { + scan, ok := scrubScannerFor(enc) + if !ok { + return b + } + rep := scrubDefaultRepl(enc) + if opts.hasReplace { + rep = []byte(opts.replace) + } + out, _ := scrubWalk(b, scan, func([]byte) []byte { return rep }) + return out +} + func scanUTF8Token(b []byte) (int, bool) { r, size := utf8.DecodeRune(b) if r == utf8.RuneError && size == 1 { diff --git a/internal/vm/unicode_normalize.go b/internal/vm/unicode_normalize.go index 6781a576..66358e2b 100644 --- a/internal/vm/unicode_normalize.go +++ b/internal/vm/unicode_normalize.go @@ -62,16 +62,77 @@ func normSource(self object.Value) string { return string(s.Bytes()) } +// unicodeNormalizeVia is the UNICODE_ENCODINGS list of +// lib/unicode_normalize/normalize.rb: the encodings whose strings are normalized +// by round-tripping through UTF-8. UCS-2BE and UCS-4BE are aliases of UTF-16BE and +// UTF-32BE, so they arrive here under their canonical names. +func unicodeNormalizeVia(enc string) bool { + switch enc { + case "UTF-16BE", "UTF-16LE", "UTF-32BE", "UTF-32LE", "GB18030": + return true + } + return false +} + +// normTarget performs the encoding dispatch at the head of +// UnicodeNormalize.normalize and .normalized? +// (lib/unicode_normalize/normalize.rb): UTF-8 is normalized directly, US-ASCII is +// already normalized in every form, the other Unicode encodings are normalized +// through UTF-8 and encoded back, and every other encoding is an +// Encoding::CompatibilityError — normalization is not defined for it. +// +// The encoding decides BEFORE the bytes are looked at, which is why an +// ISO-8859-1 string of one high byte reports the incompatibility rather than +// "invalid byte sequence in UTF-8": in MRI that ArgumentError can only come from +// the UTF-8 branch's gsub. +// +// back is the encoding the normalized text must be encoded into again ("" when the +// receiver was already UTF-8); done reports a receiver that needs no normalizing +// at all. +func (vm *VM) normTarget(self object.Value) (src, back string, done bool) { + s := self.(*object.String) + switch enc := vm.canonicalEncName(s.EncName()); enc { + case "UTF-8": + return normSource(self), "", false + case "US-ASCII": + return string(s.Bytes()), "", true + default: + if !unicodeNormalizeVia(enc) { + raise("Encoding::CompatibilityError", "Unicode Normalization not appropriate for %s", enc) + } + return vm.stringEncode(s, []object.Value{object.NewString("UTF-8")}).Str(), enc, false + } +} + +// normResult encodes a normalized UTF-8 result back into the receiver's encoding +// when it was one of the non-UTF-8 Unicode encodings (`.encode(encoding)` at the +// tail of UnicodeNormalize.normalize). +func (vm *VM) normResult(out, back string) *object.String { + u := object.NewString(out) + if back == "" { + return u + } + return vm.stringEncode(u, []object.Value{object.NewString(back)}) +} + // registerStringUnicodeNormalize adds the unicode_normalize core-ext String // methods. (Called from the String setup so it shares cString.) func (vm *VM) registerStringUnicodeNormalize() { - vm.cString.define("unicode_normalize", func(_ *VM, self object.Value, args []object.Value, _ *Proc) object.Value { + vm.cString.define("unicode_normalize", func(vm *VM, self object.Value, args []object.Value, _ *Proc) object.Value { form := normForm(args) - return object.NewString(norm.Normalize(normSource(self), form)) + src, back, done := vm.normTarget(self) + if done { + return object.NewStringBytesEnc([]byte(src), self.(*object.String).Enc) + } + return vm.normResult(norm.Normalize(src, form), back) }) - vm.cString.define("unicode_normalized?", func(_ *VM, self object.Value, args []object.Value, _ *Proc) object.Value { + vm.cString.define("unicode_normalized?", func(vm *VM, self object.Value, args []object.Value, _ *Proc) object.Value { form := normForm(args) - return object.Bool(norm.IsNormalized(normSource(self), form)) + src, _, done := vm.normTarget(self) + if done { + return object.Bool(true) + } + return object.Bool(norm.IsNormalized(src, form)) }) // unicode_normalize! normalizes the receiver in place and returns self (even // when already normalized). The form argument is validated before the frozen @@ -79,8 +140,11 @@ func (vm *VM) registerStringUnicodeNormalize() { vm.cString.define("unicode_normalize!", func(vm *VM, self object.Value, args []object.Value, _ *Proc) object.Value { form := normForm(args) s := self.(*object.String) + src, back, done := vm.normTarget(self) vm.checkFrozen(s) - s.SetBytes([]byte(norm.Normalize(normSource(self), form))) + if !done { + s.SetBytes(append([]byte(nil), vm.normResult(norm.Normalize(src, form), back).Bytes()...)) + } return self }) }