From 83ce55b120e7ab7eafeaed1e102503e7c29a4dba Mon Sep 17 00:00:00 2001 From: Pastoray Date: Mon, 9 Nov 2026 21:20:45 +0100 Subject: [PATCH 1/3] fix: make Spark encode follow Spark 4 by default --- datafusion/common/src/config.rs | 20 + .../spark/src/function/string/encode.rs | 573 ++++++++++++++++-- .../test_files/information_schema.slt | 4 + .../test_files/spark/string/encode.slt | 238 +++++++- .../library-user-guide/upgrading/56.0.0.md | 43 ++ docs/source/user-guide/configs.md | 2 + 6 files changed, 783 insertions(+), 97 deletions(-) diff --git a/datafusion/common/src/config.rs b/datafusion/common/src/config.rs index 63a7797e396ef..74149eff83b89 100644 --- a/datafusion/common/src/config.rs +++ b/datafusion/common/src/config.rs @@ -2253,6 +2253,26 @@ config_namespace! { /// /// Values are case-insensitive. pub map_key_dedup_policy: MapKeyDedupPolicy, default = MapKeyDedupPolicy::Exception + + /// If true, the Spark-compatible `encode` function accepts any charset name + /// it knows, including aliases such as `UTF8`, `LATIN1` and `UTF-32BE`, as + /// Spark 3.x did. If false (default), `encode` accepts only `us-ascii`, + /// `iso-8859-1`, `utf-8`, `utf-16be`, `utf-16le`, `utf-16` and `utf-32` + /// (case-insensitive) and raises `[INVALID_PARAMETER_VALUE.CHARSET]` for any + /// other name, as Spark 4.0 and later do. + /// + /// Mirrors Spark's + /// [`spark.sql.legacy.javaCharsets`](https://github.com/apache/spark/blob/v4.2.0/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala). + pub legacy_java_charsets: bool, default = false + + /// If true, the Spark-compatible `encode` function replaces a character that + /// the target charset cannot represent with `?`, as Spark 3.x did. If false + /// (default), `encode` raises `[MALFORMED_CHARACTER_CODING]`, as Spark 4.0 and + /// later do. + /// + /// Mirrors Spark's + /// [`spark.sql.legacy.codingErrorAction`](https://github.com/apache/spark/blob/v4.2.0/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala). + pub legacy_coding_error_action: bool, default = false } } diff --git a/datafusion/spark/src/function/string/encode.rs b/datafusion/spark/src/function/string/encode.rs index 5276ada9b658c..ac40e311238a3 100644 --- a/datafusion/spark/src/function/string/encode.rs +++ b/datafusion/spark/src/function/string/encode.rs @@ -21,27 +21,27 @@ use datafusion_expr::{ /// Binary input is interpreted as UTF-8 with lossy conversion (invalid bytes become U+FFFD). /// /// # Target Spark version -/// Emulates Spark 3.5 semantics: -/// - Accepts canonical charset names and common aliases (`UTF8`, `LATIN1`, -/// `ISO88591`, `ASCII`, `UTF-32BE`, etc.). -/// - Unmappable characters (non-ASCII in `US-ASCII`, code points above -/// `U+00FF` in `ISO-8859-1`) are silently replaced with `?`. -/// - `UTF-32` is an alias for `UTF-32BE` (no BOM), matching both Spark 3.5 -/// and Spark 4.1. +/// Follows Spark 4.0 and later by default. Two options under `datafusion.spark` +/// restore the Spark 3.x behavior; both mirror a `spark.sql.legacy.*` config and +/// default to `false`, as in Spark: /// -/// # Spark 4.0 differences (not implemented) -/// Spark 4.0 tightened `encode` in two ways, each gated by a `spark.sql.legacy.*` -/// config that can restore the 3.5 behavior: -/// -/// - Charset whitelist — rejects aliases with `INVALID_PARAMETER_VALUE.CHARSET`. -/// Controlled by `spark.sql.legacy.javaCharsets`. -/// - Unmappable characters — raises `MALFORMED_CHARACTER_CODING`. -/// Controlled by `spark.sql.legacy.codingErrorAction`. +/// - Charset names: only `us-ascii`, `iso-8859-1`, `utf-8`, `utf-16be`, +/// `utf-16le`, `utf-16` and `utf-32` (case-insensitive) are accepted. Any other +/// name, including aliases such as `UTF8`, `LATIN1` or `UTF-32BE`, raises +/// `[INVALID_PARAMETER_VALUE.CHARSET]`. Setting `legacy_java_charsets` +/// (`spark.sql.legacy.javaCharsets`) to `true` also accepts those aliases and +/// `UTF-32BE` / `UTF-32LE`. +/// - Unmappable characters: a character the charset cannot represent (non-ASCII +/// in `US-ASCII`, code points above `U+00FF` in `ISO-8859-1`) raises +/// `[MALFORMED_CHARACTER_CODING]`. Setting `legacy_coding_error_action` +/// (`spark.sql.legacy.codingErrorAction`) to `true` replaces it with `?`. /// +/// `UTF-32` is `UTF-32BE` without a BOM, as in Spark. Spark 3.x accepted any Java +/// charset name; names outside the set listed above are not implemented here and +/// raise an error even when `legacy_java_charsets` is `true`. /// /// -// TODO: wire both configs so Spark 4.0 behavior can be selected at runtime. -// See: +/// #[derive(Debug, PartialEq, Eq, Hash)] pub struct SparkEncode { signature: Signature, @@ -61,73 +61,187 @@ impl SparkEncode { } } +/// A charset that `encode` can produce. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Charset { + UsAscii, + Iso88591, + Utf8, + Utf16Be, + Utf16Le, + /// A BOM (big-endian marker) followed by UTF-16BE. + Utf16, + /// Also the target of `UTF-32`, which Spark encodes without a BOM. + Utf32Be, + Utf32Le, +} + +/// The charset names Spark 4.0 and later accept, as listed in its error message. +const SPARK_4_CHARSETS: &str = + "iso-8859-1, us-ascii, utf-16, utf-16be, utf-16le, utf-32, utf-8"; + +/// The charset names accepted when `legacy_java_charsets` is set, for the error +/// message of a name that is still unknown. +const LEGACY_CHARSETS: &str = + "US-ASCII, ISO-8859-1, UTF-8, UTF-16, UTF-16BE, UTF-16LE, UTF-32, UTF-32BE, UTF-32LE"; + +/// Resolves a charset name, case-insensitively. With `legacy_java_charsets` it also +/// accepts the aliases Spark 3.x took from Java, such as `UTF8` and `LATIN1`. +fn resolve_charset(name: &str, legacy_java_charsets: bool) -> Result { + let lower = name.to_ascii_lowercase(); + let charset = match lower.as_str() { + "us-ascii" => Some(Charset::UsAscii), + "iso-8859-1" => Some(Charset::Iso88591), + "utf-8" => Some(Charset::Utf8), + "utf-16be" => Some(Charset::Utf16Be), + "utf-16le" => Some(Charset::Utf16Le), + "utf-16" => Some(Charset::Utf16), + "utf-32" => Some(Charset::Utf32Be), + _ if legacy_java_charsets => match lower.as_str() { + "utf8" => Some(Charset::Utf8), + "ascii" => Some(Charset::UsAscii), + "iso88591" | "latin1" => Some(Charset::Iso88591), + "utf16be" => Some(Charset::Utf16Be), + "utf16le" => Some(Charset::Utf16Le), + "utf16" => Some(Charset::Utf16), + "utf32" | "utf-32be" | "utf32be" => Some(Charset::Utf32Be), + "utf-32le" | "utf32le" => Some(Charset::Utf32Le), + _ => None, + }, + _ => None, + }; + match charset { + Some(charset) => Ok(charset), + None if legacy_java_charsets => exec_err!( + "Unsupported charset for encode: '{name}'. Supported: {LEGACY_CHARSETS}" + ), + None => exec_err!( + "[INVALID_PARAMETER_VALUE.CHARSET] The value of parameter(s) `charset` in \ + `encode` is invalid: expects one of the {SPARK_4_CHARSETS}, but got {name}." + ), + } +} + +/// Encodes `s` into `bytes` for a charset that maps each code point up to `max` to +/// one byte of the same value. Any other character becomes `?` if +/// `replace_unmappable`, and is an error otherwise. +fn encode_single_byte( + s: &str, + max: u32, + charset_name: &str, + replace_unmappable: bool, + bytes: &mut Vec, +) -> Result<()> { + for c in s.chars() { + let code_point = c as u32; + if code_point <= max { + bytes.push(code_point as u8); + } else if replace_unmappable { + bytes.push(b'?'); + } else { + return exec_err!( + "[MALFORMED_CHARACTER_CODING] Invalid value found when performing \ + `encode` with {charset_name}" + ); + } + } + Ok(()) +} + /// Encodes a single string value using the specified charset into `bytes`, -/// replacing its previous contents. -/// Unmappable characters are silently replaced with `?` (Spark 3.5 behavior). -fn encode_string(s: &str, charset: &str, bytes: &mut Vec) -> Result<()> { +/// replacing its previous contents. `charset_name` is the name as the user wrote it, +/// used in error messages. Only `US-ASCII` and `ISO-8859-1` can have unmappable +/// characters, since every Unicode scalar value is representable in the others. +fn encode_string( + s: &str, + charset: Charset, + charset_name: &str, + replace_unmappable: bool, + bytes: &mut Vec, +) -> Result<()> { bytes.clear(); match charset { - "UTF-8" | "UTF8" => bytes.extend_from_slice(s.as_bytes()), - "US-ASCII" | "ASCII" => { - bytes.extend(s.chars().map(|c| if c.is_ascii() { c as u8 } else { b'?' })) + Charset::Utf8 => bytes.extend_from_slice(s.as_bytes()), + Charset::UsAscii => { + encode_single_byte(s, 0x7F, charset_name, replace_unmappable, bytes)? + } + Charset::Iso88591 => { + encode_single_byte(s, 0xFF, charset_name, replace_unmappable, bytes)? } - "ISO-8859-1" | "ISO88591" | "LATIN1" => bytes.extend(s.chars().map(|c| { - let cp = c as u32; - if cp <= 255 { cp as u8 } else { b'?' } - })), - "UTF-16BE" | "UTF16BE" => { + Charset::Utf16Be => { for code_unit in s.encode_utf16() { bytes.extend_from_slice(&code_unit.to_be_bytes()); } } - "UTF-16LE" | "UTF16LE" => { + Charset::Utf16Le => { for code_unit in s.encode_utf16() { bytes.extend_from_slice(&code_unit.to_le_bytes()); } } - "UTF-16" | "UTF16" => { - // BOM (big-endian marker) followed by UTF-16BE encoded bytes + Charset::Utf16 => { bytes.extend_from_slice(&[0xFE, 0xFF]); for code_unit in s.encode_utf16() { bytes.extend_from_slice(&code_unit.to_be_bytes()); } } - // Spark treats UTF-32 as UTF-32BE (no BOM), matching Spark 3.5 and 4.1. - "UTF-32" | "UTF32" | "UTF-32BE" | "UTF32BE" => { + Charset::Utf32Be => { for c in s.chars() { bytes.extend_from_slice(&(c as u32).to_be_bytes()); } } - "UTF-32LE" | "UTF32LE" => { + Charset::Utf32Le => { for c in s.chars() { bytes.extend_from_slice(&(c as u32).to_le_bytes()); } } - _ => { - return exec_err!( - "Unsupported charset for encode: '{}'. Supported: US-ASCII, ISO-8859-1, UTF-8, UTF-16, UTF-16BE, UTF-16LE, UTF-32, UTF-32BE, UTF-32LE", - charset - ); - } } Ok(()) } -/// The charset for each row. `Constant` is a single uppercased charset; `PerRow` -/// uppercases each row on demand, yielding `None` for null entries. +/// How `encode` treats the behavior that differs between Spark 3.x and 4.x, +/// taken from `datafusion.spark.*`. +#[derive(Debug, Clone, Copy)] +struct EncodeOptions { + legacy_java_charsets: bool, + replace_unmappable: bool, +} + +/// The charset for each row. `Constant` resolves the charset once, but only +/// reports an unknown name from a row that is actually encoded; `PerRow` resolves +/// each row's charset on demand, yielding `None` for null entries. enum Charsets<'a> { - Constant(&'a str), + Constant { + name: &'a str, + resolved: Option, + }, PerRow(&'a StringArray), } -impl Charsets<'_> { - fn get(&self, i: usize) -> Option> { - match self { - Charsets::Constant(charset) => Some(Cow::Borrowed(charset)), +impl<'a> Charsets<'a> { + fn constant(name: &'a str, options: EncodeOptions) -> Self { + Charsets::Constant { + name, + resolved: resolve_charset(name, options.legacy_java_charsets).ok(), + } + } + + /// The charset and the name it was given by, or `None` if the row's charset is + /// null. + fn get(&self, i: usize, options: EncodeOptions) -> Result> { + let (name, resolved) = match self { + Charsets::Constant { name, resolved } => (*name, *resolved), Charsets::PerRow(array) => { - (!array.is_null(i)).then(|| Cow::Owned(array.value(i).to_uppercase())) + if array.is_null(i) { + return Ok(None); + } + (array.value(i), None) } - } + }; + let charset = match resolved { + Some(charset) => charset, + None => resolve_charset(name, options.legacy_java_charsets)?, + }; + Ok(Some((charset, name))) } } @@ -140,16 +254,27 @@ fn encode_rows<'a>( is_null: impl Fn(usize) -> bool, decode: impl Fn(usize) -> Cow<'a, str>, charsets: &Charsets, + options: EncodeOptions, ) -> Result { let mut builder = BinaryBuilder::with_capacity(len, len * 4); let mut bytes = Vec::new(); for i in 0..len { - match (is_null(i), charsets.get(i)) { - (false, Some(charset)) => { - encode_string(&decode(i), &charset, &mut bytes)?; + if is_null(i) { + builder.append_null(); + continue; + } + match charsets.get(i, options)? { + Some((charset, name)) => { + encode_string( + &decode(i), + charset, + name, + options.replace_unmappable, + &mut bytes, + )?; builder.append_value(&bytes); } - _ => builder.append_null(), + None => builder.append_null(), } } Ok(Arc::new(builder.finish())) @@ -158,45 +283,59 @@ fn encode_rows<'a>( fn encode_array<'a, S: StringArrayType<'a>>( array: &S, charsets: &Charsets, + options: EncodeOptions, ) -> Result { encode_rows( array.len(), |i| array.is_null(i), |i| Cow::Borrowed(array.value(i)), charsets, + options, ) } fn encode_binary_array<'a, B: arrow::array::BinaryArrayType<'a>>( array: &'a B, charsets: &Charsets, + options: EncodeOptions, ) -> Result { encode_rows( array.len(), |i| array.is_null(i), |i| String::from_utf8_lossy(array.value(i)), charsets, + options, ) } -fn encode_dispatch(arr: &ArrayRef, charsets: &Charsets) -> Result { +fn encode_dispatch( + arr: &ArrayRef, + charsets: &Charsets, + options: EncodeOptions, +) -> Result { match arr.data_type() { - DataType::Utf8 => encode_array(&arr.as_string::(), charsets), - DataType::LargeUtf8 => encode_array(&arr.as_string::(), charsets), - DataType::Utf8View => encode_array(&arr.as_string_view(), charsets), - DataType::Binary => encode_binary_array(&arr.as_binary::(), charsets), - DataType::LargeBinary => encode_binary_array(&arr.as_binary::(), charsets), - DataType::BinaryView => encode_binary_array(&arr.as_binary_view(), charsets), + DataType::Utf8 => encode_array(&arr.as_string::(), charsets, options), + DataType::LargeUtf8 => encode_array(&arr.as_string::(), charsets, options), + DataType::Utf8View => encode_array(&arr.as_string_view(), charsets, options), + DataType::Binary => { + encode_binary_array(&arr.as_binary::(), charsets, options) + } + DataType::LargeBinary => { + encode_binary_array(&arr.as_binary::(), charsets, options) + } + DataType::BinaryView => { + encode_binary_array(&arr.as_binary_view(), charsets, options) + } DataType::Null => Ok(Arc::new(BinaryArray::new_null(arr.len()))), dt => exec_err!("encode expects a string or binary argument, got {dt:?}"), } } -/// The uppercased charset from a scalar, or `None` if it is null. -fn scalar_charset(scalar: &ScalarValue) -> Result> { +/// The charset name from a scalar, or `None` if it is null. +fn scalar_charset(scalar: &ScalarValue) -> Result> { match scalar { ScalarValue::Utf8(s) | ScalarValue::LargeUtf8(s) | ScalarValue::Utf8View(s) => { - Ok(s.as_ref().map(|s| s.to_uppercase())) + Ok(s.as_deref()) } ScalarValue::Null => Ok(None), other => exec_err!("encode charset argument must be a string, got {other:?}"), @@ -263,6 +402,11 @@ impl ScalarUDFImpl for SparkEncode { } fn invoke_with_args(&self, args: ScalarFunctionArgs) -> Result { + let spark_options = &args.config_options.spark; + let options = EncodeOptions { + legacy_java_charsets: spark_options.legacy_java_charsets, + replace_unmappable: spark_options.legacy_coding_error_action, + }; let [value, charset_arg] = take_function_args(self.name(), args.args)?; let len = [&value, &charset_arg] @@ -282,9 +426,11 @@ impl ScalarUDFImpl for SparkEncode { // A null charset yields a null result for that row (Spark NullIntolerant). let result = match &charset_arg { ColumnarValue::Scalar(scalar) => match scalar_charset(scalar)? { - Some(charset) => { - encode_dispatch(&value_arr, &Charsets::Constant(&charset))? - } + Some(charset) => encode_dispatch( + &value_arr, + &Charsets::constant(charset, options), + options, + )?, None => { return Ok(ColumnarValue::Scalar(ScalarValue::Binary(None))); } @@ -292,6 +438,7 @@ impl ScalarUDFImpl for SparkEncode { ColumnarValue::Array(charset_array) => encode_dispatch( &value_arr, &Charsets::PerRow(charset_array.as_string::()), + options, )?, }; @@ -306,6 +453,7 @@ impl ScalarUDFImpl for SparkEncode { #[cfg(test)] mod tests { use super::*; + use datafusion_common::config::ConfigOptions; #[test] fn test_encode_return_field_nullable() { @@ -334,4 +482,297 @@ mod tests { .unwrap(); assert!(!non_nullable.is_nullable()); } + + /// Runs `encode` over `values` with the given charset argument and the two + /// `datafusion.spark` legacy options. + fn run_encode( + values: &[Option<&str>], + charset: ColumnarValue, + legacy_java_charsets: bool, + legacy_coding_error_action: bool, + ) -> Result>>> { + let mut config = ConfigOptions::default(); + config.spark.legacy_java_charsets = legacy_java_charsets; + config.spark.legacy_coding_error_action = legacy_coding_error_action; + + let result = SparkEncode::new().invoke_with_args(ScalarFunctionArgs { + args: vec![ + ColumnarValue::Array(Arc::new(StringArray::from(values.to_vec()))), + charset, + ], + arg_fields: vec![ + Arc::new(Field::new("value", DataType::Utf8, true)), + Arc::new(Field::new("charset", DataType::Utf8, true)), + ], + number_rows: values.len(), + return_field: Arc::new(Field::new("encode", DataType::Binary, true)), + config_options: Arc::new(config), + })?; + let ColumnarValue::Array(array) = result else { + panic!("expected an array result"); + }; + Ok(array + .as_binary::() + .iter() + .map(|v| v.map(<[u8]>::to_vec)) + .collect()) + } + + fn charset(name: &str) -> ColumnarValue { + ColumnarValue::Scalar(ScalarValue::Utf8(Some(name.to_string()))) + } + + /// Encodes one string with a constant charset. + fn encode_one( + value: &str, + name: &str, + legacy_java_charsets: bool, + legacy_coding_error_action: bool, + ) -> Result> { + let rows = run_encode( + &[Some(value)], + charset(name), + legacy_java_charsets, + legacy_coding_error_action, + )?; + Ok(rows.into_iter().next().unwrap().unwrap()) + } + + #[test] + fn test_spark4_accepted_charsets() { + // the seven names Spark 4 accepts, in any case + for (name, expected) in [ + ("us-ascii", vec![0x41]), + ("US-ASCII", vec![0x41]), + ("iso-8859-1", vec![0x41]), + ("utf-8", vec![0x41]), + ("Utf-8", vec![0x41]), + ("utf-16be", vec![0x00, 0x41]), + ("utf-16le", vec![0x41, 0x00]), + ("utf-16", vec![0xFE, 0xFF, 0x00, 0x41]), + ("utf-32", vec![0x00, 0x00, 0x00, 0x41]), + ("UTF-32", vec![0x00, 0x00, 0x00, 0x41]), + ] { + assert_eq!( + encode_one("A", name, false, false).unwrap(), + expected, + "{name}" + ); + } + } + + #[test] + fn test_spark4_rejects_other_charsets() { + for name in [ + "utf-32be", "utf-32le", "UTF8", "ascii", "latin1", "iso88591", "utf16", + "EBCDIC", "", + ] { + let err = encode_one("A", name, false, false).unwrap_err().to_string(); + assert!( + err.contains("[INVALID_PARAMETER_VALUE.CHARSET]"), + "{name}: {err}" + ); + assert!(err.contains("`encode`"), "{name}: {err}"); + assert!( + err.contains( + "iso-8859-1, us-ascii, utf-16, utf-16be, utf-16le, utf-32, utf-8" + ), + "{name}: {err}" + ); + assert!(err.contains(&format!("but got {name}.")), "{name}: {err}"); + } + } + + #[test] + fn test_spark4_unmappable_characters_error() { + // 'é' (U+00E9) is not ASCII; 'Ā' (U+0100) and an emoji are not Latin-1 + for (value, name) in [ + ("é", "us-ascii"), + ("aé", "US-ASCII"), + ("Ā", "iso-8859-1"), + ("😀", "ISO-8859-1"), + ] { + let err = encode_one(value, name, false, false) + .unwrap_err() + .to_string(); + assert!( + err.contains("[MALFORMED_CHARACTER_CODING]"), + "{value}: {err}" + ); + assert!(err.contains("`encode`"), "{value}: {err}"); + // the charset appears as the user wrote it + assert!(err.ends_with(&format!("with {name}")), "{value}: {err}"); + } + } + + #[test] + fn test_spark4_mappable_characters_succeed() { + // 'é' is U+00E9, which Latin-1 represents; every charset below it holds + // any Unicode scalar value + assert_eq!( + encode_one("é", "iso-8859-1", false, false).unwrap(), + vec![0xE9] + ); + assert_eq!( + encode_one("\u{7f}", "us-ascii", false, false).unwrap(), + vec![0x7F] + ); + assert_eq!(encode_one("é😀", "utf-8", false, false).unwrap().len(), 6); + assert_eq!( + encode_one("😀", "utf-32", false, false).unwrap(), + vec![0x00, 0x01, 0xF6, 0x00] + ); + assert_eq!( + encode_one("", "us-ascii", false, false).unwrap(), + Vec::::new() + ); + } + + #[test] + fn test_legacy_java_charsets() { + for (name, expected) in [ + ("UTF8", vec![0x41]), + ("ascii", vec![0x41]), + ("LATIN1", vec![0x41]), + ("ISO88591", vec![0x41]), + ("utf-32be", vec![0x00, 0x00, 0x00, 0x41]), + ("UTF32BE", vec![0x00, 0x00, 0x00, 0x41]), + ("utf-32le", vec![0x41, 0x00, 0x00, 0x00]), + ("UTF32LE", vec![0x41, 0x00, 0x00, 0x00]), + ("UTF16", vec![0xFE, 0xFF, 0x00, 0x41]), + ("UTF16BE", vec![0x00, 0x41]), + ("UTF16LE", vec![0x41, 0x00]), + ("UTF32", vec![0x00, 0x00, 0x00, 0x41]), + // the seven Spark 4 names still work + ("utf-8", vec![0x41]), + ("utf-32", vec![0x00, 0x00, 0x00, 0x41]), + ] { + assert_eq!( + encode_one("A", name, true, false).unwrap(), + expected, + "{name}" + ); + } + + // a name outside the supported set still fails, without the Spark 4 code + let err = encode_one("A", "EBCDIC", true, false) + .unwrap_err() + .to_string(); + assert!( + err.contains("Unsupported charset for encode: 'EBCDIC'"), + "{err}" + ); + assert!(!err.contains("INVALID_PARAMETER_VALUE"), "{err}"); + } + + #[test] + fn test_legacy_coding_error_action() { + assert_eq!(encode_one("é", "us-ascii", false, true).unwrap(), b"?"); + assert_eq!(encode_one("aéb", "US-ASCII", false, true).unwrap(), b"a?b"); + // one '?' for each code point, including one above the BMP + assert_eq!(encode_one("Ā😀", "iso-8859-1", false, true).unwrap(), b"??"); + assert_eq!( + encode_one("é", "iso-8859-1", false, true).unwrap(), + vec![0xE9] + ); + } + + #[test] + fn test_legacy_options_are_independent() { + // legacy charsets alone still reject unmappable characters + let err = encode_one("é", "ascii", true, false) + .unwrap_err() + .to_string(); + assert!(err.contains("[MALFORMED_CHARACTER_CODING]"), "{err}"); + + // legacy coding error action alone still rejects aliases + let err = encode_one("é", "ascii", false, true) + .unwrap_err() + .to_string(); + assert!(err.contains("[INVALID_PARAMETER_VALUE.CHARSET]"), "{err}"); + + // both together are the Spark 3.5 behavior + assert_eq!(encode_one("é", "ascii", true, true).unwrap(), b"?"); + } + + #[test] + fn test_per_row_charsets() { + let names = ColumnarValue::Array(Arc::new(StringArray::from(vec![ + Some("US-ASCII"), + Some("utf-16be"), + None, + ]))); + let rows = + run_encode(&[Some("a"), Some("b"), Some("c")], names, false, false).unwrap(); + assert_eq!(rows, vec![Some(vec![0x61]), Some(vec![0x00, 0x62]), None]); + + // a row's charset is checked on its own + let names = ColumnarValue::Array(Arc::new(StringArray::from(vec![ + Some("utf-8"), + Some("utf-32le"), + ]))); + let err = run_encode(&[Some("a"), Some("b")], names, false, false) + .unwrap_err() + .to_string(); + assert!(err.contains("[INVALID_PARAMETER_VALUE.CHARSET]"), "{err}"); + assert!(err.contains("but got utf-32le."), "{err}"); + + // an unmappable character in one row fails the query + let names = ColumnarValue::Array(Arc::new(StringArray::from(vec![ + Some("utf-8"), + Some("us-ascii"), + ]))); + let err = run_encode(&[Some("a"), Some("é")], names, false, false) + .unwrap_err() + .to_string(); + assert!(err.contains("[MALFORMED_CHARACTER_CODING]"), "{err}"); + } + + #[test] + fn test_nulls_are_not_validated() { + // a NULL value or NULL charset yields NULL for the row, whatever the charset + let rows = run_encode( + &[None, Some("é")], + ColumnarValue::Array(Arc::new(StringArray::from(vec![Some("bogus"), None]))), + false, + false, + ) + .unwrap(); + assert_eq!(rows, vec![None, None]); + } + + #[test] + fn test_binary_input_with_invalid_utf8() { + // Invalid UTF-8 becomes U+FFFD, which US-ASCII cannot represent: an error by + // default, '?' with the legacy coding error action + let args = |legacy_coding_error_action: bool| { + let mut config = ConfigOptions::default(); + config.spark.legacy_coding_error_action = legacy_coding_error_action; + ScalarFunctionArgs { + args: vec![ + ColumnarValue::Scalar(ScalarValue::Binary(Some(vec![0x61, 0xFF]))), + charset("us-ascii"), + ], + arg_fields: vec![ + Arc::new(Field::new("value", DataType::Binary, true)), + Arc::new(Field::new("charset", DataType::Utf8, true)), + ], + number_rows: 1, + return_field: Arc::new(Field::new("encode", DataType::Binary, true)), + config_options: Arc::new(config), + } + }; + let err = SparkEncode::new() + .invoke_with_args(args(false)) + .unwrap_err() + .to_string(); + assert!(err.contains("[MALFORMED_CHARACTER_CODING]"), "{err}"); + + let ColumnarValue::Scalar(ScalarValue::Binary(Some(bytes))) = + SparkEncode::new().invoke_with_args(args(true)).unwrap() + else { + panic!("expected a binary scalar"); + }; + assert_eq!(bytes, b"a?"); + } } diff --git a/datafusion/sqllogictest/test_files/information_schema.slt b/datafusion/sqllogictest/test_files/information_schema.slt index c1aabc2946ea6..d20cb457fdd4b 100644 --- a/datafusion/sqllogictest/test_files/information_schema.slt +++ b/datafusion/sqllogictest/test_files/information_schema.slt @@ -349,6 +349,8 @@ datafusion.runtime.max_temp_directory_size 100G datafusion.runtime.memory_limit unlimited datafusion.runtime.metadata_cache_limit 50M datafusion.runtime.temp_directory NULL +datafusion.spark.legacy_coding_error_action false +datafusion.spark.legacy_java_charsets false datafusion.spark.map_key_dedup_policy EXCEPTION datafusion.sql_parser.collect_spans false datafusion.sql_parser.default_null_ordering nulls_max @@ -518,6 +520,8 @@ datafusion.runtime.max_temp_directory_size 100G Maximum temporary file directory datafusion.runtime.memory_limit unlimited Maximum memory limit for query execution. Supports suffixes K (kilobytes), M (megabytes), and G (gigabytes) or '0' for 0. Example: '2G' for 2 gigabytes. datafusion.runtime.metadata_cache_limit 50M Maximum memory to use for file metadata cache such as Parquet metadata. Supports suffixes K (kilobytes), M (megabytes), and G (gigabytes) or '0' for 0. Example: '2G' for 2 gigabytes. datafusion.runtime.temp_directory NULL The path to the temporary file directory. +datafusion.spark.legacy_coding_error_action false If true, the Spark-compatible `encode` function replaces a character that the target charset cannot represent with `?`, as Spark 3.x did. If false (default), `encode` raises `[MALFORMED_CHARACTER_CODING]`, as Spark 4.0 and later do. Mirrors Spark's [`spark.sql.legacy.codingErrorAction`](https://github.com/apache/spark/blob/v4.2.0/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala). +datafusion.spark.legacy_java_charsets false If true, the Spark-compatible `encode` function accepts any charset name it knows, including aliases such as `UTF8`, `LATIN1` and `UTF-32BE`, as Spark 3.x did. If false (default), `encode` accepts only `us-ascii`, `iso-8859-1`, `utf-8`, `utf-16be`, `utf-16le`, `utf-16` and `utf-32` (case-insensitive) and raises `[INVALID_PARAMETER_VALUE.CHARSET]` for any other name, as Spark 4.0 and later do. Mirrors Spark's [`spark.sql.legacy.javaCharsets`](https://github.com/apache/spark/blob/v4.2.0/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala). datafusion.spark.map_key_dedup_policy EXCEPTION Policy for handling duplicate keys in Spark-compatible map-construction functions (`map_from_arrays`, `map_from_entries`, `str_to_map`). Mirrors Spark's [`spark.sql.mapKeyDedupPolicy`](https://github.com/apache/spark/blob/cf3a34e19dfcf70e2d679217ff1ba21302212472/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala#L4961): - `EXCEPTION` (default): raise `[DUPLICATED_MAP_KEY]` at runtime on any duplicate key. - `LAST_WIN`: keep the last occurrence of each duplicate key. Values are case-insensitive. datafusion.sql_parser.collect_spans false When set to true, the source locations relative to the original SQL query (i.e. [`Span`](https://docs.rs/sqlparser/latest/sqlparser/tokenizer/struct.Span.html)) will be collected and recorded in the logical plan nodes. datafusion.sql_parser.default_null_ordering nulls_max Specifies the default null ordering for query results. There are 4 options: - `nulls_max`: Nulls appear last in ascending order. - `nulls_min`: Nulls appear first in ascending order. - `nulls_first`: Nulls always be first in any order. - `nulls_last`: Nulls always be last in any order. By default, `nulls_max` is used to follow Postgres's behavior. postgres rule: diff --git a/datafusion/sqllogictest/test_files/spark/string/encode.slt b/datafusion/sqllogictest/test_files/spark/string/encode.slt index 93df7e80ec7a3..3706d7b97c123 100644 --- a/datafusion/sqllogictest/test_files/spark/string/encode.slt +++ b/datafusion/sqllogictest/test_files/spark/string/encode.slt @@ -2,6 +2,12 @@ # SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership # SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements +# By default `encode` follows Spark 4.0 and later: only us-ascii, iso-8859-1, utf-8, +# utf-16be, utf-16le, utf-16 and utf-32 are accepted, and a character the charset +# cannot represent is an error. The legacy Spark 3.x behavior is selected by +# `datafusion.spark.legacy_java_charsets` and +# `datafusion.spark.legacy_coding_error_action`, which are tested at the end. + # UTF-8 encoding query ? SELECT encode('Spark SQL'::string, 'utf-8'::string); @@ -38,19 +44,7 @@ SELECT encode('Spark SQL'::string, 'utf-16'::string); ---- feff0053007000610072006b002000530051004c -# UTF-32BE encoding -query ? -SELECT encode('A'::string, 'utf-32be'::string); ----- -00000041 - -# UTF-32LE encoding -query ? -SELECT encode('AB'::string, 'utf-32le'::string); ----- -4100000042000000 - -# UTF-32 (Spark 3.5 / 4.1: no BOM, identical to UTF-32BE) +# UTF-32 (no BOM, big-endian) query ? SELECT encode('A'::string, 'utf-32'::string); ---- @@ -62,17 +56,16 @@ SELECT encode('hello'::string, 'Utf-8'::string); ---- 68656c6c6f -# US-ASCII: unmappable characters are silently replaced with '?' (Spark 3.5 behavior) query ? -SELECT encode('é'::string, 'us-ascii'::string); +SELECT encode('hello'::string, 'US-ASCII'::string); ---- -3f +68656c6c6f -# ISO-8859-1: code points above U+00FF are replaced with '?' (Ā is U+0100) +# Characters the charset can represent are not affected by the strict mode query ? -SELECT encode('Ā'::string, 'iso-8859-1'::string); +SELECT encode('é'::string, 'iso-8859-1'::string); ---- -3f +e9 # Multi-byte input (emoji 😀 = U+1F600) across charsets query ? @@ -95,16 +88,6 @@ SELECT encode('😀'::string, 'utf-16'::string); ---- feffd83dde00 -query ? -SELECT encode('😀'::string, 'utf-32be'::string); ----- -0001f600 - -query ? -SELECT encode('😀'::string, 'utf-32le'::string); ----- -00f60100 - query ? SELECT encode('😀'::string, 'utf-32'::string); ---- @@ -140,6 +123,10 @@ SELECT encode(X'FEFF00610073', 'utf-8'::string); ---- efbfbdefbfbd00610073 +# U+FFFD is not representable in US-ASCII +statement error \[MALFORMED_CHARACTER_CODING\] Invalid value found when performing `encode` with us-ascii +SELECT encode(X'61FF', 'us-ascii'::string); + # NULL input query ? SELECT encode(NULL::string, 'utf-8'::string); @@ -174,10 +161,199 @@ SELECT encode(a, b) FROM VALUES (X'0a', 'US-ASCII'), (X'deadbeef', 'UTF-16be') A 0a 07adfffdfffd -# Error: unsupported charset -statement error Unsupported charset for encode +# Error: charsets Spark 4 does not accept, including aliases and UTF-32BE/LE +statement error \[INVALID_PARAMETER_VALUE\.CHARSET\] .*`charset` in `encode` is invalid: expects one of the iso-8859-1, us-ascii, utf-16, utf-16be, utf-16le, utf-32, utf-8, but got utf-32be\. +SELECT encode('A'::string, 'utf-32be'::string); + +statement error \[INVALID_PARAMETER_VALUE\.CHARSET\] .*but got utf-32le\. +SELECT encode('A'::string, 'utf-32le'::string); + +statement error \[INVALID_PARAMETER_VALUE\.CHARSET\] .*but got UTF8\. +SELECT encode('A'::string, 'UTF8'::string); + +statement error \[INVALID_PARAMETER_VALUE\.CHARSET\] .*but got latin1\. +SELECT encode('A'::string, 'latin1'::string); + +statement error \[INVALID_PARAMETER_VALUE\.CHARSET\] .*but got EBCDIC\. SELECT encode('hello'::string, 'EBCDIC'::string); +# Error: an invalid charset in one row fails the query +statement error \[INVALID_PARAMETER_VALUE\.CHARSET\] .*but got utf-32be\. +SELECT encode(a, b) FROM VALUES ('x', 'utf-8'), ('y', 'utf-32be') AS t(a, b); + +# Error: a character the charset cannot represent (é is U+00E9, Ā is U+0100) +statement error \[MALFORMED_CHARACTER_CODING\] Invalid value found when performing `encode` with us-ascii +SELECT encode('é'::string, 'us-ascii'::string); + +statement error \[MALFORMED_CHARACTER_CODING\] Invalid value found when performing `encode` with iso-8859-1 +SELECT encode('Ā'::string, 'iso-8859-1'::string); + +statement error \[MALFORMED_CHARACTER_CODING\] Invalid value found when performing `encode` with iso-8859-1 +SELECT encode('😀'::string, 'iso-8859-1'::string); + +# Error: an unmappable character in one row fails the query +statement error \[MALFORMED_CHARACTER_CODING\] Invalid value found when performing `encode` with US-ASCII +SELECT encode(a, b) FROM VALUES ('x', 'utf-8'), ('é', 'US-ASCII') AS t(a, b); + # Error: no arguments statement error encode function requires 2 arguments SELECT encode(); + +# --------------------------------------------------------------------------- +# datafusion.spark.legacy_java_charsets (spark.sql.legacy.javaCharsets) +# --------------------------------------------------------------------------- + +statement ok +set datafusion.spark.legacy_java_charsets = true; + +# UTF-32BE and UTF-32LE are accepted +query ? +SELECT encode('A'::string, 'utf-32be'::string); +---- +00000041 + +query ? +SELECT encode('AB'::string, 'utf-32le'::string); +---- +4100000042000000 + +query ? +SELECT encode('😀'::string, 'utf-32be'::string); +---- +0001f600 + +query ? +SELECT encode('😀'::string, 'utf-32le'::string); +---- +00f60100 + +# Aliases are accepted +query ? +SELECT encode('naïve'::string, 'LATIN1'::string); +---- +6e61ef7665 + +query ? +SELECT encode('naïve'::string, 'ISO88591'::string); +---- +6e61ef7665 + +query ? +SELECT encode('hello'::string, 'UTF8'::string); +---- +68656c6c6f + +query ? +SELECT encode('hello'::string, 'ASCII'::string); +---- +68656c6c6f + +query ? +SELECT encode('A'::string, 'UTF16'::string); +---- +feff0041 + +query ? +SELECT encode('A'::string, 'UTF32'::string); +---- +00000041 + +# The names Spark 4 accepts still work +query ? +SELECT encode('A'::string, 'utf-8'::string); +---- +41 + +# Per-row charsets +query ? +SELECT encode(a, b) FROM VALUES ('A', 'utf-32be'), ('A', 'latin1') AS t(a, b); +---- +00000041 +41 + +# A charset that is not supported at all is still an error, without the Spark 4 error class +statement error Unsupported charset for encode: 'EBCDIC' +SELECT encode('hello'::string, 'EBCDIC'::string); + +# The legacy charsets option does not change what happens to unmappable characters +statement error \[MALFORMED_CHARACTER_CODING\] Invalid value found when performing `encode` with ascii +SELECT encode('é'::string, 'ascii'::string); + +statement ok +set datafusion.spark.legacy_java_charsets = false; + +# Back to Spark 4: aliases are rejected again +statement error \[INVALID_PARAMETER_VALUE\.CHARSET\] .*but got latin1\. +SELECT encode('A'::string, 'latin1'::string); + +# --------------------------------------------------------------------------- +# datafusion.spark.legacy_coding_error_action (spark.sql.legacy.codingErrorAction) +# --------------------------------------------------------------------------- + +statement ok +set datafusion.spark.legacy_coding_error_action = true; + +# US-ASCII: unmappable characters are replaced with '?' +query ? +SELECT encode('é'::string, 'us-ascii'::string); +---- +3f + +query ? +SELECT encode('aéb'::string, 'us-ascii'::string); +---- +613f62 + +# ISO-8859-1: code points above U+00FF are replaced with '?' (Ā is U+0100) +query ? +SELECT encode('Ā'::string, 'iso-8859-1'::string); +---- +3f + +# One '?' for each code point, including one above the Basic Multilingual Plane +query ? +SELECT encode('Ā😀'::string, 'iso-8859-1'::string); +---- +3f3f + +# Mappable characters are unchanged +query ? +SELECT encode('é'::string, 'iso-8859-1'::string); +---- +e9 + +# Invalid UTF-8 in binary input becomes U+FFFD, then '?' +query ? +SELECT encode(X'61FF', 'us-ascii'::string); +---- +613f + +# Per-row charsets +query ? +SELECT encode(a, b) FROM VALUES ('é', 'us-ascii'), ('é', 'utf-8') AS t(a, b); +---- +3f +c3a9 + +# The coding error option does not change which charsets are accepted +statement error \[INVALID_PARAMETER_VALUE\.CHARSET\] .*but got ascii\. +SELECT encode('é'::string, 'ascii'::string); + +# Both legacy options together restore the Spark 3.5 behavior +statement ok +set datafusion.spark.legacy_java_charsets = true; + +query ? +SELECT encode('é'::string, 'ascii'::string); +---- +3f + +statement ok +set datafusion.spark.legacy_java_charsets = false; + +statement ok +set datafusion.spark.legacy_coding_error_action = false; + +# Back to Spark 4: unmappable characters are an error again +statement error \[MALFORMED_CHARACTER_CODING\] Invalid value found when performing `encode` with us-ascii +SELECT encode('é'::string, 'us-ascii'::string); diff --git a/docs/source/library-user-guide/upgrading/56.0.0.md b/docs/source/library-user-guide/upgrading/56.0.0.md index 07714dcd5589c..771227be46135 100644 --- a/docs/source/library-user-guide/upgrading/56.0.0.md +++ b/docs/source/library-user-guide/upgrading/56.0.0.md @@ -1621,3 +1621,46 @@ let return_type: Option = definition `CreateFunction::try_new` takes `Option` for the return type. See [issue #18235](https://github.com/apache/datafusion/issues/18235) for details. + +### Spark `encode` follows Spark 4 by default + +The Spark-compatibility `encode` function (`datafusion-spark`) previously +followed Spark 3.5. It now follows Spark 4.0 and later, which tightened it in +two ways: + +- Only `us-ascii`, `iso-8859-1`, `utf-8`, `utf-16be`, `utf-16le`, `utf-16` and + `utf-32` are accepted, in any case. Any other name, such as `utf-32be`, + `UTF8` or `LATIN1`, raises `[INVALID_PARAMETER_VALUE.CHARSET]`. +- A character the charset cannot represent, such as `é` in `us-ascii` or a code + point above `U+00FF` in `iso-8859-1`, raises `[MALFORMED_CHARACTER_CODING]` + instead of becoming `?`. + +Two new config options restore the Spark 3.x behavior. Each mirrors a Spark +config of the same purpose and defaults to `false`, as in Spark: + +| DataFusion option | Spark config | +| --------------------------------------------- | ------------------------------------ | +| `datafusion.spark.legacy_java_charsets` | `spark.sql.legacy.javaCharsets` | +| `datafusion.spark.legacy_coding_error_action` | `spark.sql.legacy.codingErrorAction` | + +**Who is affected:** + +- Queries that call the Spark `encode` with a charset outside the seven names + above, or on text that the charset cannot represent. Previously these + succeeded, replacing unmappable characters with `?`. + +**Migration guide:** + +To restore the previous behavior, set both options: + +```sql +SET datafusion.spark.legacy_java_charsets = true; +SET datafusion.spark.legacy_coding_error_action = true; +``` + +Even with `datafusion.spark.legacy_java_charsets` set, `encode` accepts only the +charsets it already supported: the seven above, aliases such as `UTF8` and +`LATIN1`, `UTF-32BE` and `UTF-32LE`. It does not accept every Java charset name +as Spark 3.x did. + +See [issue #27186](https://github.com/apache/datafusion/issues/27186) for details. diff --git a/docs/source/user-guide/configs.md b/docs/source/user-guide/configs.md index 2c3ca887d4a38..9fd3ac171cccf 100644 --- a/docs/source/user-guide/configs.md +++ b/docs/source/user-guide/configs.md @@ -208,6 +208,8 @@ The following configuration settings are available: | datafusion.format.duration_format | pretty | Duration format. Can be either `"pretty"` or `"ISO8601"` | | datafusion.format.types_info | false | Show types in visual representation batches | | datafusion.spark.map_key_dedup_policy | EXCEPTION | Policy for handling duplicate keys in Spark-compatible map-construction functions (`map_from_arrays`, `map_from_entries`, `str_to_map`). Mirrors Spark's [`spark.sql.mapKeyDedupPolicy`](https://github.com/apache/spark/blob/cf3a34e19dfcf70e2d679217ff1ba21302212472/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala#L4961): - `EXCEPTION` (default): raise `[DUPLICATED_MAP_KEY]` at runtime on any duplicate key. - `LAST_WIN`: keep the last occurrence of each duplicate key. Values are case-insensitive. | +| datafusion.spark.legacy_java_charsets | false | If true, the Spark-compatible `encode` function accepts any charset name it knows, including aliases such as `UTF8`, `LATIN1` and `UTF-32BE`, as Spark 3.x did. If false (default), `encode` accepts only `us-ascii`, `iso-8859-1`, `utf-8`, `utf-16be`, `utf-16le`, `utf-16` and `utf-32` (case-insensitive) and raises `[INVALID_PARAMETER_VALUE.CHARSET]` for any other name, as Spark 4.0 and later do. Mirrors Spark's [`spark.sql.legacy.javaCharsets`](https://github.com/apache/spark/blob/v4.2.0/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala). | +| datafusion.spark.legacy_coding_error_action | false | If true, the Spark-compatible `encode` function replaces a character that the target charset cannot represent with `?`, as Spark 3.x did. If false (default), `encode` raises `[MALFORMED_CHARACTER_CODING]`, as Spark 4.0 and later do. Mirrors Spark's [`spark.sql.legacy.codingErrorAction`](https://github.com/apache/spark/blob/v4.2.0/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala). | You can also reset configuration options to default settings via SQL using the `RESET` command. For example, to set and reset `datafusion.execution.batch_size`: From 268deb199ab4c43494395aecbe79c70ab6711f4d Mon Sep 17 00:00:00 2001 From: Pastoray Date: Mon, 9 Nov 2026 22:16:29 +0100 Subject: [PATCH 2/3] test: don't assume error message ends the string (backtrace feature) --- datafusion/spark/src/function/string/encode.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/datafusion/spark/src/function/string/encode.rs b/datafusion/spark/src/function/string/encode.rs index ac40e311238a3..91309f612920a 100644 --- a/datafusion/spark/src/function/string/encode.rs +++ b/datafusion/spark/src/function/string/encode.rs @@ -600,8 +600,12 @@ mod tests { "{value}: {err}" ); assert!(err.contains("`encode`"), "{value}: {err}"); - // the charset appears as the user wrote it - assert!(err.ends_with(&format!("with {name}")), "{value}: {err}"); + // the charset appears as the user wrote it. Use `contains`, not + // `ends_with`: with the `backtrace` feature a backtrace follows the message + assert!( + err.contains(&format!("encode` with {name}")), + "{value}: {err}" + ); } } From 9e0501025f0c8d00f848e01bf52c6f317e92fd00 Mon Sep 17 00:00:00 2001 From: Pastoray Date: Wed, 11 Nov 2026 22:18:04 +0100 Subject: [PATCH 3/3] fix: match Spark's Java charset aliases for legacy_java_charsets --- .../spark/src/function/string/encode.rs | 77 +++++++++++++------ .../test_files/spark/string/encode.slt | 9 ++- 2 files changed, 60 insertions(+), 26 deletions(-) diff --git a/datafusion/spark/src/function/string/encode.rs b/datafusion/spark/src/function/string/encode.rs index 91309f612920a..f36969b24630b 100644 --- a/datafusion/spark/src/function/string/encode.rs +++ b/datafusion/spark/src/function/string/encode.rs @@ -85,6 +85,33 @@ const SPARK_4_CHARSETS: &str = const LEGACY_CHARSETS: &str = "US-ASCII, ISO-8859-1, UTF-8, UTF-16, UTF-16BE, UTF-16LE, UTF-32, UTF-32BE, UTF-32LE"; +/// The names Java accepts for the charsets `encode` supports, lowercased. Spark +/// looks charsets up with `Charset.forName`, which is case-insensitive and also +/// accepts a charset's aliases, so with `legacy_java_charsets` they work here too. +/// This is the output of `Charset.aliases()` on Java 21 plus each canonical name +/// that Spark 4 does not accept by default (`utf-32be`, `utf-32le`). A name Java +/// would resolve to another charset, such as `UnicodeLittle` or `Shift_JIS`, is not +/// recognized, nor is one that Java rejects, such as `iso88591` or `utf16be`. +fn java_charset_alias(lower: &str) -> Option { + Some(match lower { + "819" | "8859_1" | "cp819" | "csisolatin1" | "ibm-819" | "ibm819" + | "iso-ir-100" | "iso8859-1" | "iso8859_1" | "iso_8859-1" | "iso_8859-1:1987" + | "iso_8859_1" | "l1" | "latin1" => Charset::Iso88591, + "646" | "ansi_x3.4-1968" | "ansi_x3.4-1986" | "ascii" | "ascii7" | "cp367" + | "csascii" | "ibm367" | "iso-ir-6" | "iso646-us" | "iso_646.irv:1983" + | "iso_646.irv:1991" | "us" => Charset::UsAscii, + "unicode-1-1-utf-8" | "utf8" => Charset::Utf8, + "unicode" | "unicodebig" | "utf16" | "utf_16" => Charset::Utf16, + "iso-10646-ucs-2" | "unicodebigunmarked" | "utf_16be" | "x-utf-16be" => { + Charset::Utf16Be + } + "unicodelittleunmarked" | "utf_16le" | "x-utf-16le" => Charset::Utf16Le, + "utf32" | "utf_32" | "utf-32be" | "utf_32be" | "x-utf-32be" => Charset::Utf32Be, + "utf-32le" | "utf_32le" | "x-utf-32le" => Charset::Utf32Le, + _ => return None, + }) +} + /// Resolves a charset name, case-insensitively. With `legacy_java_charsets` it also /// accepts the aliases Spark 3.x took from Java, such as `UTF8` and `LATIN1`. fn resolve_charset(name: &str, legacy_java_charsets: bool) -> Result { @@ -97,17 +124,7 @@ fn resolve_charset(name: &str, legacy_java_charsets: bool) -> Result { "utf-16le" => Some(Charset::Utf16Le), "utf-16" => Some(Charset::Utf16), "utf-32" => Some(Charset::Utf32Be), - _ if legacy_java_charsets => match lower.as_str() { - "utf8" => Some(Charset::Utf8), - "ascii" => Some(Charset::UsAscii), - "iso88591" | "latin1" => Some(Charset::Iso88591), - "utf16be" => Some(Charset::Utf16Be), - "utf16le" => Some(Charset::Utf16Le), - "utf16" => Some(Charset::Utf16), - "utf32" | "utf-32be" | "utf32be" => Some(Charset::Utf32Be), - "utf-32le" | "utf32le" => Some(Charset::Utf32Le), - _ => None, - }, + _ if legacy_java_charsets => java_charset_alias(&lower), _ => None, }; match charset { @@ -635,17 +652,23 @@ mod tests { #[test] fn test_legacy_java_charsets() { for (name, expected) in [ + // Java aliases, in any case ("UTF8", vec![0x41]), + ("unicode-1-1-utf-8", vec![0x41]), ("ascii", vec![0x41]), + ("ASCII7", vec![0x41]), + ("iso-ir-6", vec![0x41]), ("LATIN1", vec![0x41]), - ("ISO88591", vec![0x41]), + ("ISO_8859_1", vec![0x41]), + ("cp819", vec![0x41]), ("utf-32be", vec![0x00, 0x00, 0x00, 0x41]), - ("UTF32BE", vec![0x00, 0x00, 0x00, 0x41]), + ("UTF_32BE", vec![0x00, 0x00, 0x00, 0x41]), ("utf-32le", vec![0x41, 0x00, 0x00, 0x00]), - ("UTF32LE", vec![0x41, 0x00, 0x00, 0x00]), + ("UTF_32LE", vec![0x41, 0x00, 0x00, 0x00]), ("UTF16", vec![0xFE, 0xFF, 0x00, 0x41]), - ("UTF16BE", vec![0x00, 0x41]), - ("UTF16LE", vec![0x41, 0x00]), + ("unicode", vec![0xFE, 0xFF, 0x00, 0x41]), + ("UTF_16BE", vec![0x00, 0x41]), + ("UTF_16LE", vec![0x41, 0x00]), ("UTF32", vec![0x00, 0x00, 0x00, 0x41]), // the seven Spark 4 names still work ("utf-8", vec![0x41]), @@ -658,15 +681,19 @@ mod tests { ); } - // a name outside the supported set still fails, without the Spark 4 code - let err = encode_one("A", "EBCDIC", true, false) - .unwrap_err() - .to_string(); - assert!( - err.contains("Unsupported charset for encode: 'EBCDIC'"), - "{err}" - ); - assert!(!err.contains("INVALID_PARAMETER_VALUE"), "{err}"); + // Names Java itself rejects, and names of charsets not implemented here, + // fail with a message that does not claim Spark 4's list + for name in [ + "EBCDIC", "ISO88591", "UTF16BE", "UTF16LE", "UTF32BE", "UTF32LE", "US_ASCII", + "UTF_8", + ] { + let err = encode_one("A", name, true, false).unwrap_err().to_string(); + assert!( + err.contains(&format!("Unsupported charset for encode: '{name}'")), + "{name}: {err}" + ); + assert!(!err.contains("INVALID_PARAMETER_VALUE"), "{name}: {err}"); + } } #[test] diff --git a/datafusion/sqllogictest/test_files/spark/string/encode.slt b/datafusion/sqllogictest/test_files/spark/string/encode.slt index 3706d7b97c123..3709802e22a92 100644 --- a/datafusion/sqllogictest/test_files/spark/string/encode.slt +++ b/datafusion/sqllogictest/test_files/spark/string/encode.slt @@ -234,7 +234,7 @@ SELECT encode('naïve'::string, 'LATIN1'::string); 6e61ef7665 query ? -SELECT encode('naïve'::string, 'ISO88591'::string); +SELECT encode('naïve'::string, 'ISO_8859_1'::string); ---- 6e61ef7665 @@ -271,6 +271,13 @@ SELECT encode(a, b) FROM VALUES ('A', 'utf-32be'), ('A', 'latin1') AS t(a, b); 00000041 41 +# Names that Java rejects are still errors, as they are in Spark with the same option +statement error Unsupported charset for encode: 'ISO88591' +SELECT encode('A'::string, 'ISO88591'::string); + +statement error Unsupported charset for encode: 'UTF16BE' +SELECT encode('A'::string, 'UTF16BE'::string); + # A charset that is not supported at all is still an error, without the Spark 4 error class statement error Unsupported charset for encode: 'EBCDIC' SELECT encode('hello'::string, 'EBCDIC'::string);