From ce46786c6e24ee200284da2d7c3287dbdbf118db Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Sat, 3 Aug 2024 15:24:33 +0100 Subject: [PATCH 1/3] fix large integer handling --- CHANGELOG.md | 2 ++ src/de.rs | 57 +++++++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 56 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d9356e5..68ba131 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,7 @@ ## Unreleased +- Support `u128` / `i128` integers. +- Fix overflow error attempting to depythonize `u64` values greater than `i64::MAX` to types like `serde_json::Value` - `depythonize()` now take a `&Bound` and is no longer depreciate - `depythonize_object()` replace the old `depythonize()` and is depreciated - `depythonize_bound()` is depreciated diff --git a/src/de.rs b/src/de.rs index 62bb7cb..5aa4b34 100644 --- a/src/de.rs +++ b/src/de.rs @@ -71,6 +71,38 @@ impl<'py, 'bound> Depythonizer<'py, 'bound> { fn dict_access(&self) -> Result> { PyMappingAccess::new(self.input.downcast()?) } + + fn deserialize_any_int<'de, V>(&self, int: &Bound<'_, PyInt>, visitor: V) -> Result + where + V: de::Visitor<'de>, + { + if let Ok(x) = int.extract::() { + if let Ok(x) = u8::try_from(x) { + visitor.visit_u8(x) + } else if let Ok(x) = u16::try_from(x) { + visitor.visit_u16(x) + } else if let Ok(x) = u32::try_from(x) { + visitor.visit_u32(x) + } else if let Ok(x) = u64::try_from(x) { + visitor.visit_u64(x) + } else { + visitor.visit_u128(x) + } + } else { + let x: i128 = int.extract()?; + if let Ok(x) = i8::try_from(x) { + visitor.visit_i8(x) + } else if let Ok(x) = i16::try_from(x) { + visitor.visit_i16(x) + } else if let Ok(x) = i32::try_from(x) { + visitor.visit_i32(x) + } else if let Ok(x) = i64::try_from(x) { + visitor.visit_i64(x) + } else { + visitor.visit_i128(x) + } + } + } } macro_rules! deserialize_type { @@ -99,8 +131,8 @@ impl<'a, 'py, 'de, 'bound> de::Deserializer<'de> for &'a mut Depythonizer<'py, ' self.deserialize_unit(visitor) } else if obj.is_instance_of::() { self.deserialize_bool(visitor) - } else if obj.is_instance_of::() { - self.deserialize_i64(visitor) + } else if let Ok(x) = obj.downcast::() { + self.deserialize_any_int(x, visitor) } else if obj.is_instance_of::() || obj.is_instance_of::() { self.deserialize_tuple(obj.len()?, visitor) } else if obj.is_instance_of::() { @@ -151,10 +183,12 @@ impl<'a, 'py, 'de, 'bound> de::Deserializer<'de> for &'a mut Depythonizer<'py, ' deserialize_type!(deserialize_i16 => visit_i16); deserialize_type!(deserialize_i32 => visit_i32); deserialize_type!(deserialize_i64 => visit_i64); + deserialize_type!(deserialize_i128 => visit_i128); deserialize_type!(deserialize_u8 => visit_u8); deserialize_type!(deserialize_u16 => visit_u16); deserialize_type!(deserialize_u32 => visit_u32); deserialize_type!(deserialize_u64 => visit_u64); + deserialize_type!(deserialize_u128 => visit_u128); deserialize_type!(deserialize_f32 => visit_f32); deserialize_type!(deserialize_f64 => visit_f64); @@ -459,7 +493,7 @@ mod test { use super::*; use crate::error::ErrorImpl; use maplit::hashmap; - use pyo3::Python; + use pyo3::{IntoPy, Python}; use serde_json::{json, Value as JsonValue}; fn test_de(code: &str, expected: &T, expected_json: &JsonValue) @@ -749,4 +783,21 @@ mod test { let code = "{'name': 'SomeFoo', 'bar': {'value': 13, 'variant': {'Tuple': [-1.5, 8]}}}"; test_de(code, &expected, &expected_json); } + + #[test] + fn test_int_limits() { + Python::with_gil(|py| { + // serde_json::Value supports u64 and i64 as maxiumum sizes + let _: serde_json::Value = depythonize(&u64::MAX.into_py(py).into_bound(py)).unwrap(); + let _: serde_json::Value = depythonize(&u64::MIN.into_py(py).into_bound(py)).unwrap(); + let _: serde_json::Value = depythonize(&i64::MAX.into_py(py).into_bound(py)).unwrap(); + let _: serde_json::Value = depythonize(&i64::MIN.into_py(py).into_bound(py)).unwrap(); + + let _: u128 = depythonize(&u128::MAX.into_py(py).into_bound(py)).unwrap(); + let _: i128 = depythonize(&u128::MIN.into_py(py).into_bound(py)).unwrap(); + + let _: i128 = depythonize(&i128::MAX.into_py(py).into_bound(py)).unwrap(); + let _: i128 = depythonize(&i128::MIN.into_py(py).into_bound(py)).unwrap(); + }); + } } From 5996e628652e25f2d55ccee018358e39589c2ef5 Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Sat, 3 Aug 2024 15:32:02 +0100 Subject: [PATCH 2/3] fix macos workflows --- .github/workflows/ci.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9ec1f92..2c7c6d6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,9 +36,10 @@ jobs: strategy: fail-fast: false # If one platform fails, allow the rest to keep testing. matrix: + python-architecture: ["x64"] python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] os: [ - "macos-latest", + "macos-13", "ubuntu-latest", "windows-latest", ] @@ -47,6 +48,10 @@ jobs: - python-version: "3.12" os: "ubuntu-latest" rust: "1.56" + - python-version: "3.12" + python-architecture: "arm64" + os: "macos-latest" + rust: "stable" steps: - uses: actions/checkout@v4 @@ -55,7 +60,7 @@ jobs: uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - architecture: x64 + architecture: ${{ matrix.python-python-architecture }} - name: Install Rust toolchain uses: dtolnay/rust-toolchain@master From 61192c03da3fa819ff4594f5b18dde9e7850bd2c Mon Sep 17 00:00:00 2001 From: David Hewitt Date: Sat, 3 Aug 2024 15:39:18 +0100 Subject: [PATCH 3/3] extend coverage --- src/de.rs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/de.rs b/src/de.rs index 5aa4b34..b22bbdd 100644 --- a/src/de.rs +++ b/src/de.rs @@ -788,6 +788,21 @@ mod test { fn test_int_limits() { Python::with_gil(|py| { // serde_json::Value supports u64 and i64 as maxiumum sizes + let _: serde_json::Value = depythonize(&u8::MAX.into_py(py).into_bound(py)).unwrap(); + let _: serde_json::Value = depythonize(&u8::MIN.into_py(py).into_bound(py)).unwrap(); + let _: serde_json::Value = depythonize(&i8::MAX.into_py(py).into_bound(py)).unwrap(); + let _: serde_json::Value = depythonize(&i8::MIN.into_py(py).into_bound(py)).unwrap(); + + let _: serde_json::Value = depythonize(&u16::MAX.into_py(py).into_bound(py)).unwrap(); + let _: serde_json::Value = depythonize(&u16::MIN.into_py(py).into_bound(py)).unwrap(); + let _: serde_json::Value = depythonize(&i16::MAX.into_py(py).into_bound(py)).unwrap(); + let _: serde_json::Value = depythonize(&i16::MIN.into_py(py).into_bound(py)).unwrap(); + + let _: serde_json::Value = depythonize(&u32::MAX.into_py(py).into_bound(py)).unwrap(); + let _: serde_json::Value = depythonize(&u32::MIN.into_py(py).into_bound(py)).unwrap(); + let _: serde_json::Value = depythonize(&i32::MAX.into_py(py).into_bound(py)).unwrap(); + let _: serde_json::Value = depythonize(&i32::MIN.into_py(py).into_bound(py)).unwrap(); + let _: serde_json::Value = depythonize(&u64::MAX.into_py(py).into_bound(py)).unwrap(); let _: serde_json::Value = depythonize(&u64::MIN.into_py(py).into_bound(py)).unwrap(); let _: serde_json::Value = depythonize(&i64::MAX.into_py(py).into_bound(py)).unwrap();