Port upstream c0f9647: one temporal encoder for the Python JSON writer

src/arcadedb_embedded/results.py

@@ -529,6 +529,18 @@ def to_json_list(self, batch_size: int = 10_000) -> List[Dict[str, Any]]:
         floats. Use ``to_list()`` when you want ``date``, ``datetime`` and
         ``Decimal`` values: it costs about the same.
 
+        The encoding is one rule for every row, at the top level and inside a
+        list or a map: a DATE is the epoch milliseconds of its UTC midnight (not
+        a count of days) whatever the JVM time zone, the same number
+        ``to_columns()`` carries as ``datetime64[ms]``. A date too far from 1970
+        for a long of milliseconds (past about year 292 million) arrives as its
+        ISO-8601 text instead of failing the batch.
+
+        This is NOT the form of ``Result.to_json()`` for a record row (``SELECT
+        FROM T``), which is the engine's round-trip JSON of the record: there a
+        DATE column at the top level is epoch DAYS, with the ``@rid``, ``@type``
+        and ``@cat`` metadata. Do not mix the two when decoding a DATE.
+
         Args:
             batch_size: Rows serialized per Java crossing (default 10000)
 
@@ -1303,6 +1315,18 @@ def to_json(self) -> str:
         """
         Convert result to JSON string.
 
+        This is the engine's own JSON of the row. For a record row (``SELECT
+        FROM T``) it is the record's round-trip form, which ``fromJSON`` reads
+        back: it carries the ``@rid``, ``@type`` and ``@cat`` metadata and writes
+        a top-level DATE as epoch DAYS (a DATETIME as epoch milliseconds, or as
+        text when its column keeps microseconds or nanoseconds). A DATE inside a
+        list or a map, and a DATE in a projection row (``SELECT a, b FROM T``),
+        is written as the epoch milliseconds of its UTC midnight, the form
+        ``ResultSet.to_json_list()`` uses everywhere; a DATETIME in a projection
+        row is written as text in the schema's date-time format. The engine's
+        JSON is not one encoding across row shapes, which is why
+        ``to_json_list()`` does not delegate to it.
+
         Returns:
             JSON string representation
 

src/java/com/arcadedb/python/ColumnBatcher.java

@@ -42,7 +42,10 @@ private ColumnBatcher() {
   }
 
   /** Wall-clock-consistent epoch millis: naive temporals are read as UTC so
-   * numpy's (naive) datetime64 shows the same wall-clock the record carries. */
+   * numpy's (naive) datetime64 shows the same wall-clock the record carries.
+   * A DATE is its UTC midnight in milliseconds, not a count of days: the column is a datetime64[ms], and it is the
+   * number RowBatcher gets from the engine's JSONObject for the same value (#10522, test_temporal_writers_agree.py).
+   * It is not a JSON writer, so there is nothing for it to disagree with in Result.toJSON(). */
   private static long toEpochMillis(final Object v) {
     if (v instanceof Date d)
       return d.getTime();

src/java/com/arcadedb/python/RowBatcher.java

@@ -12,6 +12,16 @@
  * which since 26.7.2 (#4967) serializes primitive arrays like float[] as
  * real JSON arrays natively.
  *
+ * This class has NO temporal encoding of its own (#10522): every date and
+ * date-time is written by JSONObject.put, the writer the engine uses for the
+ * rows of a projection and for the members of a list or a map of any row, and
+ * ColumnBatcher's epoch milliseconds are the same numbers. A DATE is the epoch
+ * milliseconds of its UTC midnight, never a count of days, whatever the JVM
+ * time zone. The one place the engine writes days is the top level of a
+ * record's own toJSON() (Result.to_json() of a SELECT FROM T row), which is
+ * the form fromJSON() reads back for a DATE column; to_json_list() is not that
+ * round trip, and test_temporal_writers_agree.py pins both.
+ *
  * Compiled into arcadedb-python-bridge.jar during the wheel build
  * (scripts/Dockerfile.build and scripts/build-native.sh) and consumed by
  * ResultSet.to_json_list() in the Python bindings. Values carry JSON-native
@@ -24,14 +34,6 @@
 import com.arcadedb.query.sql.executor.ResultSet;
 import com.arcadedb.serializer.json.JSONObject;
 
-import java.time.LocalDate;
-import java.time.ZoneOffset;
-import java.util.ArrayList;
-import java.util.Calendar;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-
 public final class RowBatcher {
 
   private RowBatcher() {
@@ -79,49 +81,14 @@ static void closeDrained(final ResultSet rs) {
 
   private static void appendRow(final StringBuilder sb, final Result row) {
     final JSONObject obj = new JSONObject();
-    for (final String property : row.getPropertyNames())
-      obj.put(property, utcDates(row.getProperty(property)));
-    sb.append(obj);
-  }
-
-  /**
-   * JSONObject.put writes a LocalDate as the epoch milliseconds of midnight in the JVM's default time zone, while
-   * Result.toJSON() and a DATETIME (a UTC wall clock) are written in UTC, so the same row read twice gave two integers
-   * and decoded to the previous day east of UTC. A DATE is written as midnight UTC here, at the top level and inside a
-   * list or map. A collection without a LocalDate is returned as it is.
-   * <p>
-   * A java.util.Calendar (arcadedb.dateTimeImplementation / dateImplementation) is written as its epoch milliseconds, as
-   * a java.util.Date is. The engine's JSONObject wrote it as its toString() before #9630, and this bridge runs against
-   * whatever engine the wheel bundles, so it does not rely on that fix.
-   */
-  private static Object utcDates(final Object value) {
-    if (value instanceof LocalDate date)
-      return date.atStartOfDay(ZoneOffset.UTC).toInstant().toEpochMilli();
-    if (value instanceof Calendar calendar)
-      return calendar.getTimeInMillis();
-    if (value instanceof List<?> list) {
-      List<Object> copy = null;
-      for (int i = 0; i < list.size(); i++) {
-        final Object element = list.get(i);
-        final Object converted = utcDates(element);
-        if (converted != element && copy == null)
-          copy = new ArrayList<>(list);
-        if (copy != null)
-          copy.set(i, converted);
-      }
-      return copy != null ? copy : value;
+    for (final String property : row.getPropertyNames()) {
+      // No conversion of our own: JSONObject.put anchors a LocalDate at UTC midnight whatever the JVM zone (#9037), writes a
+      // java.util.Calendar as its epoch milliseconds like a Date (#9630), and writes a date past the range of a long of
+      // milliseconds as its ISO text (#10483), where the encoder this replaced threw and failed the whole batch. The members
+      // of a list or a map go the same way
+      final Object value = row.getProperty(property);
+      obj.put(property, value);
     }
-    if (value instanceof Map<?, ?> map) {
-      Map<Object, Object> copy = null;
-      for (final Map.Entry<?, ?> entry : map.entrySet()) {
-        final Object converted = utcDates(entry.getValue());
-        if (converted != entry.getValue() && copy == null)
-          copy = new LinkedHashMap<>(map);
-        if (copy != null)
-          copy.put(entry.getKey(), converted);
-      }
-      return copy != null ? copy : value;
-    }
-    return value;
+    sb.append(obj);
   }
 }

tests/test_bridge_fixes.py

@@ -85,7 +85,10 @@ def test_to_json_list_writes_a_date_as_midnight_utc_in_any_jvm_zone(tmp_path, zo
     it writes this undeclared LocalDate as days since the epoch, as it writes a
     declared DATE; before that it wrote midnight UTC milliseconds. The engine is
     asked which of the two it does (a version number cannot tell: 26.11.1
-    snapshots from before #10253 exist), and the test holds it to that one."""
+    snapshots from before #10253 exist), and the test holds it to that one.
+    That the bridge writes milliseconds where the record's own JSON writes days
+    is a decision and not a drift (#10522): test_temporal_writers_agree.py pins
+    it, and pins that every bridge writer agrees with the engine's generic one."""
     proc = subprocess.run(  # nosec B603 - fixed argv, no shell, test-owned
         [sys.executable, "-c", _DATE_SCRIPT, str(tmp_path / "db"), zone],
         capture_output=True,

tests/test_temporal_writers_agree.py

@@ -0,0 +1,172 @@
+"""The bridge's bulk writers have one temporal encoding, and it is the engine's (#10522).
+
+``to_json_list()`` (``RowBatcher``) and ``to_columns()`` (``ColumnBatcher``) used to carry an encoder of their
+own for a date, written when the engine had no type for a ``LocalDate``. The engine has since fixed what that
+encoder worked around (a ``LocalDate`` is UTC midnight whatever the JVM zone, a ``Calendar`` is its milliseconds,
+a date past the range of a long is text), so the private copy only differed from the engine's generic JSON writer
+in the one place it was worse: a date past year 292 million threw and failed the whole batch.
+
+What these tests hold is the rule the issue asks for, in a form that survives the next engine-side type mapping:
+for every temporal type the JSON writer, the columnar writer and ``JSONObject.put`` write the same number, at the
+top level and inside a list or a map, in a record row and in a projection row. The one number that differs, a
+DATE column at the top level of ``Result.to_json()`` of a record row (days), is the engine's round-trip form of
+the record and is pinned separately, so its difference is a decision written down and not a drift.
+
+Author: Luca Garulli (l.garulli@arcadedata.com)
+"""
+
+import datetime as dt
+import json
+
+import arcadedb_embedded as arcadedb  # noqa: F401  (starts the JVM through the fixtures)
+import jpype
+import numpy as np
+import pytest
+from arcadedb_embedded import results
+from arcadedb_embedded.exceptions import ArcadeDBError
+
+MIDNIGHT_UTC_2024_01_02 = 1704153600000
+DAY_MS = 86_400_000
+
+
+def _in_memory(rows):
+    """A result set over rows built in Java, for values SQL cannot produce."""
+    internal = jpype.JClass("com.arcadedb.query.sql.executor.InternalResultSet")()
+    result_internal = jpype.JClass("com.arcadedb.query.sql.executor.ResultInternal")
+    for row in rows:
+        java_row = result_internal()
+        for key, value in row.items():
+            java_row.setProperty(key, value)
+        internal.add(java_row)
+    return internal
+
+
+def _java_temporals():
+    """One value of every temporal class a row can hold, with sub-millisecond digits and a pre-epoch instant."""
+    java = jpype.JClass
+    instant = java("java.time.Instant").parse("2024-01-02T03:04:05.123456789Z")
+    calendar = java("java.util.Calendar").getInstance(
+        java("java.util.TimeZone").getTimeZone("UTC")
+    )
+    calendar.setTimeInMillis(jpype.JLong(1704164645123))
+    return {
+        "local_date": java("java.time.LocalDate").of(2024, 1, 2),
+        "pre_epoch_date": java("java.time.LocalDate").of(1969, 12, 31),
+        "local_date_time": java("java.time.LocalDateTime").of(
+            2024, 1, 2, 3, 4, 5, 123_456_789
+        ),
+        "pre_epoch_date_time": java("java.time.LocalDateTime").of(
+            1969, 12, 31, 23, 59, 59, 999_500_000
+        ),
+        "instant": instant,
+        "zoned": java("java.time.ZonedDateTime").ofInstant(
+            instant, java("java.time.ZoneId").of("Asia/Seoul")
+        ),
+        "offset": java("java.time.OffsetDateTime").parse("2024-01-02T03:04:05.5+09:00"),
+        "util_date": java("java.util.Date")(jpype.JLong(1704164645123)),
+        "calendar": calendar,
+    }
+
+
+def _engine_json(name, value):
+    """What the engine's generic JSON writer makes of ``value``: the parsed text, or the exception's class name."""
+    json_object = jpype.JClass("com.arcadedb.serializer.json.JSONObject")()
+    try:
+        json_object.put(name, value)
+    except jpype.JException as exc:
+        return {"raises": str(exc.getClass().getName())}
+    return json.loads(str(json_object.toString()))
+
+
+def test_the_json_and_columnar_writers_write_the_same_number_for_every_temporal(
+    temp_db,
+):
+    values = _java_temporals()
+
+    as_json = results.ResultSet(_in_memory([values]), temp_db).to_json_list()[0]
+    columns = results.ResultSet(_in_memory([values]), temp_db).to_columns()
+
+    assert set(columns) == set(values)
+    for name in values:
+        column = columns[name]
+        assert column.dtype == np.dtype("datetime64[ms]"), name
+        assert int(column[0].astype("int64")) == as_json[name], name
+
+
+def test_the_json_writer_is_the_engines_generic_writer_for_every_temporal(temp_db):
+    """No encoder of its own: if the engine's JSONObject changes how it writes a type, this fails and a decision is made."""
+    values = _java_temporals()
+    # a date past the range of a long of milliseconds: text for an engine that has #10483, a failure for one that has not,
+    # and the bridge follows either way
+    values["far_date"] = jpype.JClass("java.time.LocalDate").of(999_999_999, 12, 31)
+    dates = jpype.JClass("java.util.ArrayList")()
+    dates.add(values["local_date"])
+    dates.add(values["pre_epoch_date"])
+    values["dates"] = dates
+    members = jpype.JClass("java.util.LinkedHashMap")()
+    members.put("on", values["local_date"])
+    values["meta"] = members
+
+    for name, value in values.items():
+        expected = _engine_json(name, value)
+        if "raises" in expected:
+            # the bridge follows the engine it bundles, and the bindings raise a Java failure of a read as ArcadeDBError
+            with pytest.raises(ArcadeDBError):
+                results.ResultSet(_in_memory([{name: value}]), temp_db).to_json_list()
+        else:
+            row = results.ResultSet(_in_memory([{name: value}]), temp_db).to_json_list()
+            assert row == [expected], name
+
+
+def test_a_date_is_one_number_in_a_record_row_a_projection_row_a_list_and_a_map(
+    temp_db,
+):
+    temp_db.command("sql", "CREATE DOCUMENT TYPE Event")
+    temp_db.command("sql", "CREATE PROPERTY Event.day DATE")
+    with temp_db.transaction():
+        temp_db.command(
+            "sql",
+            "INSERT INTO Event SET day = :day, days = :days, meta = :meta",
+            {
+                "day": dt.date(2024, 1, 2),
+                "days": [dt.date(2024, 1, 2)],
+                "meta": {"on": dt.date(2024, 1, 2)},
+            },
+        )
+
+    record_row = temp_db.query("sql", "SELECT FROM Event").to_json_list()[0]
+    projection_row = temp_db.query(
+        "sql", "SELECT day, days, meta FROM Event"
+    ).to_json_list()[0]
+
+    expected = {
+        "day": MIDNIGHT_UTC_2024_01_02,
+        "days": [MIDNIGHT_UTC_2024_01_02],
+        "meta": {"on": MIDNIGHT_UTC_2024_01_02},
+    }
+    assert {key: record_row[key] for key in expected} == expected
+    assert projection_row == expected
+
+
+def test_result_to_json_writes_the_top_level_date_of_a_record_row_as_days(temp_db):
+    """The one place the engine writes days: the round-trip JSON of the record, pinned so the difference is a decision.
+
+    A DATE column read back by ``fromJSON`` is a count of days, which is why ``Result.to_json()`` of a record row
+    writes it so. ``to_json_list()`` is a read-only bulk transport and writes milliseconds everywhere, so a caller
+    decodes every temporal the same way; the two differ by exactly the milliseconds in a day, and by nothing else.
+    """
+    temp_db.command("sql", "CREATE DOCUMENT TYPE Event")
+    temp_db.command("sql", "CREATE PROPERTY Event.day DATE")
+    with temp_db.transaction():
+        temp_db.command(
+            "sql",
+            "INSERT INTO Event SET day = :day",
+            {"day": dt.date(2024, 1, 2)},
+        )
+
+    single = json.loads(temp_db.query("sql", "SELECT FROM Event").first().to_json())
+    batch = temp_db.query("sql", "SELECT FROM Event").to_json_list()[0]
+
+    assert single["day"] == MIDNIGHT_UTC_2024_01_02 // DAY_MS
+    assert batch["day"] == single["day"] * DAY_MS
+    assert "@rid" in single and "@rid" not in batch