diff --git a/src/arcadedb_embedded/results.py b/src/arcadedb_embedded/results.py index fb65f82..a8f78f8 100644 --- a/src/arcadedb_embedded/results.py +++ b/src/arcadedb_embedded/results.py @@ -529,6 +529,18 @@ def to_json_list(self, batch_size: int = 10_000) -> List[Dict[str, Any]]: floats. Use ``to_list()`` when you want ``date``, ``datetime`` and ``Decimal`` values: it costs about the same. + The encoding is one rule for every row, at the top level and inside a + list or a map: a DATE is the epoch milliseconds of its UTC midnight (not + a count of days) whatever the JVM time zone, the same number + ``to_columns()`` carries as ``datetime64[ms]``. A date too far from 1970 + for a long of milliseconds (past about year 292 million) arrives as its + ISO-8601 text instead of failing the batch. + + This is NOT the form of ``Result.to_json()`` for a record row (``SELECT + FROM T``), which is the engine's round-trip JSON of the record: there a + DATE column at the top level is epoch DAYS, with the ``@rid``, ``@type`` + and ``@cat`` metadata. Do not mix the two when decoding a DATE. + Args: batch_size: Rows serialized per Java crossing (default 10000) @@ -1303,6 +1315,18 @@ def to_json(self) -> str: """ Convert result to JSON string. + This is the engine's own JSON of the row. For a record row (``SELECT + FROM T``) it is the record's round-trip form, which ``fromJSON`` reads + back: it carries the ``@rid``, ``@type`` and ``@cat`` metadata and writes + a top-level DATE as epoch DAYS (a DATETIME as epoch milliseconds, or as + text when its column keeps microseconds or nanoseconds). A DATE inside a + list or a map, and a DATE in a projection row (``SELECT a, b FROM T``), + is written as the epoch milliseconds of its UTC midnight, the form + ``ResultSet.to_json_list()`` uses everywhere; a DATETIME in a projection + row is written as text in the schema's date-time format. The engine's + JSON is not one encoding across row shapes, which is why + ``to_json_list()`` does not delegate to it. + Returns: JSON string representation diff --git a/src/java/com/arcadedb/python/ColumnBatcher.java b/src/java/com/arcadedb/python/ColumnBatcher.java index fbedcef..6211ac0 100644 --- a/src/java/com/arcadedb/python/ColumnBatcher.java +++ b/src/java/com/arcadedb/python/ColumnBatcher.java @@ -42,7 +42,10 @@ private ColumnBatcher() { } /** Wall-clock-consistent epoch millis: naive temporals are read as UTC so - * numpy's (naive) datetime64 shows the same wall-clock the record carries. */ + * numpy's (naive) datetime64 shows the same wall-clock the record carries. + * A DATE is its UTC midnight in milliseconds, not a count of days: the column is a datetime64[ms], and it is the + * number RowBatcher gets from the engine's JSONObject for the same value (#10522, test_temporal_writers_agree.py). + * It is not a JSON writer, so there is nothing for it to disagree with in Result.toJSON(). */ private static long toEpochMillis(final Object v) { if (v instanceof Date d) return d.getTime(); diff --git a/src/java/com/arcadedb/python/RowBatcher.java b/src/java/com/arcadedb/python/RowBatcher.java index a08edc7..74249cb 100644 --- a/src/java/com/arcadedb/python/RowBatcher.java +++ b/src/java/com/arcadedb/python/RowBatcher.java @@ -12,6 +12,16 @@ * which since 26.7.2 (#4967) serializes primitive arrays like float[] as * real JSON arrays natively. * + * This class has NO temporal encoding of its own (#10522): every date and + * date-time is written by JSONObject.put, the writer the engine uses for the + * rows of a projection and for the members of a list or a map of any row, and + * ColumnBatcher's epoch milliseconds are the same numbers. A DATE is the epoch + * milliseconds of its UTC midnight, never a count of days, whatever the JVM + * time zone. The one place the engine writes days is the top level of a + * record's own toJSON() (Result.to_json() of a SELECT FROM T row), which is + * the form fromJSON() reads back for a DATE column; to_json_list() is not that + * round trip, and test_temporal_writers_agree.py pins both. + * * Compiled into arcadedb-python-bridge.jar during the wheel build * (scripts/Dockerfile.build and scripts/build-native.sh) and consumed by * ResultSet.to_json_list() in the Python bindings. Values carry JSON-native @@ -24,14 +34,6 @@ import com.arcadedb.query.sql.executor.ResultSet; import com.arcadedb.serializer.json.JSONObject; -import java.time.LocalDate; -import java.time.ZoneOffset; -import java.util.ArrayList; -import java.util.Calendar; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; - public final class RowBatcher { private RowBatcher() { @@ -79,49 +81,14 @@ static void closeDrained(final ResultSet rs) { private static void appendRow(final StringBuilder sb, final Result row) { final JSONObject obj = new JSONObject(); - for (final String property : row.getPropertyNames()) - obj.put(property, utcDates(row.getProperty(property))); - sb.append(obj); - } - - /** - * JSONObject.put writes a LocalDate as the epoch milliseconds of midnight in the JVM's default time zone, while - * Result.toJSON() and a DATETIME (a UTC wall clock) are written in UTC, so the same row read twice gave two integers - * and decoded to the previous day east of UTC. A DATE is written as midnight UTC here, at the top level and inside a - * list or map. A collection without a LocalDate is returned as it is. - *

- * A java.util.Calendar (arcadedb.dateTimeImplementation / dateImplementation) is written as its epoch milliseconds, as - * a java.util.Date is. The engine's JSONObject wrote it as its toString() before #9630, and this bridge runs against - * whatever engine the wheel bundles, so it does not rely on that fix. - */ - private static Object utcDates(final Object value) { - if (value instanceof LocalDate date) - return date.atStartOfDay(ZoneOffset.UTC).toInstant().toEpochMilli(); - if (value instanceof Calendar calendar) - return calendar.getTimeInMillis(); - if (value instanceof List list) { - List copy = null; - for (int i = 0; i < list.size(); i++) { - final Object element = list.get(i); - final Object converted = utcDates(element); - if (converted != element && copy == null) - copy = new ArrayList<>(list); - if (copy != null) - copy.set(i, converted); - } - return copy != null ? copy : value; + for (final String property : row.getPropertyNames()) { + // No conversion of our own: JSONObject.put anchors a LocalDate at UTC midnight whatever the JVM zone (#9037), writes a + // java.util.Calendar as its epoch milliseconds like a Date (#9630), and writes a date past the range of a long of + // milliseconds as its ISO text (#10483), where the encoder this replaced threw and failed the whole batch. The members + // of a list or a map go the same way + final Object value = row.getProperty(property); + obj.put(property, value); } - if (value instanceof Map map) { - Map copy = null; - for (final Map.Entry entry : map.entrySet()) { - final Object converted = utcDates(entry.getValue()); - if (converted != entry.getValue() && copy == null) - copy = new LinkedHashMap<>(map); - if (copy != null) - copy.put(entry.getKey(), converted); - } - return copy != null ? copy : value; - } - return value; + sb.append(obj); } } diff --git a/tests/test_bridge_fixes.py b/tests/test_bridge_fixes.py index c84a8bc..b998926 100644 --- a/tests/test_bridge_fixes.py +++ b/tests/test_bridge_fixes.py @@ -85,7 +85,10 @@ def test_to_json_list_writes_a_date_as_midnight_utc_in_any_jvm_zone(tmp_path, zo it writes this undeclared LocalDate as days since the epoch, as it writes a declared DATE; before that it wrote midnight UTC milliseconds. The engine is asked which of the two it does (a version number cannot tell: 26.11.1 - snapshots from before #10253 exist), and the test holds it to that one.""" + snapshots from before #10253 exist), and the test holds it to that one. + That the bridge writes milliseconds where the record's own JSON writes days + is a decision and not a drift (#10522): test_temporal_writers_agree.py pins + it, and pins that every bridge writer agrees with the engine's generic one.""" proc = subprocess.run( # nosec B603 - fixed argv, no shell, test-owned [sys.executable, "-c", _DATE_SCRIPT, str(tmp_path / "db"), zone], capture_output=True, diff --git a/tests/test_temporal_writers_agree.py b/tests/test_temporal_writers_agree.py new file mode 100644 index 0000000..f37cdb2 --- /dev/null +++ b/tests/test_temporal_writers_agree.py @@ -0,0 +1,172 @@ +"""The bridge's bulk writers have one temporal encoding, and it is the engine's (#10522). + +``to_json_list()`` (``RowBatcher``) and ``to_columns()`` (``ColumnBatcher``) used to carry an encoder of their +own for a date, written when the engine had no type for a ``LocalDate``. The engine has since fixed what that +encoder worked around (a ``LocalDate`` is UTC midnight whatever the JVM zone, a ``Calendar`` is its milliseconds, +a date past the range of a long is text), so the private copy only differed from the engine's generic JSON writer +in the one place it was worse: a date past year 292 million threw and failed the whole batch. + +What these tests hold is the rule the issue asks for, in a form that survives the next engine-side type mapping: +for every temporal type the JSON writer, the columnar writer and ``JSONObject.put`` write the same number, at the +top level and inside a list or a map, in a record row and in a projection row. The one number that differs, a +DATE column at the top level of ``Result.to_json()`` of a record row (days), is the engine's round-trip form of +the record and is pinned separately, so its difference is a decision written down and not a drift. + +Author: Luca Garulli (l.garulli@arcadedata.com) +""" + +import datetime as dt +import json + +import arcadedb_embedded as arcadedb # noqa: F401 (starts the JVM through the fixtures) +import jpype +import numpy as np +import pytest +from arcadedb_embedded import results +from arcadedb_embedded.exceptions import ArcadeDBError + +MIDNIGHT_UTC_2024_01_02 = 1704153600000 +DAY_MS = 86_400_000 + + +def _in_memory(rows): + """A result set over rows built in Java, for values SQL cannot produce.""" + internal = jpype.JClass("com.arcadedb.query.sql.executor.InternalResultSet")() + result_internal = jpype.JClass("com.arcadedb.query.sql.executor.ResultInternal") + for row in rows: + java_row = result_internal() + for key, value in row.items(): + java_row.setProperty(key, value) + internal.add(java_row) + return internal + + +def _java_temporals(): + """One value of every temporal class a row can hold, with sub-millisecond digits and a pre-epoch instant.""" + java = jpype.JClass + instant = java("java.time.Instant").parse("2024-01-02T03:04:05.123456789Z") + calendar = java("java.util.Calendar").getInstance( + java("java.util.TimeZone").getTimeZone("UTC") + ) + calendar.setTimeInMillis(jpype.JLong(1704164645123)) + return { + "local_date": java("java.time.LocalDate").of(2024, 1, 2), + "pre_epoch_date": java("java.time.LocalDate").of(1969, 12, 31), + "local_date_time": java("java.time.LocalDateTime").of( + 2024, 1, 2, 3, 4, 5, 123_456_789 + ), + "pre_epoch_date_time": java("java.time.LocalDateTime").of( + 1969, 12, 31, 23, 59, 59, 999_500_000 + ), + "instant": instant, + "zoned": java("java.time.ZonedDateTime").ofInstant( + instant, java("java.time.ZoneId").of("Asia/Seoul") + ), + "offset": java("java.time.OffsetDateTime").parse("2024-01-02T03:04:05.5+09:00"), + "util_date": java("java.util.Date")(jpype.JLong(1704164645123)), + "calendar": calendar, + } + + +def _engine_json(name, value): + """What the engine's generic JSON writer makes of ``value``: the parsed text, or the exception's class name.""" + json_object = jpype.JClass("com.arcadedb.serializer.json.JSONObject")() + try: + json_object.put(name, value) + except jpype.JException as exc: + return {"raises": str(exc.getClass().getName())} + return json.loads(str(json_object.toString())) + + +def test_the_json_and_columnar_writers_write_the_same_number_for_every_temporal( + temp_db, +): + values = _java_temporals() + + as_json = results.ResultSet(_in_memory([values]), temp_db).to_json_list()[0] + columns = results.ResultSet(_in_memory([values]), temp_db).to_columns() + + assert set(columns) == set(values) + for name in values: + column = columns[name] + assert column.dtype == np.dtype("datetime64[ms]"), name + assert int(column[0].astype("int64")) == as_json[name], name + + +def test_the_json_writer_is_the_engines_generic_writer_for_every_temporal(temp_db): + """No encoder of its own: if the engine's JSONObject changes how it writes a type, this fails and a decision is made.""" + values = _java_temporals() + # a date past the range of a long of milliseconds: text for an engine that has #10483, a failure for one that has not, + # and the bridge follows either way + values["far_date"] = jpype.JClass("java.time.LocalDate").of(999_999_999, 12, 31) + dates = jpype.JClass("java.util.ArrayList")() + dates.add(values["local_date"]) + dates.add(values["pre_epoch_date"]) + values["dates"] = dates + members = jpype.JClass("java.util.LinkedHashMap")() + members.put("on", values["local_date"]) + values["meta"] = members + + for name, value in values.items(): + expected = _engine_json(name, value) + if "raises" in expected: + # the bridge follows the engine it bundles, and the bindings raise a Java failure of a read as ArcadeDBError + with pytest.raises(ArcadeDBError): + results.ResultSet(_in_memory([{name: value}]), temp_db).to_json_list() + else: + row = results.ResultSet(_in_memory([{name: value}]), temp_db).to_json_list() + assert row == [expected], name + + +def test_a_date_is_one_number_in_a_record_row_a_projection_row_a_list_and_a_map( + temp_db, +): + temp_db.command("sql", "CREATE DOCUMENT TYPE Event") + temp_db.command("sql", "CREATE PROPERTY Event.day DATE") + with temp_db.transaction(): + temp_db.command( + "sql", + "INSERT INTO Event SET day = :day, days = :days, meta = :meta", + { + "day": dt.date(2024, 1, 2), + "days": [dt.date(2024, 1, 2)], + "meta": {"on": dt.date(2024, 1, 2)}, + }, + ) + + record_row = temp_db.query("sql", "SELECT FROM Event").to_json_list()[0] + projection_row = temp_db.query( + "sql", "SELECT day, days, meta FROM Event" + ).to_json_list()[0] + + expected = { + "day": MIDNIGHT_UTC_2024_01_02, + "days": [MIDNIGHT_UTC_2024_01_02], + "meta": {"on": MIDNIGHT_UTC_2024_01_02}, + } + assert {key: record_row[key] for key in expected} == expected + assert projection_row == expected + + +def test_result_to_json_writes_the_top_level_date_of_a_record_row_as_days(temp_db): + """The one place the engine writes days: the round-trip JSON of the record, pinned so the difference is a decision. + + A DATE column read back by ``fromJSON`` is a count of days, which is why ``Result.to_json()`` of a record row + writes it so. ``to_json_list()`` is a read-only bulk transport and writes milliseconds everywhere, so a caller + decodes every temporal the same way; the two differ by exactly the milliseconds in a day, and by nothing else. + """ + temp_db.command("sql", "CREATE DOCUMENT TYPE Event") + temp_db.command("sql", "CREATE PROPERTY Event.day DATE") + with temp_db.transaction(): + temp_db.command( + "sql", + "INSERT INTO Event SET day = :day", + {"day": dt.date(2024, 1, 2)}, + ) + + single = json.loads(temp_db.query("sql", "SELECT FROM Event").first().to_json()) + batch = temp_db.query("sql", "SELECT FROM Event").to_json_list()[0] + + assert single["day"] == MIDNIGHT_UTC_2024_01_02 // DAY_MS + assert batch["day"] == single["day"] * DAY_MS + assert "@rid" in single and "@rid" not in batch