diff --git a/.github/workflows/test-python-bindings.yml b/.github/workflows/test-python-bindings.yml index ea23925..bf092cd 100644 --- a/.github/workflows/test-python-bindings.yml +++ b/.github/workflows/test-python-bindings.yml @@ -369,7 +369,7 @@ jobs: # to_dataframe() tests skipped themselves in this job while it # reported green. The guard step below now fails the build if any # test skips for a missing import, so the next drift is loud. - uv pip install --system test-install/*.whl pytest pytest-cov requests numpy "psycopg[binary]" redis neo4j pyarrow pandas polars "adbc-driver-postgresql>=1.12" + uv pip install --system test-install/*.whl pytest pytest-cov requests numpy "psycopg[binary]" redis neo4j pyarrow pandas polars "adbc-driver-postgresql>=1.12" "duckdb>=1.4.2" "ladybug>=0.21.2" "networkx>=3.4" - name: Run pytest on host id: pytest diff --git a/docs/antora.yml b/docs/antora.yml index 857714c..cb59a7c 100644 --- a/docs/antora.yml +++ b/docs/antora.yml @@ -1,9 +1,9 @@ name: arcadedb-python title: ArcadeDB Python # Unversioned, so the component is published at /arcadedb-python/. -version: ~ +version: nav: - - modules/ROOT/nav.adoc +- modules/ROOT/nav.adoc asciidoc: attributes: # Package names. Every page takes them from here. diff --git a/pyproject.toml b/pyproject.toml index 5097372..e2edaf8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,6 +68,11 @@ test = [ # rather than an install command, so no amount of care at install time # would have caught it. "pandas>=2.2.0", + # Reference engines for tests/test_equivalence.py (humemai/arcadedb-embedded-python#347): the same question asked + # of a second engine over the same data. All three have wheels for every CI platform, so the file never skips. + "duckdb>=1.4.2", + "ladybug>=0.21.2", + "networkx>=3.4", ] dev = [ "black", diff --git a/tests/conftest.py b/tests/conftest.py index 1f1e4d4..732373d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -18,6 +18,10 @@ collect_ignore.append("test_sigint.py") # test_python_function_interrupt.py sends SIGINT to its own process, which Windows cannot do either collect_ignore.append("test_python_function_interrupt.py") + # LadybugDB's compiled module does not load in this process on Windows (it falls back to a C-API library its + # wheel does not ship), so the Cypher half of the equivalence checks runs on Linux and macOS + # (humemai/arcadedb-embedded-python#347) + collect_ignore.append("test_equivalence_ladybug.py") # runs scripts/stage_jars.sh under bash with a fake docker; the bash a Windows # subprocess finds may be WSL's rather than Git Bash collect_ignore.append("test_stage_jars.py") diff --git a/tests/test_collection_equality_known_issues.py b/tests/test_collection_equality_known_issues.py new file mode 100644 index 0000000..b1421ff --- /dev/null +++ b/tests/test_collection_equality_known_issues.py @@ -0,0 +1,121 @@ +"""SQL `=` on a list or map and a list written from Python (known-issues.md), fixed in 26.11.1. + +`Document.set`, `Vertex.set` and `GraphBatch.create_vertices` store a Python `int` as a `Long`, +and a SQL literal such as `[1, 2]` parses to `Integer` elements. SQL `=` used to compare list and +map elements with Java `equals()`, so the list written from Python was not equal to the literal. +ArcadeData/arcadedb#10448 (fixed by ArcadeData/arcadedb#10457, in 26.11.1) makes `=` and `<>` compare the elements +numerically, as openCypher, `CONTAINSALL` and `GROUP BY` already did, and a `BY ITEM` index is no +longer used to answer a whole-list `=` (the query scans, so the answer is right). + +The workarounds that were documented (openCypher `=`, and a named SQL parameter, which crosses as +`Long` elements) must keep passing on every engine. The three SQL tests that were strict `xfail` +tripwires are plain tests now: they assert the fixed answer. + +CI builds its wheel from the engine jars of the `arcadedata/arcadedb` image. The 26.11.1-SNAPSHOT +image carries no build number and is republished only when upstream's deploy job succeeds, so it can hold +the engine from before ArcadeData/arcadedb#10457, with the same version string as the fixed engine. So the +version cannot tell the two apart, and `fixed` probes the one behavior that changed: two lists that differ only in +the element class are equal. On an engine without the fix the same tests assert the old answer (no +row), as `test_vector_sql.py` does for `where` (the repository accepts no skips, +scripts/check_test_skips.py). When 26.11.1 is released and the wheel the suite runs on is built +from it, delete `fixed` and the old-answer branches. +""" + +import pytest + +import arcadedb + + +@pytest.fixture +def db(temp_db_path): + with arcadedb.create_database(temp_db_path) as db: + db.command("sql", "CREATE VERTEX TYPE V") + with db.transaction(): + v = db.new_vertex("V") + v.set("k", "set") + v.set("li", [1, 2]) + v.set("mp", {"a": 1}) + v.save() + with db.graph_batch(parallel_flush=False) as batch: + batch.create_vertices("V", [{"k": "batch", "li": [1, 2], "mp": {"a": 1}}]) + yield db + + +@pytest.fixture +def indexed_db(temp_db_path): + with arcadedb.create_database(temp_db_path) as db: + db.command("sql", "CREATE VERTEX TYPE X") + db.command("sql", "CREATE PROPERTY X.li LIST") + db.command("sql", "CREATE INDEX ON X (li BY ITEM) NOTUNIQUE") + with db.transaction(): + v = db.new_vertex("X") + v.set("k", "set") + v.set("li", [1, 2]) + v.save() + yield db + + +@pytest.fixture +def fixed(tmp_path): + """True when the engine has the #10448 fix (see the module docstring).""" + with arcadedb.create_database(str(tmp_path / "probe")) as probe: + row = probe.query("sql", "SELECT ([1, 2] = :p) AS eq", {"p": [1, 2]}).to_list() + return row[0]["eq"] is True + + +def _keys(db, language, query, *args): + return sorted(row["k"] for row in db.query(language, query, *args).to_list()) + + +def test_opencypher_equality_matches_lists_and_maps_written_from_python(db): + assert _keys( + db, "opencypher", "MATCH (n:V) WHERE n.li = [1, 2] RETURN n.k AS k" + ) == ["batch", "set"] + assert _keys( + db, "opencypher", "MATCH (n:V) WHERE n.mp = {a: 1} RETURN n.k AS k" + ) == ["batch", "set"] + + +def test_a_named_sql_parameter_matches_lists_written_from_python(db): + assert _keys(db, "sql", "SELECT k FROM V WHERE li = :p", {"p": [1, 2]}) == [ + "batch", + "set", + ] + + +def test_a_sql_list_literal_matches_lists_written_from_python(db, fixed): + assert _keys(db, "sql", "SELECT k FROM V WHERE li = [1, 2]") == ( + ["batch", "set"] if fixed else [] + ) + + +def test_a_sql_map_literal_matches_maps_written_from_python(db, fixed): + assert _keys(db, "sql", 'SELECT k FROM V WHERE mp = {"a": 1}') == ( + ["batch", "set"] if fixed else [] + ) + + +def test_sql_inequality_on_a_list_excludes_lists_written_from_python(db, fixed): + assert _keys(db, "sql", "SELECT k FROM V WHERE li <> [1, 2]") == ( + [] if fixed else ["batch", "set"] + ) + + +def test_opencypher_equality_matches_through_a_by_item_index(indexed_db): + assert _keys( + indexed_db, "opencypher", "MATCH (n:X) WHERE n.li = [1, 2] RETURN n.k AS k" + ) == ["set"] + + +def test_a_named_sql_parameter_matches_through_a_by_item_index(indexed_db, fixed): + # Before the fix, SQL = on a list with a BY ITEM index was answered from the index and + # matched nothing; now the query scans + assert _keys(indexed_db, "sql", "SELECT k FROM X WHERE li = :p", {"p": [1, 2]}) == ( + ["set"] if fixed else [] + ) + + +def test_a_sql_list_literal_matches_through_a_by_item_index(indexed_db, fixed): + assert _keys(indexed_db, "sql", "SELECT k FROM X WHERE li = [1, 2]") == ( + ["set"] if fixed else [] + ) diff --git a/tests/test_equivalence.py b/tests/test_equivalence.py new file mode 100644 index 0000000..525465b --- /dev/null +++ b/tests/test_equivalence.py @@ -0,0 +1,366 @@ +"""Equivalence checks (humemai/arcadedb-embedded-python#347): ArcadeDB answers like a reference engine over the same small seeded data. + +Cypher against LadybugDB (test_equivalence_ladybug.py, which shares this file's graph and queries), SQL against DuckDB, graph algorithms against NetworkX, and vector search against NumPy brute +force. ArcadeDB is also checked against itself: an index, a Graph Analytical View, and parameters in place of literals +must not change an answer. Answers are compared as multisets, or as sequences when the query orders, with every number +compared as a float rounded to 9 significant digits. +""" + +import datetime +import random + +import duckdb +import networkx as nx +import numpy as np +import pytest + +import arcadedb + +# Two directed rings of 60 people plus random edges inside each half: no dangling node, so PageRank needs no dangling +# rule, two weakly connected components, and every pair inside a half is reachable. No self-loops or parallel edges, +# whose counting differs between Cypher engines. +HALF = 60 +PEOPLE = [] +KNOWS = [] + + +def _seed_graph(): + r = random.Random(7) # nosec B311 - seeded test data, not a secret + for i in range(2 * HALF): + age = None if r.random() < 0.1 else r.randint(18, 70) + PEOPLE.append( + {"id": i, "age": age, "name": r.choice(["ann", "bob", "cy", "dee", "ed"])} + ) + pairs = set() + for base in (0, HALF): + for i in range(HALF): + pairs.add((base + i, base + (i + 1) % HALF)) + while len(pairs) < (base // HALF + 1) * 4 * HALF: + a, b = base + r.randrange(HALF), base + r.randrange(HALF) + if a != b: + pairs.add((a, b)) + for a, b in sorted(pairs): + KNOWS.append((a, b, r.randint(1, 9))) + + +_seed_graph() + +CYPHER = [ + "MATCH (p:Person) RETURN count(*) AS c", + "MATCH (p:Person) WHERE p.age > 40 RETURN p.id AS id", + "MATCH (p:Person) WHERE p.age IS NULL RETURN p.id AS id", + "MATCH (p:Person) RETURN p.name AS name, count(*) AS c, avg(p.age) AS a", + "MATCH (p:Person) WHERE p.name STARTS WITH 'd' OR p.name IN ['ann', 'ed'] RETURN p.id AS id", + "MATCH (p:Person) WHERE p.age IS NOT NULL RETURN p.id AS id ORDER BY p.age DESC, p.id LIMIT 10", + "MATCH (a:Person)-[k:KNOWS]->(b:Person) RETURN count(*) AS c", + "MATCH (a:Person)-[k:KNOWS]->(b:Person) WHERE k.w >= 7 RETURN a.id AS a, b.id AS b, k.w AS w", + "MATCH (a:Person)-[:KNOWS]->(b:Person) RETURN a.name AS name, sum(b.age) AS s, max(b.age) AS m", + "MATCH (a:Person)-[:KNOWS]->(b:Person) WITH a, count(b) AS d WHERE d >= 6 RETURN a.id AS a, d", + "MATCH (a:Person)-[:KNOWS]-(b:Person) WHERE a.id = 3 RETURN b.id AS b", + "MATCH (a:Person)-[:KNOWS]->(b:Person)-[:KNOWS]->(c:Person) RETURN count(*) AS c", + "MATCH (a:Person)-[:KNOWS]->(b:Person)-[:KNOWS]->(c:Person) WHERE a.id < c.id RETURN a.id AS a, count(DISTINCT c.id) AS n", + "MATCH (a:Person {id: 5})-[:KNOWS*1..3]->(b:Person) RETURN DISTINCT b.id AS b", + "MATCH (a:Person) OPTIONAL MATCH (a)-[:KNOWS]->(b:Person) WHERE b.age > 60 RETURN a.id AS a, count(b) AS n", +] + +# The same question with literals and with parameters (ArcadeDB against itself) +CYPHER_PARAMS = [ + ( + "MATCH (p:Person) WHERE p.age > 40 RETURN p.id AS id", + "MATCH (p:Person) WHERE p.age > $x RETURN p.id AS id", + {"x": 40}, + ), + ( + "MATCH (p:Person) WHERE p.name IN ['ann', 'ed'] RETURN p.id AS id", + "MATCH (p:Person) WHERE p.name IN $x RETURN p.id AS id", + {"x": ["ann", "ed"]}, + ), + ( + "MATCH (a:Person {id: 5})-[:KNOWS*1..3]->(b:Person) RETURN DISTINCT b.id AS b", + "MATCH (a:Person {id: $x})-[:KNOWS*1..3]->(b:Person) RETURN DISTINCT b.id AS b", + {"x": 5}, + ), + ( + "MATCH (a:Person)-[k:KNOWS]->(b:Person) WHERE k.w >= 7 RETURN a.id AS a, b.id AS b", + "MATCH (a:Person)-[k:KNOWS]->(b:Person) WHERE k.w >= $x RETURN a.id AS a, b.id AS b", + {"x": 7}, + ), +] + + +def _num(v): + if isinstance(v, bool) or not isinstance(v, (int, float)): + return v + return float(f"{float(v):.9g}") + + +def _canon(rows, ordered): + out = [tuple(sorted((k, _num(v)) for k, v in row.items())) for row in rows] + return out if ordered else sorted(out, key=repr) + + +def _arcade(db, language, query, *params): + return [dict(row) for row in db.query(language, query, *params).to_list()] + + +@pytest.fixture(scope="module") +def graph_db(tmp_path_factory): + db = arcadedb.create_database(str(tmp_path_factory.mktemp("eq") / "graph")) + for s in ( + "CREATE VERTEX TYPE Person", + "CREATE PROPERTY Person.id LONG", + "CREATE PROPERTY Person.age LONG", + "CREATE PROPERTY Person.name STRING", + "CREATE EDGE TYPE KNOWS", + "CREATE PROPERTY KNOWS.w LONG", + ): + db.command("sql", s) + with db.transaction(): + for p in PEOPLE: + db.command( + "opencypher", "CREATE (:Person {id: $id, age: $age, name: $name})", p + ) + for a, b, w in KNOWS: + db.command( + "opencypher", + "MATCH (a:Person {id: $a}), (b:Person {id: $b}) CREATE (a)-[:KNOWS {w: $w}]->(b)", + {"a": a, "b": b, "w": w}, + ) + yield db + db.close() + + +def test_cypher_unchanged_by_parameters(graph_db): + for literal, param, params in CYPHER_PARAMS: + assert _canon(_arcade(graph_db, "opencypher", param, params), False) == _canon( + _arcade(graph_db, "opencypher", literal), False + ), param + + +def test_cypher_unchanged_by_index_and_view(graph_db): + """Answers taken without an index or a view, then with indexes, then with a SYNCHRONOUS view as well.""" + before = { + q: _canon(_arcade(graph_db, "opencypher", q), "ORDER BY" in q) for q in CYPHER + } + for s in ( + "CREATE INDEX ON Person (age) NOTUNIQUE", + "CREATE INDEX ON Person (name) NOTUNIQUE", + "CREATE INDEX ON KNOWS (w) NOTUNIQUE", + ): + graph_db.command("sql", s) + for q in CYPHER: + assert ( + _canon(_arcade(graph_db, "opencypher", q), "ORDER BY" in q) == before[q] + ), f"with indexes: {q}" + graph_db.command( + "sql", + "CREATE GRAPH ANALYTICAL VIEW eq VERTEX TYPES (Person) EDGE TYPES (KNOWS) PROPERTIES (id, age, name) " + "EDGE PROPERTIES (w) UPDATE MODE SYNCHRONOUS", + ) + for _ in range(300): + if ( + graph_db.query( + "sql", + "SELECT status FROM schema:graphAnalyticalViews WHERE name = 'eq'", + ).to_list()[0]["status"] + == "READY" + ): + break + import time + + time.sleep(0.1) + try: + for q in CYPHER: + assert ( + _canon(_arcade(graph_db, "opencypher", q), "ORDER BY" in q) == before[q] + ), f"with the view: {q}" + finally: + graph_db.command("sql", "DROP GRAPH ANALYTICAL VIEW eq") + + +def _pagerank(g, damping): + """PageRank by power iteration in NumPy. nx.pagerank needs SciPy, which the test extra does not carry, and the graph + has no dangling node, so the textbook iteration is the whole algorithm.""" + nodes = sorted(g) + pos = {n: i for i, n in enumerate(nodes)} + out_deg = np.array([g.out_degree(n) for n in nodes], dtype=float) + src = np.array([pos[a] for a, _ in g.edges()]) + dst = np.array([pos[b] for _, b in g.edges()]) + x = np.full(len(nodes), 1.0 / len(nodes)) + for _ in range(1000): + nxt = np.full(len(nodes), (1 - damping) / len(nodes)) + np.add.at(nxt, dst, damping * x[src] / out_deg[src]) + if np.abs(nxt - x).sum() < 1e-15: + x = nxt + break + x = nxt + return {n: x[pos[n]] for n in nodes} + + +def test_graph_algorithms_match_networkx(graph_db): + g = nx.DiGraph() + g.add_nodes_from(p["id"] for p in PEOPLE) + g.add_edges_from((a, b) for a, b, _ in KNOWS) + + rows = _arcade( + graph_db, + "opencypher", + "CALL algo.pagerank({dampingFactor: 0.85, maxIterations: 200, tolerance: 0.0}) " + "YIELD node, score RETURN node.id AS id, score", + ) + total = sum(r["score"] for r in rows) + expected = _pagerank(g, 0.85) + for r in rows: + assert r["score"] / total == pytest.approx(expected[r["id"]], rel=1e-6), r["id"] + + rows = _arcade( + graph_db, + "opencypher", + "CALL algo.wcc() YIELD node, componentId RETURN node.id AS id, componentId AS c", + ) + components = {} + for r in rows: + components.setdefault(r["c"], set()).add(r["id"]) + assert sorted(map(sorted, components.values())) == sorted( + map(sorted, nx.weakly_connected_components(g)) + ) + + rows = _arcade( + graph_db, + "opencypher", + "CALL algo.degree() YIELD node, inDegree, outDegree RETURN node.id AS id, inDegree AS i, outDegree AS o", + ) + assert {r["id"]: (r["i"], r["o"]) for r in rows} == { + n: (g.in_degree(n), g.out_degree(n)) for n in g + } + + r = random.Random(3) # nosec B311 - seeded test data, not a secret + for _ in range(10): + base = r.choice((0, HALF)) + a, b = base + r.randrange(HALF), base + r.randrange(HALF) + if a == b: + continue + got = _arcade( + graph_db, + "opencypher", + "MATCH p = shortestPath((a:Person {id: $a})-[:KNOWS*]->(b:Person {id: $b})) RETURN length(p) AS l", + {"a": a, "b": b}, + ) + assert got[0]["l"] == nx.shortest_path_length(g, a, b), (a, b) + + +ITEMS = [] + + +def _seed_items(): + r = random.Random(11) # nosec B311 - seeded test data, not a secret + for i in range(300): + ITEMS.append( + { + "id": i, + "qty": None if r.random() < 0.15 else r.randint(0, 9), + "price": round(r.uniform(1, 99), 2), + "cat": r.choice(["a", "b", "c", "bd"]), + "day": datetime.date(2026, 1, 1) + + datetime.timedelta(days=r.randrange(180)), + } + ) + + +_seed_items() + +# (query with {T} for the table, parameters); the same text runs on both engines +SQL = [ + ("SELECT count(*) AS c FROM {T}", ()), + ("SELECT id FROM {T} WHERE qty > 5", ()), + ("SELECT id FROM {T} WHERE qty IS NULL", ()), + ("SELECT cat, count(*) AS c, sum(qty) AS s FROM {T} GROUP BY cat", ()), + ( + "SELECT cat, avg(price) AS a, min(price) AS lo, max(price) AS hi FROM {T} GROUP BY cat", + (), + ), + ("SELECT DISTINCT cat FROM {T}", ()), + ("SELECT id FROM {T} WHERE cat IN ('a', 'c') AND price BETWEEN 10 AND 50", ()), + ("SELECT id FROM {T} WHERE cat LIKE 'b%'", ()), + ("SELECT id, price FROM {T} WHERE price > 40 ORDER BY price DESC, id LIMIT 7", ()), + ("SELECT id, day FROM {T} WHERE id < 12", ()), + ("SELECT id FROM {T} WHERE day >= ?", (datetime.date(2026, 4, 1),)), + ( + "SELECT count(*) AS c FROM {T} WHERE day BETWEEN ? AND ?", + (datetime.date(2026, 2, 1), datetime.date(2026, 3, 15)), + ), + ("SELECT id FROM {T} WHERE qty = ? AND cat = ?", (3, "b")), +] + + +@pytest.fixture(scope="module") +def item_db(tmp_path_factory): + db = arcadedb.create_database(str(tmp_path_factory.mktemp("eq") / "items")) + for s in ( + "CREATE DOCUMENT TYPE Item", + "CREATE PROPERTY Item.id INTEGER", + "CREATE PROPERTY Item.qty INTEGER", + "CREATE PROPERTY Item.price DOUBLE", + "CREATE PROPERTY Item.cat STRING", + "CREATE PROPERTY Item.day DATE", + ): + db.command("sql", s) + with db.transaction(): + for it in ITEMS: + db.command( + "sql", + "INSERT INTO Item SET id = ?, qty = ?, price = ?, cat = ?, day = ?", + it["id"], + it["qty"], + it["price"], + it["cat"], + it["day"], + ) + yield db + db.close() + + +@pytest.fixture(scope="module") +def duck(): + con = duckdb.connect() + con.execute( + "CREATE TABLE item (id INTEGER, qty INTEGER, price DOUBLE, cat VARCHAR, day DATE)" + ) + con.executemany( + "INSERT INTO item VALUES (?, ?, ?, ?, ?)", + [(i["id"], i["qty"], i["price"], i["cat"], i["day"]) for i in ITEMS], + ) + return con + + +@pytest.mark.parametrize("query,params", SQL) +def test_sql_matches_duckdb(item_db, duck, query, params): + cur = duck.execute(query.replace("{T}", "item"), list(params)) + cols = [d[0] for d in cur.description] + expected = [dict(zip(cols, row)) for row in cur.fetchall()] + ordered = "ORDER BY" in query + assert _canon( + _arcade(item_db, "sql", query.replace("{T}", "Item"), *params), ordered + ) == _canon(expected, ordered) + + +def test_vector_search_recall_against_brute_force(temp_db): + rng = np.random.default_rng(5) + vectors = rng.standard_normal((1000, 16)).astype(np.float32) + temp_db.command("sql", "CREATE VERTEX TYPE Doc") + temp_db.command("sql", "CREATE PROPERTY Doc.id INTEGER") + temp_db.command("sql", "CREATE PROPERTY Doc.vec ARRAY_OF_FLOATS") + with temp_db.transaction(): + for i, v in enumerate(vectors): + temp_db.command("sql", "INSERT INTO Doc SET id = ?, vec = ?", i, v.tolist()) + index = temp_db.create_vector_index( + "Doc", "vec", dimensions=16, distance_function="cosine", quantization="NONE" + ) + unit = vectors / np.linalg.norm(vectors, axis=1, keepdims=True) + hits = 0 + queries = rng.standard_normal((20, 16)).astype(np.float32) + for q in queries: + exact = set(np.argsort(-(unit @ (q / np.linalg.norm(q))))[:10].tolist()) + found = {record.get("id") for record, _ in index.find_nearest(q.tolist(), k=10)} + hits += len(found & exact) + assert hits / (10 * len(queries)) >= 0.9 diff --git a/tests/test_equivalence_ladybug.py b/tests/test_equivalence_ladybug.py new file mode 100644 index 0000000..89243d0 --- /dev/null +++ b/tests/test_equivalence_ladybug.py @@ -0,0 +1,54 @@ +"""Equivalence checks (humemai/arcadedb-embedded-python#347), the Cypher half: ArcadeDB answers like LadybugDB over test_equivalence.py's graph and queries. + +A file of its own because LadybugDB's compiled module does not load in the Windows test process: the import of +`_lbug` fails and LadybugDB falls back to a C-API library its wheel does not ship. The likeliest cause is a DLL the +already-running bundled JRE loaded first, which could not be confirmed without a Windows machine, so conftest.py does +not collect this file on Windows and the comparison runs on Linux and macOS. +""" + +import ladybug +import pytest + +from tests.test_equivalence import ( # noqa: F401 (graph_db: fixture) + CYPHER, + KNOWS, + PEOPLE, + _arcade, + _canon, + graph_db, +) + + +@pytest.fixture(scope="module") +def ladybug_conn(tmp_path_factory): + conn = ladybug.Connection( + ladybug.Database(str(tmp_path_factory.mktemp("eq") / "ladybug")) + ) + conn.execute( + "CREATE NODE TABLE Person(id INT64, age INT64, name STRING, PRIMARY KEY(id))" + ) + conn.execute("CREATE REL TABLE KNOWS(FROM Person TO Person, w INT64)") + for p in PEOPLE: + conn.execute("CREATE (:Person {id: $id, age: $age, name: $name})", p) + for a, b, w in KNOWS: + conn.execute( + "MATCH (a:Person {id: $a}), (b:Person {id: $b}) CREATE (a)-[:KNOWS {w: $w}]->(b)", + {"a": a, "b": b, "w": w}, + ) + return conn + + +def _ladybug(conn, query): + res = conn.execute(query) + cols, rows = res.get_column_names(), [] + while res.has_next(): + rows.append(dict(zip(cols, res.get_next()))) + return rows + + +@pytest.mark.parametrize("query", CYPHER) +def test_cypher_matches_ladybug(graph_db, ladybug_conn, query): + ordered = "ORDER BY" in query + assert _canon(_arcade(graph_db, "opencypher", query), ordered) == _canon( + _ladybug(ladybug_conn, query), ordered + ) diff --git a/tests/test_guide_graph_analytics.py b/tests/test_guide_graph_analytics.py new file mode 100644 index 0000000..9763023 --- /dev/null +++ b/tests/test_guide_graph_analytics.py @@ -0,0 +1,379 @@ +"""Every example in the graph-analytics guide, run and asserted (humemai/arcadedb-embedded-python#388). + +The statements below were measured on the 26.10.2 engine. In the repository this was ported from, each +statement a test sends had to appear verbatim in the guide page (docs/guide/graph-analytics.md), so an edit to the +page that this file did not follow failed there instead of drifting. The guide pages are not in this +repository yet (tracker #19, docs): until the Antora page exists `doc()` only returns the statement, +and it checks the page again once the page does. +""" + +import time + +import pytest + +import arcadedb + +NAMES = ["ada", "bob", "cy", "dee", "eve", "fay"] +EDGES = [ + ("ada", "bob", 1.0), + ("bob", "cy", 1.0), + ("ada", "cy", 5.0), + ("cy", "dee", 1.0), + ("dee", "ada", 1.0), + ("eve", "fay", 1.0), +] +CREATE_VIEW = ( + "CREATE GRAPH ANALYTICAL VIEW social VERTEX TYPES (Person) EDGE TYPES (Follows) " + "PROPERTIES (name) EDGE PROPERTIES (weight) UPDATE MODE SYNCHRONOUS" +) +VIEW_STATE = ( + "SELECT status, nodeCount, edgeCount, updateMode " + "FROM schema:graphAnalyticalViews WHERE name = 'social'" +) + + +def doc(statement): + """Return `statement`; the check against the guide page returns with the Antora page (see the module docstring).""" + return statement + + +def view_state(db): + return db.query("sql", doc(VIEW_STATE)).first().to_dict() + + +def wait_for(db, **expected): + """Poll the view row until it shows `expected` (a build runs in the background).""" + deadline = time.time() + 30 + state = view_state(db) + while time.time() < deadline: + state = view_state(db) + if all(state.get(k) == v for k, v in expected.items()): + return state + time.sleep(0.05) + raise AssertionError(f"view never reached {expected}: {state}") + + +def cypher_raw(db, statement): + return [row.to_dict() for row in db.query("opencypher", statement)] + + +def cypher(db, statement): + return [row.to_dict() for row in db.query("opencypher", doc(statement))] + + +@pytest.fixture +def db(temp_db_path): + with arcadedb.create_database(temp_db_path) as database: + database.command("sql", doc("CREATE VERTEX TYPE Person")) + database.command("sql", doc("CREATE EDGE TYPE Follows")) + database.command("sql", doc("CREATE PROPERTY Follows.weight DOUBLE")) + with database.transaction(): + for name in NAMES: + database.command("sql", doc("CREATE VERTEX Person SET name = ?"), name) + for src, dst, weight in EDGES: + database.command( + "sql", + doc( + "CREATE EDGE Follows FROM (SELECT FROM Person WHERE name = ?) " + "TO (SELECT FROM Person WHERE name = ?) SET weight = ?" + ), + src, + dst, + weight, + ) + yield database + + +@pytest.fixture +def view_db(db): + row = db.command("sql", doc(CREATE_VIEW)).first() + assert row.to_dict() == { + "operation": "create graph analytical view", + "name": "social", + "created": True, + } + wait_for(db, status="READY") + return db + + +def test_create_view_and_read_its_state(db): + db.command("sql", doc(CREATE_VIEW)) + state = wait_for(db, status="READY") + assert state == { + "status": "READY", + "nodeCount": 6, + "edgeCount": 6, + "updateMode": "SYNCHRONOUS", + } + memory = db.query( + "sql", "SELECT memoryUsageBytes FROM schema:graphAnalyticalViews" + ).first() + assert memory.get("memoryUsageBytes") > 0 + + +def test_pagerank(view_db): + query = ( + "CALL algo.pagerank() YIELD node, score " + "RETURN node.name AS name, round(score * 1000) / 1000.0 AS score " + "ORDER BY score DESC, name" + ) + rows = cypher(view_db, query) + assert [(r["name"], r["score"]) for r in rows] == [ + ("cy", 0.259), + ("dee", 0.254), + ("ada", 0.25), + ("bob", 0.14), + ("fay", 0.063), + ("eve", 0.034), + ] + # The options go in one map; a tighter run changes the digits, not the order. + tuned = cypher( + view_db, + "CALL algo.pagerank({dampingFactor: 0.9, maxIterations: 50}) " + "YIELD node, score RETURN node.name AS name ORDER BY score DESC, name LIMIT 1", + ) + assert tuned == [{"name": "cy"}] + + +def test_same_answers_with_and_without_the_view(db): + query = ( + "CALL algo.pagerank() YIELD node, score " + "RETURN node.name AS name, round(score * 1000) / 1000.0 AS score " + "ORDER BY score DESC, name" + ) + before = cypher(db, query) + db.command("sql", doc(CREATE_VIEW)) + wait_for(db, status="READY") + with_view = cypher(db, query) + assert db.command( + "sql", doc("DROP GRAPH ANALYTICAL VIEW social") + ).first().to_dict() == { + "operation": "drop graph analytical view", + "name": "social", + "dropped": True, + } + assert db.query("sql", "SELECT FROM schema:graphAnalyticalViews").to_list() == [] + after = cypher(db, query) + assert before == with_view == after + + +def test_connected_components(view_db): + weak = cypher( + view_db, + "CALL algo.wcc() YIELD node, componentId " + "RETURN componentId, collect(node.name) AS members", + ) + assert sorted(sorted(r["members"]) for r in weak) == [ + ["ada", "bob", "cy", "dee"], + ["eve", "fay"], + ] + strong = cypher( + view_db, + "CALL algo.scc() YIELD node, componentId " + "RETURN componentId, collect(node.name) AS members", + ) + assert sorted(sorted(r["members"]) for r in strong) == [ + ["ada", "bob", "cy", "dee"], + ["eve"], + ["fay"], + ] + + +def test_bfs(view_db): + rows = cypher( + view_db, + "MATCH (a:Person {name: 'ada'}) CALL algo.bfs(a, 'Follows', 'OUT', 3) " + "YIELD node, depth RETURN node.name AS name, depth ORDER BY depth, name", + ) + assert rows == [ + {"name": "bob", "depth": 1}, + {"name": "cy", "depth": 1}, + {"name": "dee", "depth": 2}, + ] + + +def test_dijkstra(view_db): + rows = cypher( + view_db, + "MATCH (a:Person {name: 'ada'}), (d:Person {name: 'dee'}) " + "CALL algo.dijkstra(a, d, 'Follows', 'weight', 'OUT') YIELD path, weight " + "RETURN [n IN nodes(path) | n.name] AS names, weight", + ) + # ada -> cy costs 5 directly and 2 through bob, so the cheap path wins. + assert rows == [{"names": ["ada", "bob", "cy", "dee"], "weight": 3.0}] + + +def test_degree_and_triangles(view_db): + degrees = cypher( + view_db, + "CALL algo.degree() YIELD node, inDegree, outDegree " + "RETURN node.name AS name, inDegree, outDegree ORDER BY name", + ) + assert [(r["name"], r["inDegree"], r["outDegree"]) for r in degrees] == [ + ("ada", 1, 2), + ("bob", 1, 1), + ("cy", 2, 1), + ("dee", 1, 1), + ("eve", 0, 1), + ("fay", 1, 0), + ] + triangles = cypher( + view_db, + "CALL algo.triangleCount() YIELD node, triangles " + "RETURN node.name AS name, triangles ORDER BY name", + ) + assert [(r["name"], r["triangles"]) for r in triangles] == [ + ("ada", 2), + ("bob", 1), + ("cy", 2), + ("dee", 1), + ("eve", 0), + ("fay", 0), + ] + + +def test_louvain_and_betweenness(view_db): + communities = cypher( + view_db, + "CALL algo.louvain() YIELD node, communityId " + "RETURN communityId, collect(node.name) AS members", + ) + groups = [set(r["members"]) for r in communities] + # eve and fay form their own community: no edge joins them to the other four. + assert {"eve", "fay"} in groups + assert all(not ({"eve", "fay"} & g) for g in groups if g != {"eve", "fay"}) + scores = cypher( + view_db, + "CALL algo.betweenness() YIELD node, score " + "RETURN node.name AS name, score ORDER BY score DESC, name", + ) + by_name = {r["name"]: r["score"] for r in scores} + assert by_name["eve"] == by_name["fay"] == 0.0 + assert min(by_name[n] for n in ["ada", "bob", "cy", "dee"]) >= 0.0 + assert max(by_name.values()) == pytest.approx(0.15) + + +def test_update_mode_synchronous_then_alter_to_off_and_rebuild(view_db): + with view_db.transaction(): + view_db.command("sql", "CREATE VERTEX Person SET name = 'gus'") + # SYNCHRONOUS: the commit that returned is already in the view. + assert view_state(view_db) == { + "status": "READY", + "nodeCount": 7, + "edgeCount": 6, + "updateMode": "SYNCHRONOUS", + } + + altered = view_db.command( + "sql", doc("ALTER GRAPH ANALYTICAL VIEW social UPDATE MODE OFF") + ).first() + assert altered.to_dict() == { + "operation": "alter graph analytical view", + "name": "social", + "updateMode": "OFF", + } + with view_db.transaction(): + view_db.command("sql", "CREATE VERTEX Person SET name = 'hal'") + # OFF: the view is marked STALE and keeps its old counts ... + stale = view_state(view_db) + assert stale["status"] == "STALE" + assert stale["nodeCount"] == 7 + # ... yet a whole-graph call still answers from the live graph (it scans). + count = "CALL algo.wcc() YIELD node RETURN count(node) AS n" + assert cypher(view_db, count) == [{"n": 8}] + + rebuilt = view_db.command( + "sql", doc("REBUILD GRAPH ANALYTICAL VIEW social") + ).first() + assert rebuilt.get("status") == "READY" + state = wait_for(view_db, status="READY") + assert state["nodeCount"] == 8 + assert cypher(view_db, count) == [{"n": 8}] + + +def test_update_mode_asynchronous_catches_up(db): + db.command( + "sql", + doc( + "CREATE GRAPH ANALYTICAL VIEW social VERTEX TYPES (Person) " + "EDGE TYPES (Follows) UPDATE MODE ASYNCHRONOUS COMPACTION THRESHOLD 500" + ), + ) + wait_for(db, status="READY", nodeCount=6) + with db.transaction(): + db.command("sql", "CREATE VERTEX Person SET name = 'gus'") + db.command( + "sql", + "CREATE EDGE Follows FROM (SELECT FROM Person WHERE name = 'gus') " + "TO (SELECT FROM Person WHERE name = 'ada')", + ) + # ASYNCHRONOUS: the view folds the change in after the commit returned. + state = wait_for(db, status="READY", nodeCount=7, edgeCount=7) + assert state["updateMode"] == "ASYNCHRONOUS" + row = db.query( + "sql", + "SELECT updateMode, compactionThreshold FROM schema:graphAnalyticalViews " + "WHERE name = 'social'", + ).first() + assert row.to_dict() == {"updateMode": "ASYNCHRONOUS", "compactionThreshold": 500} + + +def test_count_answers_the_same_and_plan_does_not_show_the_view(view_db): + count = "MATCH (a:Person)-[:Follows]->(b:Person) RETURN count(*) AS n" + explain = "EXPLAIN MATCH (a:Person)-[:Follows]->(b:Person) RETURN count(*) AS n" + plan_with = ( + view_db.query("opencypher", explain).first().get("executionPlanAsString") + ) + assert cypher_raw(view_db, count) == [{"n": 6}] + view_db.command("sql", "DROP GRAPH ANALYTICAL VIEW social") + plan_without = ( + view_db.query("opencypher", explain).first().get("executionPlanAsString") + ) + assert cypher_raw(view_db, count) == [{"n": 6}] + # The guide says the plan text is the same either way. + doc("COUNT CHAIN PATHS (CSR count-push-down)") + assert "COUNT CHAIN PATHS (CSR count-push-down)" in plan_with + assert "COUNT CHAIN PATHS (CSR count-push-down)" in plan_without + + +def test_partial_view_does_not_serve_a_whole_graph_call(db): + db.command("sql", "CREATE VERTEX TYPE City") + with db.transaction(): + db.command("sql", "CREATE VERTEX City SET name = 'X'") + db.command( + "sql", + doc( + "CREATE GRAPH ANALYTICAL VIEW people VERTEX TYPES (Person) " + "EDGE TYPES (Follows)" + ), + ) + deadline = time.time() + 30 + while time.time() < deadline: + row = db.query( + "sql", + "SELECT status FROM schema:graphAnalyticalViews WHERE name = 'people'", + ).first() + if row.get("status") == "READY": + break + time.sleep(0.05) + # The view leaves City out, so the call scans and still sees all 7 vertices. + assert cypher(db, "CALL algo.degree() YIELD node RETURN count(node) AS n") == [ + {"n": 7} + ] + + +def test_ddl_errors_and_if_clauses(db): + db.command("sql", doc(CREATE_VIEW)) + with pytest.raises(arcadedb.ArcadeDBError, match="already exists"): + db.command("sql", "CREATE GRAPH ANALYTICAL VIEW social") + again = db.command( + "sql", doc("CREATE GRAPH ANALYTICAL VIEW IF NOT EXISTS social") + ).first() + assert again.get("created") is False + with pytest.raises(arcadedb.ArcadeDBError, match="Unknown update mode"): + db.command("sql", "ALTER GRAPH ANALYTICAL VIEW social UPDATE MODE SOMETIMES") + with pytest.raises(arcadedb.ArcadeDBError, match="does not exist"): + db.command("sql", "DROP GRAPH ANALYTICAL VIEW nope") + db.command("sql", doc("DROP GRAPH ANALYTICAL VIEW social")) + gone = db.command("sql", doc("DROP GRAPH ANALYTICAL VIEW IF EXISTS social")).first() + assert gone.get("name") == "social" diff --git a/tests/test_guide_timeseries.py b/tests/test_guide_timeseries.py new file mode 100644 index 0000000..4f6f290 --- /dev/null +++ b/tests/test_guide_timeseries.py @@ -0,0 +1,291 @@ +"""Every example in the timeseries guide, run and asserted (humemai/arcadedb-embedded-python#388). + +The statements below were measured on the 26.10.2 engine. In the repository this was ported from, each +statement a test sends had to appear verbatim in the guide page (docs/guide/timeseries.md), so an edit to the +page that this file did not follow failed there instead of drifting. The guide pages are not in this +repository yet (tracker #19, docs): until the Antora page exists `doc()` only returns the statement, +and it checks the page again once the page does. +""" + +from datetime import datetime + +import numpy as np +import pyarrow as pa +import pytest + +import arcadedb + +T0 = 1_767_225_600_000 # 2026-01-01T00:00:00Z in epoch milliseconds +MINUTE = 60_000 + +CREATE_TYPE = ( + "CREATE TIMESERIES TYPE Reading TIMESTAMP ts PRECISION MILLISECOND " + "TAGS (sensor STRING) FIELDS (temp DOUBLE, hum DOUBLE) SHARDS 2 " + "RETENTION 90 DAYS COMPACTION_INTERVAL 1 HOURS " + "DOWNSAMPLING POLICY AFTER 7 DAYS GRANULARITY 1 HOURS " + "AFTER 30 DAYS GRANULARITY 1 DAYS" +) + + +def doc(statement): + """Return `statement`; the check against the guide page returns with the Antora page (see the module docstring).""" + return statement + + +def rows(db, statement, *params): + return [r.to_dict() for r in db.query("sql", doc(statement), *params)] + + +def utc(minute): + """The naive datetime a TIMESERIES timestamp comes back as (UTC).""" + return datetime(2026, 1, 1, 0, minute) + + +@pytest.fixture +def db(temp_db_path): + with arcadedb.create_database(temp_db_path) as database: + database.command("sql", doc(CREATE_TYPE)) + yield database + + +@pytest.fixture +def loaded(db): + """Reading with six samples: sensor a at minutes 0, 1, 3; sensor b at 1, 2, 3.""" + db.command( + "sql", + doc("INSERT INTO Reading SET ts = ?, sensor = 'a', temp = 20.0, hum = 40.0"), + T0, + ) + db.command( + "sql", + doc( + "INSERT INTO Reading (ts, sensor, temp, hum) VALUES (?, 'a', 21.0, 41.0), (?, 'b', 30.0, 50.0)" + ), + T0 + MINUTE, + T0 + MINUTE, + ) + db.command( + "sql", + doc( + 'INSERT INTO Reading CONTENT {"ts": 1767225780000, "sensor": "a", "temp": 22.0, "hum": 42.0}' + ), + ) + for minute, temp in [(2, 31.0), (3, 32.0)]: + db.command( + "sql", + "INSERT INTO Reading SET ts = ?, sensor = 'b', temp = ?, hum = 51.0", + T0 + minute * MINUTE, + temp, + ) + return db + + +def test_type_is_declared_as_written(db): + row = db.query( + "sql", doc("SELECT FROM schema:types WHERE name = 'Reading'") + ).first() + assert row.get("timestampColumn") == "ts" + assert row.get("precision") == "MILLISECOND" + assert row.get("shardCount") == 2 + assert row.get("retentionMs") == 90 * 24 * 3600 * 1000 + assert row.get("compactionBucketIntervalMs") == 3600 * 1000 + assert len(row.get("downsamplingTiers")) == 2 + + +def test_create_if_not_exists(db): + again = db.command( + "sql", + doc( + "CREATE TIMESERIES TYPE Reading IF NOT EXISTS TIMESTAMP ts TAGS (sensor STRING) FIELDS (temp DOUBLE)" + ), + ).first() + assert again.to_dict() == { + "operation": "create timeseries type", + "typeName": "Reading", + "created": False, + } + + +def test_insert_forms_and_read_back(loaded): + assert rows(loaded, "SELECT count(*) AS n FROM Reading") == [{"n": 6}] + got = rows(loaded, "SELECT FROM Reading WHERE sensor = 'a' ORDER BY ts") + assert [(r["ts"], r["temp"]) for r in got] == [ + (utc(0), 20.0), + (utc(1), 21.0), + (utc(3), 22.0), + ] + + +def test_a_text_timestamp_is_refused_on_insert(db): + with pytest.raises(arcadedb.ArcadeDBError, match="Cannot convert type"): + db.command( + "sql", + doc( + "INSERT INTO Reading SET ts = '2026-01-01T00:02:00Z', sensor = 'b', temp = 31.0, hum = 51.0" + ), + ) + + +def test_range_queries_take_numbers_or_text(loaded): + by_number = rows( + loaded, + "SELECT FROM Reading WHERE ts BETWEEN ? AND ? AND sensor = 'a' ORDER BY ts", + T0, + T0 + 2 * MINUTE, + ) + assert [r["ts"] for r in by_number] == [utc(0), utc(1)] + by_text = rows( + loaded, + "SELECT FROM Reading WHERE ts >= '2026-01-01T00:01:00Z' AND ts < '2026-01-01T00:03:00Z' ORDER BY ts, sensor", + ) + assert [(r["ts"], r["sensor"]) for r in by_text] == [ + (utc(1), "a"), + (utc(1), "b"), + (utc(2), "b"), + ] + + +def test_time_bucket_group_by(loaded): + got = rows( + loaded, + "SELECT ts.timeBucket('2m', ts) AS slot, sensor, avg(temp) AS t, max(temp) AS peak, count(*) AS n FROM Reading GROUP BY slot, sensor ORDER BY slot, sensor", + ) + assert [(r["slot"], r["sensor"], r["t"], r["peak"], r["n"]) for r in got] == [ + (utc(0), "a", 20.5, 21.0, 2), + (utc(0), "b", 30.0, 30.0, 1), + (utc(2), "a", 22.0, 22.0, 1), + (utc(2), "b", 31.5, 32.0, 2), + ] + + +def test_time_bucket_with_a_timezone(loaded): + got = rows( + loaded, + "SELECT ts.timeBucket('1d', ts, {timezone: '+08:00'}) AS day, count(*) AS n FROM Reading GROUP BY day", + ) + # local midnight in UTC+8 is 16:00 UTC the day before + assert got == [{"day": datetime(2025, 12, 31, 16, 0), "n": 6}] + + +def test_first_last_delta_rate_percentile(loaded): + got = rows( + loaded, + "SELECT sensor, ts.first(temp, ts) AS first, ts.last(temp, ts) AS last, ts.delta(temp, ts) AS delta, ts.rate(temp, ts) AS per_second FROM Reading GROUP BY sensor ORDER BY sensor", + ) + a, b = got + assert (a["sensor"], a["first"], a["last"], a["delta"]) == ("a", 20.0, 22.0, 2.0) + assert a["per_second"] == pytest.approx(2.0 / 180) + assert (b["sensor"], b["first"], b["last"], b["delta"]) == ("b", 30.0, 32.0, 2.0) + assert b["per_second"] == pytest.approx(2.0 / 120) + median = rows( + loaded, + "SELECT sensor, ts.percentile(temp, 0.5) AS median FROM Reading GROUP BY sensor ORDER BY sensor", + ) + assert median == [ + {"sensor": "a", "median": 21.0}, + {"sensor": "b", "median": 31.0}, + ] + + +def test_window_functions_return_one_array(loaded): + got = rows( + loaded, + "SELECT ts.movingAvg(temp, 2) AS ma FROM Reading WHERE sensor = 'a' ORDER BY ts", + ) + assert len(got) == 1 + assert list(got[0]["ma"]) == [20.0, 20.5, 21.5] + + +def test_append_samples_then_compact(db): + n = 1_000 + executor = db.async_executor() + executor.append_samples( + "Reading", + T0 + np.arange(n, dtype=np.int64) * 1_000, + [f"s{i % 2}" for i in range(n)], + np.arange(n, dtype=np.float64), + np.full(n, 50.0), + ) + executor.wait_completion() + assert rows(db, "SELECT count(*) AS n FROM Reading") == [{"n": n}] + sealed = db.command("sql", doc("COMPACT TIMESERIES TYPE Reading")).first() + assert sealed.get("typeName") == "Reading" + assert sealed.get("mutableSamples") == 0 + per_tag = rows( + db, "SELECT sensor, count(*) AS n FROM Reading GROUP BY sensor ORDER BY sensor" + ) + assert per_tag == [{"sensor": "s0", "n": 500}, {"sensor": "s1", "n": 500}] + + +def test_result_to_arrow(loaded): + result = loaded.query( + "sql", + doc( + "SELECT ts.timeBucket('2m', ts) AS slot, avg(temp) AS t FROM Reading GROUP BY slot ORDER BY slot" + ), + ) + table = result.to_arrow() + assert table.schema.field("slot").type == pa.timestamp("ms") + assert table.num_rows == 2 + assert table.column("t").to_pylist() == [ + pytest.approx(71 / 3), + pytest.approx(85 / 3), + ] + + +def test_alter_downsampling_policy(db): + dropped = db.command( + "sql", doc("ALTER TIMESERIES TYPE Reading DROP DOWNSAMPLING POLICY") + ).first() + assert dropped.get("operation") == "drop downsampling policy" + tiers = db.query("sql", "SELECT FROM schema:types WHERE name = 'Reading'").first() + assert not tiers.get("downsamplingTiers") + added = db.command( + "sql", + doc( + "ALTER TIMESERIES TYPE Reading ADD DOWNSAMPLING POLICY AFTER 7 DAYS GRANULARITY 1 HOURS" + ), + ).first() + assert added.get("operation") == "add downsampling policy" + tiers = db.query("sql", "SELECT FROM schema:types WHERE name = 'Reading'").first() + assert len(tiers.get("downsamplingTiers")) == 1 + + +def test_continuous_aggregate(loaded): + created = loaded.command( + "sql", + doc( + "CREATE CONTINUOUS AGGREGATE hourly AS SELECT ts.timeBucket('1h', ts) AS hour, sensor, avg(temp) AS t FROM Reading GROUP BY hour, sensor" + ), + ).first() + assert created.get("created") is True + got = rows(loaded, "SELECT FROM hourly ORDER BY hour, sensor") + assert [(r["hour"], r["sensor"], r["t"]) for r in got] == [ + (utc(0), "a", 21.0), + (utc(0), "b", 31.0), + ] + # A new sample reaches the aggregate after a refresh. + loaded.command( + "sql", + "INSERT INTO Reading SET ts = ?, sensor = 'a', temp = 24.0, hum = 40.0", + T0 + 4 * MINUTE, + ) + loaded.command("sql", doc("REFRESH CONTINUOUS AGGREGATE hourly")) + got = rows(loaded, "SELECT FROM hourly ORDER BY hour, sensor") + assert got[0]["t"] == pytest.approx(21.75) + + # The type cannot be dropped while the aggregate reads it. + with pytest.raises(arcadedb.ArcadeDBError, match="continuous aggregate"): + loaded.command("sql", doc("DROP TIMESERIES TYPE Reading")) + loaded.command("sql", doc("DROP CONTINUOUS AGGREGATE hourly")) + loaded.command("sql", "DROP TIMESERIES TYPE Reading") + + +def test_bucket_is_a_reserved_alias(loaded): + with pytest.raises(arcadedb.ArcadeDBError): + loaded.query( + "sql", + doc( + "SELECT ts.timeBucket('2m', ts) AS bucket, count(*) AS n FROM Reading GROUP BY bucket" + ), + ).to_list() diff --git a/tests/test_sql_constraints_and_maintenance.py b/tests/test_sql_constraints_and_maintenance.py new file mode 100644 index 0000000..5ca5a8a --- /dev/null +++ b/tests/test_sql_constraints_and_maintenance.py @@ -0,0 +1,175 @@ +"""The SQL forms the docs give for what humemai/arcadedb-embedded-python#383 asked the bindings to wrap. + +The bindings are DSL-first (see the access methods in the user docs): property +constraints, type scans, the transaction isolation level and the admin +statements are each one SQL statement, so the docs show the statement and this +file runs it. A statement the documentation names and no test runs is a claim +nobody checked. + +`COMPACT INDEX` runs in `test_sparse_quantization_compact.py`. +""" + +import threading +import zipfile + +import jpype +import pytest + +import arcadedb +from arcadedb import ArcadeDBError + + +def _insert(db, statement): + with db.transaction(): + db.command("sql", statement) + + +def test_create_property_constraints_refuse_bad_values(temp_db): + db = temp_db + db.command("sql", "CREATE DOCUMENT TYPE Item") + db.command( + "sql", + "CREATE PROPERTY Item.qty INTEGER (mandatory true, notnull true, min 0, max 100)", + ) + db.command( + "sql", "CREATE PROPERTY Item.code STRING (regexp '[A-Z]+', default 'AB')" + ) + db.command("sql", "CREATE PROPERTY Item.tag STRING (readonly true)") + + # min, max, mandatory, notnull and regexp each refuse a record + for bad in ( + "INSERT INTO Item SET qty = 101", + "INSERT INTO Item SET qty = -1", + "INSERT INTO Item SET code = 'ABC'", + "INSERT INTO Item SET qty = null", + "INSERT INTO Item SET qty = 1, code = 'abc'", + ): + with pytest.raises(ArcadeDBError, match="ValidationException"): + _insert(db, bad) + assert db.count_type("Item") == 0 + + # a valid record is accepted and takes the default + _insert(db, "INSERT INTO Item SET qty = 5, tag = 'first'") + row = db.query("sql", "SELECT qty, code, tag FROM Item").first().to_dict() + assert row == {"qty": 5, "code": "AB", "tag": "first"} + + # readonly: the value cannot change once the record exists + with pytest.raises(ArcadeDBError, match="ValidationException"): + _insert(db, "UPDATE Item SET tag = 'second'") + assert db.query("sql", "SELECT tag FROM Item").first().get("tag") == "first" + + +def test_alter_property_changes_a_constraint_on_an_existing_property(temp_db): + db = temp_db + db.command("sql", "CREATE DOCUMENT TYPE Item") + db.command("sql", "CREATE PROPERTY Item.qty INTEGER") + _insert(db, "INSERT INTO Item SET qty = 500") # no max yet + + # a constraint the stored records already break is refused, not applied + with pytest.raises(ArcadeDBError, match="violates it"): + db.command("sql", "ALTER PROPERTY Item.qty MAX 100") + _insert(db, "UPDATE Item SET qty = 50") + + db.command("sql", "ALTER PROPERTY Item.qty MAX 100") + with pytest.raises(ArcadeDBError, match="ValidationException"): + _insert(db, "INSERT INTO Item SET qty = 101") + _insert(db, "INSERT INTO Item SET qty = 100") + assert db.count_type("Item") == 2 + + +def test_select_from_type_scans_subtypes(temp_db): + db = temp_db + db.command("sql", "CREATE DOCUMENT TYPE Animal") + db.command("sql", "CREATE DOCUMENT TYPE Dog EXTENDS Animal") + db.command("sql", "CREATE DOCUMENT TYPE Puppy EXTENDS Dog") + with db.transaction(): + db.command("sql", "INSERT INTO Animal SET name = 'a'") + db.command("sql", "INSERT INTO Dog SET name = 'd'") + db.command("sql", "INSERT INTO Puppy SET name = 'p'") + + names = {r["name"] for r in db.query("sql", "SELECT FROM Animal").to_list()} + assert names == {"a", "d", "p"} + names = {r["name"] for r in db.query("sql", "SELECT FROM Dog").to_list()} + assert names == {"d", "p"} + + +def _versions_seen_by(db, level): + """Read the counter, let another thread bump it, read again, in one transaction.""" + db.command("sql", f"BEGIN ISOLATION {level}") + try: + first = db.query("sql", "SELECT v FROM Counter").first().get("v") + + def bump(): + with db.transaction(): + db.command("sql", "UPDATE Counter SET v = v + 1") + + worker = threading.Thread(target=bump) + worker.start() + worker.join() + second = db.query("sql", "SELECT v FROM Counter").first().get("v") + finally: + db.commit() + return first, second + + +def test_begin_isolation_read_committed_vs_repeatable_read(temp_db): + db = temp_db + db.command("sql", "CREATE DOCUMENT TYPE Counter") + _insert(db, "INSERT INTO Counter SET v = 1") + + first, second = _versions_seen_by(db, "READ_COMMITTED") + assert (first, second) == (1, 2) # the other thread's commit is visible + + first, second = _versions_seen_by(db, "REPEATABLE_READ") + assert first == second == 2 # the committed 3 is not + assert db.query("sql", "SELECT v FROM Counter").first().get("v") == 3 + + +def test_backup_check_and_rebuild_index_statements(temp_db, tmp_path): + db = temp_db + db.command("sql", "CREATE DOCUMENT TYPE Person") + db.command("sql", "CREATE PROPERTY Person.email STRING") + db.command("sql", "CREATE INDEX ON Person (email) UNIQUE") + with db.transaction(): + for i in range(20): + db.command( + "sql", f"INSERT INTO Person SET email = 'p{i}@x.org'" + ) # nosec B608 + + # CHECK DATABASE on a clean database reports no error + report = db.command("sql", "CHECK DATABASE").first().to_dict() + assert report["operation"] == "check database" + assert report.get("totalErrors", 0) == 0 + report = db.command("sql", "CHECK DATABASE FIX").first().to_dict() + assert report.get("totalErrors", 0) == 0 + + # REBUILD INDEX re-indexes every record + rebuilt = db.command("sql", "REBUILD INDEX *").first().to_dict() + assert rebuilt["operation"] == "rebuild index" + assert rebuilt["totalIndexed"] == 20 + assert not rebuilt.get("failedIndexes") + assert len(rebuilt["indexes"]) >= 1 + one = db.command("sql", "REBUILD INDEX `Person[email]`").first() + assert one.get("operation") == "rebuild index" + + # BACKUP DATABASE writes // + config = jpype.JClass("com.arcadedb.GlobalConfiguration").SERVER_BACKUP_DIRECTORY + config.setValue(str(tmp_path / "backups")) + try: + result = db.command("sql", "BACKUP DATABASE 'person.zip'").first().to_dict() + finally: + config.reset() + assert result["result"] == "OK" + archive = tmp_path / "backups" / str(db.get_name()) / "person.zip" + assert archive.is_file() + + # the archive is the database: unzip it and open it + restored = tmp_path / "restored" + with zipfile.ZipFile(archive) as zf: + zf.extractall(restored) + with arcadedb.open_database(str(restored)) as copy: + assert copy.count_type("Person") == 20 + found = copy.query( + "sql", "SELECT FROM Person WHERE email = 'p7@x.org'" + ).to_list() + assert len(found) == 1