From 48250cd671007958d68fe494ca209f63139ef0f8 Mon Sep 17 00:00:00 2001 From: Alexis Montagne Date: Fri, 23 Oct 2026 11:23:27 -0700 Subject: [PATCH 1/6] *: expose the driver of transactions Only sql.DB exposed its driver, so code holding a transaction, such as the x/sqlbuilder upserter, could not tell which dialect to write. Transactions now implement an optional Driver method, read through sql.DriverOf. --- backend/balancer/db.go | 2 ++ backend/postgres/db.go | 1 + backend/simple/db.go | 13 +++++++++---- backend/sqlite3/db.go | 1 + backend/static/db.go | 1 + db.go | 10 ++++++++++ middleware/logger/middleware.go | 2 ++ 7 files changed, 26 insertions(+), 4 deletions(-) diff --git a/backend/balancer/db.go b/backend/balancer/db.go index 91268cb..04921f4 100644 --- a/backend/balancer/db.go +++ b/backend/balancer/db.go @@ -120,6 +120,8 @@ type tx struct { cfn CloseFunc } +func (tx *tx) Driver() string { return sql.DriverOf(tx.Tx) } + func (tx *tx) Commit() error { err := tx.Tx.Commit() diff --git a/backend/postgres/db.go b/backend/postgres/db.go index 967157c..41d5642 100644 --- a/backend/postgres/db.go +++ b/backend/postgres/db.go @@ -42,6 +42,7 @@ type tx struct { func (tx *tx) Commit() error { return wrapErr(tx.tx.Commit()) } func (tx *tx) Rollback() error { return wrapErr(tx.tx.Rollback()) } +func (tx *tx) Driver() string { return sql.DriverOf(tx.tx) } type queryer struct { q sql.Queryer diff --git a/backend/simple/db.go b/backend/simple/db.go index 97d7ba2..da7d07e 100644 --- a/backend/simple/db.go +++ b/backend/simple/db.go @@ -37,8 +37,12 @@ type tx struct { q *queryer tx *stdsql.Tx + + driver string } +func (tx *tx) Driver() string { return tx.driver } + func (tx *tx) Commit() error { select { case <-tx.ctx.Done(): @@ -160,9 +164,10 @@ func (d *db) BeginTx(ctx context.Context, opts sql.TxOptions) (sql.Tx, error) { } return &tx{ - ctx: ctx, - ch: make(chan struct{}, 1), - q: &queryer{t}, - tx: t, + ctx: ctx, + ch: make(chan struct{}, 1), + q: &queryer{t}, + tx: t, + driver: d.driver, }, nil } diff --git a/backend/sqlite3/db.go b/backend/sqlite3/db.go index 29d49dc..868f4c4 100644 --- a/backend/sqlite3/db.go +++ b/backend/sqlite3/db.go @@ -46,6 +46,7 @@ type tx struct { func (tx *tx) Commit() error { return wrapErr(tx.tx.Commit()) } func (tx *tx) Rollback() error { return wrapErr(tx.tx.Rollback()) } +func (tx *tx) Driver() string { return sql.DriverOf(tx.tx) } func (db *db) Driver() string { return db.db.Driver() } diff --git a/backend/static/db.go b/backend/static/db.go index 63641ef..f941989 100644 --- a/backend/static/db.go +++ b/backend/static/db.go @@ -57,6 +57,7 @@ type Tx struct { func (tx *Tx) Commit() error { return tx.CommitErr } func (tx *Tx) Rollback() error { return tx.RollbackErr } +func (tx *Tx) Driver() string { return "sqltest" } func (db *DB) Driver() string { return "sqltest" } diff --git a/db.go b/db.go index b6e4861..f9f13ca 100644 --- a/db.go +++ b/db.go @@ -40,6 +40,16 @@ type DB interface { Driver() string } +// DriverOf returns the driver of q, the DBs and the transactions they +// begin expose it. It returns an empty string when q does not. +func DriverOf(q Queryer) string { + if d, ok := q.(interface{ Driver() string }); ok { + return d.Driver() + } + + return "" +} + type TxOptions struct { Isolation IsolationLevel } diff --git a/middleware/logger/middleware.go b/middleware/logger/middleware.go index a669f68..7e2edd1 100644 --- a/middleware/logger/middleware.go +++ b/middleware/logger/middleware.go @@ -101,6 +101,8 @@ type tx struct { tx sql.Tx } +func (t *tx) Driver() string { return sql.DriverOf(t.tx) } + func (t *tx) Commit() error { var t0 = time.Now() From b41d73998956b6fd53b07d9963028b541b4af41e Mon Sep 17 00:00:00 2001 From: Alexis Montagne Date: Fri, 23 Oct 2026 11:23:28 -0700 Subject: [PATCH 2/6] x/sqlbuilder!: expose the driver to QueryWriter Lets query segments write SQL specific to the database they target, the driver being read from the Queryer of the QueryBuilder. Implementations of QueryWriter outside this package have to add the Driver method. --- x/sqlbuilder/delete_statement.go | 4 ++-- x/sqlbuilder/delete_statement_test.go | 2 +- x/sqlbuilder/execer.go | 2 +- x/sqlbuilder/insert_statement.go | 8 ++++---- x/sqlbuilder/insert_statement_test.go | 4 ++-- x/sqlbuilder/query_builder.go | 22 ++++++++++++++++------ x/sqlbuilder/select_statement.go | 5 +++-- x/sqlbuilder/select_statement_test.go | 2 +- x/sqlbuilder/update_statement.go | 4 ++-- x/sqlbuilder/update_statement_test.go | 2 +- 10 files changed, 33 insertions(+), 22 deletions(-) diff --git a/x/sqlbuilder/delete_statement.go b/x/sqlbuilder/delete_statement.go index 48cb3c5..c069751 100644 --- a/x/sqlbuilder/delete_statement.go +++ b/x/sqlbuilder/delete_statement.go @@ -15,8 +15,8 @@ func (ds DeleteStatement) Clone() DeleteStatement { } } -func (ds DeleteStatement) buildQuery(vs map[string]any) (string, []any, error) { - var qw queryWriter +func (ds DeleteStatement) buildQuery(driver string, vs map[string]any) (string, []any, error) { + qw := queryWriter{driver: driver} fmt.Fprintf(&qw, "DELETE FROM %s WHERE ", ds.Table) diff --git a/x/sqlbuilder/delete_statement_test.go b/x/sqlbuilder/delete_statement_test.go index af9ae42..0de6161 100644 --- a/x/sqlbuilder/delete_statement_test.go +++ b/x/sqlbuilder/delete_statement_test.go @@ -34,7 +34,7 @@ func TestDeleteQuery(t *testing.T) { }, } { t.Run(tt.name, func(t *testing.T) { - stmt, args, err := tt.ds.Clone().buildQuery(tt.vs) + stmt, args, err := tt.ds.Clone().buildQuery("", tt.vs) assert.Equal(t, tt.stmt, stmt) assert.Equal(t, tt.args, args) diff --git a/x/sqlbuilder/execer.go b/x/sqlbuilder/execer.go index 1d7d2ee..4e8e5a3 100644 --- a/x/sqlbuilder/execer.go +++ b/x/sqlbuilder/execer.go @@ -16,7 +16,7 @@ type execer struct { } func (e execer) Exec(ctx context.Context, qvs map[string]any) (sql.Result, error) { - stmt, vs, err := e.stmt.buildQuery(qvs) + stmt, vs, err := e.stmt.buildQuery(e.qb.driver(), qvs) if err != nil { return nil, err diff --git a/x/sqlbuilder/insert_statement.go b/x/sqlbuilder/insert_statement.go index d3b217d..189cb0c 100644 --- a/x/sqlbuilder/insert_statement.go +++ b/x/sqlbuilder/insert_statement.go @@ -156,12 +156,12 @@ func (is InsertStatement) returnings() []*sql.Returning { return res } -func (is InsertStatement) buildQuery(qvs map[string]any) (string, []any, error) { - return is.buildQueries([]map[string]any{qvs}, qvs) +func (is InsertStatement) buildQuery(driver string, qvs map[string]any) (string, []any, error) { + return is.buildQueries(driver, []map[string]any{qvs}, qvs) } -func (is InsertStatement) buildQueries(vvs []map[string]any, qvs map[string]any) (string, []any, error) { - var qw queryWriter +func (is InsertStatement) buildQueries(driver string, vvs []map[string]any, qvs map[string]any) (string, []any, error) { + qw := queryWriter{driver: driver} if len(is.Fields) == 0 { return "", nil, errNoMarkers diff --git a/x/sqlbuilder/insert_statement_test.go b/x/sqlbuilder/insert_statement_test.go index 55f39f2..a4c02e4 100644 --- a/x/sqlbuilder/insert_statement_test.go +++ b/x/sqlbuilder/insert_statement_test.go @@ -176,7 +176,7 @@ func TestInsertQuery(t *testing.T) { }, } { t.Run(tt.name, func(t *testing.T) { - stmt, args, err := tt.is.Clone().buildQuery(tt.vs) + stmt, args, err := tt.is.Clone().buildQuery("", tt.vs) assert.Equal(t, tt.err, err) assert.Equal(t, tt.stmt, stmt) @@ -237,7 +237,7 @@ func TestInsertMultiQuery(t *testing.T) { }, } { t.Run(tt.name, func(t *testing.T) { - stmt, args, err := tt.haveStmt.buildQueries(tt.haveVVs, nil) + stmt, args, err := tt.haveStmt.buildQueries("", tt.haveVVs, nil) assert.Equal(t, tt.wantErr, err) assert.Equal(t, tt.wantStmt, stmt) diff --git a/x/sqlbuilder/query_builder.go b/x/sqlbuilder/query_builder.go index 82c7ce9..38aa630 100644 --- a/x/sqlbuilder/query_builder.go +++ b/x/sqlbuilder/query_builder.go @@ -15,6 +15,8 @@ type QueryBuilder struct { sql.Queryer } +func (qb *QueryBuilder) driver() string { return sql.DriverOf(qb.Queryer) } + func (qb *QueryBuilder) PrepareSelect(ss SelectStatement) *SelectQueryer { return &SelectQueryer{QueryBuilder: qb, Statement: ss} } @@ -44,7 +46,7 @@ func (qb *QueryBuilder) PrepareDelete(ds DeleteStatement) *DeleteExecer { } type statement interface { - buildQuery(map[string]any) (string, []any, error) + buildQuery(string, map[string]any) (string, []any, error) } type InsertExecer struct { @@ -66,7 +68,7 @@ func (ie *InsertExecer) QueryRow(ctx context.Context, qvs map[string]any) sql.Sc stmt := ie.Statement stmt.isQuery = true - sstmt, vs, err := stmt.buildQuery(qvs) + sstmt, vs, err := stmt.buildQuery(ie.qb.driver(), qvs) if err != nil { return errScanner{error: err} @@ -76,7 +78,7 @@ func (ie *InsertExecer) QueryRow(ctx context.Context, qvs map[string]any) sql.Sc } func (ie *InsertExecer) MultiExec(ctx context.Context, vvs []map[string]any, qvs map[string]any) (sql.Result, error) { - stmt, vs, err := ie.Statement.buildQueries(vvs, qvs) + stmt, vs, err := ie.Statement.buildQueries(ie.qb.driver(), vvs, qvs) if err != nil { return nil, err @@ -94,7 +96,7 @@ func (ie *InsertExecer) MultiQuery(ctx context.Context, vvs []map[string]any, qv stmt := ie.Statement stmt.isQuery = true - sstmt, vs, err := stmt.buildQueries(vvs, qvs) + sstmt, vs, err := stmt.buildQueries(ie.qb.driver(), vvs, qvs) if err != nil { return nil, err @@ -136,7 +138,7 @@ type SelectQueryer struct { } func (sq *SelectQueryer) Query(ctx context.Context, qvs map[string]any) (Cursor, error) { - stmt, vs, ks, err := sq.Statement.buildQuery(qvs) + stmt, vs, ks, err := sq.Statement.buildQuery(sq.QueryBuilder.driver(), qvs) if err != nil { return nil, err @@ -152,7 +154,7 @@ func (sq *SelectQueryer) Query(ctx context.Context, qvs map[string]any) (Cursor, } func (sq *SelectQueryer) QueryRow(ctx context.Context, qvs map[string]any) Scanner { - stmt, vs, ks, err := sq.Statement.buildQuery(qvs) + stmt, vs, ks, err := sq.Statement.buildQuery(sq.QueryBuilder.driver(), qvs) if err != nil { return ErrScanner{Err: err} @@ -165,6 +167,10 @@ type QueryWriter interface { io.Writer RedeemVariable(any) string + + // Driver returns the driver the query is built for, or an empty string + // when the queryer of the QueryBuilder does not expose it. + Driver() string } type queryWriter struct { @@ -172,8 +178,12 @@ type queryWriter struct { i int vs []any + + driver string } +func (qw *queryWriter) Driver() string { return qw.driver } + func (qw *queryWriter) RedeemVariable(v any) string { qw.i++ qw.vs = append(qw.vs, v) diff --git a/x/sqlbuilder/select_statement.go b/x/sqlbuilder/select_statement.go index 4cd203e..c8b371a 100644 --- a/x/sqlbuilder/select_statement.go +++ b/x/sqlbuilder/select_statement.go @@ -51,10 +51,11 @@ func writeSelectClause(c Marker, qw *queryWriter, vs map[string]any) error { return nil } -func (ss SelectStatement) buildQuery(vs map[string]any) (string, []any, []string, error) { +func (ss SelectStatement) buildQuery(driver string, vs map[string]any) (string, []any, []string, error) { var ( - qw queryWriter bindings []string + + qw = queryWriter{driver: driver} ) if len(ss.SelectClauses) == 0 { diff --git a/x/sqlbuilder/select_statement_test.go b/x/sqlbuilder/select_statement_test.go index 661bb28..3e140db 100644 --- a/x/sqlbuilder/select_statement_test.go +++ b/x/sqlbuilder/select_statement_test.go @@ -411,7 +411,7 @@ func TestSelectQuery(t *testing.T) { }, } { t.Run(tt.name, func(t *testing.T) { - stmt, args, _, err := tt.ss.Clone().buildQuery(tt.vs) + stmt, args, _, err := tt.ss.Clone().buildQuery("", tt.vs) assert.Equal(t, tt.err, err) assert.Equal(t, tt.stmt, stmt) diff --git a/x/sqlbuilder/update_statement.go b/x/sqlbuilder/update_statement.go index f4bff54..bf8cec2 100644 --- a/x/sqlbuilder/update_statement.go +++ b/x/sqlbuilder/update_statement.go @@ -52,8 +52,8 @@ func writeUpdateClauses(fs []Marker, qw QueryWriter, vs map[string]any) error { return nil } -func (us UpdateStatement) buildQuery(vs map[string]any) (string, []any, error) { - var qw queryWriter +func (us UpdateStatement) buildQuery(driver string, vs map[string]any) (string, []any, error) { + qw := queryWriter{driver: driver} if len(us.Fields) == 0 { return "", nil, errNoMarkers diff --git a/x/sqlbuilder/update_statement_test.go b/x/sqlbuilder/update_statement_test.go index b5d5f17..836bc32 100644 --- a/x/sqlbuilder/update_statement_test.go +++ b/x/sqlbuilder/update_statement_test.go @@ -58,7 +58,7 @@ func TestUpdateQuery(t *testing.T) { }, } { t.Run(tt.name, func(t *testing.T) { - stmt, args, err := tt.us.Clone().buildQuery(tt.vs) + stmt, args, err := tt.us.Clone().buildQuery("", tt.vs) assert.Equal(t, tt.err, err) assert.Equal(t, tt.stmt, stmt) From 20c26925032d3c377910bb557bb530ebe4ce58b7 Mon Sep 17 00:00:00 2001 From: Alexis Montagne Date: Fri, 23 Oct 2026 11:23:28 -0700 Subject: [PATCH 3/6] x/sqlbuilder: add JSONB containment predicates Adds JSONBArrayContainsAny, JSONBArrayContainsAll and JSONBObjectContains, written with the @> operator on PostgreSQL and with the json1 functions on SQLite, so the same predicate can be used by services and their SQLite backed tests. Only strings and numbers are supported as values. --- x/sqlbuilder/jsonb_predicate_clause.go | 398 ++++++++++++++++++++ x/sqlbuilder/jsonb_predicate_clause_test.go | 257 +++++++++++++ 2 files changed, 655 insertions(+) create mode 100644 x/sqlbuilder/jsonb_predicate_clause.go create mode 100644 x/sqlbuilder/jsonb_predicate_clause_test.go diff --git a/x/sqlbuilder/jsonb_predicate_clause.go b/x/sqlbuilder/jsonb_predicate_clause.go new file mode 100644 index 0000000..a3d6f49 --- /dev/null +++ b/x/sqlbuilder/jsonb_predicate_clause.go @@ -0,0 +1,398 @@ +package sqlbuilder + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "slices" + "strings" + + "github.com/upfluence/errors" +) + +const ( + sqlite3Driver = "sqlite3" + + // sqliteJSONBElementMatch matches the elements of a JSON array iterated + // as jsonb_elem with the ones of the array of wanted values iterated as + // jsonb_want. Values of different storage classes are never equal in + // SQLite, strings never match numbers, and 1 matches 1.0 as it does in + // PostgreSQL. + sqliteJSONBElementMatch = "jsonb_elem.type IN ('integer', 'real', 'text') " + + "AND jsonb_elem.value = jsonb_want.value" +) + +var ( + errInvalidJSONBValue = errors.New("JSONB predicates only support strings and numbers") + errInvalidJSONBKey = errors.New("SQLite does not support object keys holding a double quote") +) + +// JSONBArrayContainsAny matches the rows whose column, a JSONB array, holds +// at least one of the elements of the slice bound to the marker. The +// elements have to be strings or numbers. +func JSONBArrayContainsAny(m Marker) PredicateClause { + return &basicClause{m: m, fn: writeJSONBArrayContainsAny} +} + +// JSONBArrayContainsAll matches the rows whose column, a JSONB array, holds +// every element of the slice bound to the marker. The elements have to be +// strings or numbers. +func JSONBArrayContainsAll(m Marker) PredicateClause { + return &basicClause{m: m, fn: writeJSONBArrayContainsAll} +} + +// JSONBObjectContains matches the rows whose column, a JSONB object, +// contains the object bound to the marker, as the @> operator of +// PostgreSQL does. The object can only hold strings, numbers, objects and +// arrays of strings and numbers. +func JSONBObjectContains(m Marker) PredicateClause { + return &basicClause{m: m, fn: writeJSONBObjectContains} +} + +func StaticJSONBArrayContainsAny(m Marker, v any) PredicateClause { + return Static(JSONBArrayContainsAny(m), map[string]any{m.Binding(): v}) +} + +func StaticJSONBArrayContainsAll(m Marker, v any) PredicateClause { + return Static(JSONBArrayContainsAll(m), map[string]any{m.Binding(): v}) +} + +func StaticJSONBObjectContains(m Marker, v any) PredicateClause { + return Static(JSONBObjectContains(m), map[string]any{m.Binding(): v}) +} + +func writeJSONBArrayContainsAny(w QueryWriter, vv any, k string) error { + es, err := decodeJSONBArray(vv) + + if err != nil { + return err + } + + if len(es) == 0 { + _, err := io.WriteString(w, "1=0") + + return errors.WithStack(err) + } + + if w.Driver() == sqlite3Driver { + want, err := encodeJSONB(es) + + if err != nil { + return err + } + + _, err = fmt.Fprintf( + w, + "(json_type(%s) = 'array' AND EXISTS (SELECT 1 FROM json_each(%s) "+ + "AS jsonb_elem, json_each(%s) AS jsonb_want WHERE %s))", + k, + k, + w.RedeemVariable(want), + sqliteJSONBElementMatch, + ) + + return errors.WithStack(err) + } + + // A disjunction of containments, unlike the ?| operator, matches + // numbers and can use both the jsonb_ops and jsonb_path_ops GIN indexes. + cs := make([]string, len(es)) + + for i, e := range es { + v, err := encodeJSONB([]any{e}) + + if err != nil { + return err + } + + cs[i] = fmt.Sprintf("%s @> %s::jsonb", k, w.RedeemVariable(v)) + } + + _, err = fmt.Fprintf(w, "(%s)", strings.Join(cs, " OR ")) + + return errors.WithStack(err) +} + +func writeJSONBArrayContainsAll(w QueryWriter, vv any, k string) error { + es, err := decodeJSONBArray(vv) + + if err != nil { + return err + } + + if w.Driver() == sqlite3Driver { + c, err := sqliteArrayContainsAll(w, k, "", es) + + if err != nil { + return err + } + + _, err = fmt.Fprintf(w, "(%s)", c) + + return errors.WithStack(err) + } + + return writePostgresContains(w, k, es) +} + +func writeJSONBObjectContains(w QueryWriter, vv any, k string) error { + d, err := decodeJSONB(vv) + + if err != nil { + return err + } + + obj, ok := d.(map[string]any) + + if !ok { + return errors.Wrapf(errInvalidJSONBValue, "%T is not an object", vv) + } + + if err := checkJSONBObject(obj); err != nil { + return err + } + + if w.Driver() != sqlite3Driver { + return writePostgresContains(w, k, obj) + } + + cs, err := sqliteObjectContains( + w, + k, + "$", + obj, + []string{fmt.Sprintf("json_type(%s) = 'object'", k)}, + ) + + if err != nil { + return err + } + + _, err = fmt.Fprintf(w, "(%s)", strings.Join(cs, " AND ")) + + return errors.WithStack(err) +} + +func writePostgresContains(w QueryWriter, k string, v any) error { + want, err := encodeJSONB(v) + + if err != nil { + return err + } + + _, err = fmt.Fprintf(w, "%s @> %s::jsonb", k, w.RedeemVariable(want)) + + return errors.WithStack(err) +} + +// sqliteObjectContains appends to cs the conditions matching when the JSON +// object at path in k contains obj: one per leaf of obj. +func sqliteObjectContains(w QueryWriter, k, path string, obj map[string]any, cs []string) ([]string, error) { + keys := make([]string, 0, len(obj)) + + for key := range obj { + keys = append(keys, key) + } + + slices.Sort(keys) + + for _, key := range keys { + if strings.Contains(key, `"`) { + return nil, errors.Wrapf(errInvalidJSONBKey, "%q", key) + } + + p := fmt.Sprintf(`%s."%s"`, path, key) + + switch v := obj[key].(type) { + case map[string]any: + if len(v) > 0 { + var err error + + cs, err = sqliteObjectContains(w, k, p, v, cs) + + if err != nil { + return nil, err + } + + continue + } + + cs = append( + cs, + fmt.Sprintf("json_type(%s, %s) = 'object'", k, w.RedeemVariable(p)), + ) + case []any: + c, err := sqliteArrayContainsAll(w, k, p, v) + + if err != nil { + return nil, err + } + + cs = append(cs, c) + default: + var ( + typ = w.RedeemVariable(p) + value = w.RedeemVariable(p) + ) + + cs = append( + cs, + fmt.Sprintf( + "json_type(%s, %s) IN ('integer', 'real', 'text') AND "+ + "json_extract(%s, %s) = %s", + k, + typ, + k, + value, + w.RedeemVariable(sqliteScalar(v)), + ), + ) + } + } + + return cs, nil +} + +// sqliteArrayContainsAll returns the condition matching when the JSON array +// at path in k, or k itself when path is empty, holds every element of es. +// The column is iterated first so that its name is never resolved to one +// of the columns of the iteration of the wanted values. +func sqliteArrayContainsAll(w QueryWriter, k, path string, es []any) (string, error) { + want, err := encodeJSONB(es) + + if err != nil { + return "", err + } + + var ( + typ = sqliteJSONArgs(w, k, path) + elem = sqliteJSONArgs(w, k, path) + ) + + return fmt.Sprintf( + "json_type(%s) = 'array' AND (SELECT count(DISTINCT jsonb_want.key) "+ + "FROM json_each(%s) AS jsonb_elem, json_each(%s) AS jsonb_want "+ + "WHERE %s) = %d", + typ, + elem, + w.RedeemVariable(want), + sqliteJSONBElementMatch, + len(es), + ), nil +} + +func sqliteJSONArgs(w QueryWriter, k, path string) string { + if path == "" { + return k + } + + return k + ", " + w.RedeemVariable(path) +} + +// sqliteScalar converts a string or a JSON number to the value json_extract +// returns for it. +func sqliteScalar(v any) any { + n, ok := v.(json.Number) + + if !ok { + return v + } + + if i, err := n.Int64(); err == nil { + return i + } + + f, _ := n.Float64() + + return f +} + +func decodeJSONB(v any) (any, error) { + buf, err := json.Marshal(v) + + if err != nil { + return nil, errors.Wrap(err, "cant encode the JSONB value") + } + + var ( + res any + + dec = json.NewDecoder(bytes.NewReader(buf)) + ) + + dec.UseNumber() + + if err := dec.Decode(&res); err != nil { + return nil, errors.Wrap(err, "cant decode the JSONB value") + } + + return res, nil +} + +func decodeJSONBArray(v any) ([]any, error) { + d, err := decodeJSONB(v) + + if err != nil { + return nil, err + } + + switch d := d.(type) { + case nil: + return []any{}, nil + case []any: + for _, e := range d { + if err := checkJSONBScalar(e); err != nil { + return nil, err + } + } + + return d, nil + } + + return nil, errors.Wrapf(errInvalidJSONBValue, "%T is not an array", v) +} + +func checkJSONBScalar(v any) error { + switch v.(type) { + case string, json.Number: + return nil + } + + return errors.Wrapf(errInvalidJSONBValue, "unexpected %T", v) +} + +func checkJSONBObject(obj map[string]any) error { + for _, v := range obj { + var err error + + switch v := v.(type) { + case map[string]any: + err = checkJSONBObject(v) + case []any: + for _, e := range v { + if err = checkJSONBScalar(e); err != nil { + break + } + } + default: + err = checkJSONBScalar(v) + } + + if err != nil { + return err + } + } + + return nil +} + +func encodeJSONB(v any) (string, error) { + buf, err := json.Marshal(v) + + if err != nil { + return "", errors.Wrap(err, "cant encode the JSONB value") + } + + return string(buf), nil +} diff --git a/x/sqlbuilder/jsonb_predicate_clause_test.go b/x/sqlbuilder/jsonb_predicate_clause_test.go new file mode 100644 index 0000000..7364fc6 --- /dev/null +++ b/x/sqlbuilder/jsonb_predicate_clause_test.go @@ -0,0 +1,257 @@ +package sqlbuilder + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/upfluence/errors/errtest" + + "github.com/upfluence/sql/backend/static" +) + +const postgresDriver = "postgres" + +func TestJSONBPredicateClause(t *testing.T) { + var ( + tags = Column("tags") + data = Column("data") + ) + + for _, tt := range []struct { + name string + haveDriver string + haveClause PredicateClause + want string + wantArgs []any + wantErr errtest.ErrorAssertion + }{ + { + name: "postgres contains any", + haveDriver: postgresDriver, + haveClause: StaticJSONBArrayContainsAny(tags, []any{"a", 1}), + want: "(tags @> $1::jsonb OR tags @> $2::jsonb)", + wantArgs: []any{`["a"]`, `[1]`}, + wantErr: errtest.NoError(), + }, + { + name: "unknown driver contains any", + haveClause: StaticJSONBArrayContainsAny(tags, []string{"z"}), + want: "(tags @> $1::jsonb)", + wantArgs: []any{`["z"]`}, + wantErr: errtest.NoError(), + }, + { + name: "postgres contains any of nothing", + haveDriver: postgresDriver, + haveClause: StaticJSONBArrayContainsAny(tags, []string{}), + want: "1=0", + wantErr: errtest.NoError(), + }, + { + name: "postgres contains all", + haveDriver: postgresDriver, + haveClause: StaticJSONBArrayContainsAll(tags, []any{"a", 1.5}), + want: "tags @> $1::jsonb", + wantArgs: []any{`["a",1.5]`}, + wantErr: errtest.NoError(), + }, + { + name: "postgres contains all of nil", + haveDriver: postgresDriver, + haveClause: StaticJSONBArrayContainsAll(tags, []string(nil)), + want: "tags @> $1::jsonb", + wantArgs: []any{`[]`}, + wantErr: errtest.NoError(), + }, + { + name: "postgres object contains", + haveDriver: postgresDriver, + haveClause: StaticJSONBObjectContains( + data, + map[string]any{ + "a": map[string]any{"b": "x"}, + "n": 1, + "labels": []string{"t"}, + }, + ), + want: "data @> $1::jsonb", + wantArgs: []any{`{"a":{"b":"x"},"labels":["t"],"n":1}`}, + wantErr: errtest.NoError(), + }, + { + name: "sqlite contains any", + haveDriver: sqlite3Driver, + haveClause: StaticJSONBArrayContainsAny(tags, []string{"a", "b"}), + want: "(json_type(tags) = 'array' AND EXISTS (SELECT 1 FROM " + + "json_each(tags) AS jsonb_elem, json_each($1) AS jsonb_want " + + "WHERE " + sqliteJSONBElementMatch + "))", + wantArgs: []any{`["a","b"]`}, + wantErr: errtest.NoError(), + }, + { + name: "sqlite contains any of nothing", + haveDriver: sqlite3Driver, + haveClause: StaticJSONBArrayContainsAny(tags, []string{}), + want: "1=0", + wantErr: errtest.NoError(), + }, + { + name: "sqlite contains all", + haveDriver: sqlite3Driver, + haveClause: StaticJSONBArrayContainsAll(tags, []int{1, 2}), + want: "(json_type(tags) = 'array' AND (SELECT count(DISTINCT " + + "jsonb_want.key) FROM json_each(tags) AS jsonb_elem, " + + "json_each($1) AS jsonb_want WHERE " + sqliteJSONBElementMatch + + ") = 2)", + wantArgs: []any{`[1,2]`}, + wantErr: errtest.NoError(), + }, + { + name: "sqlite object contains", + haveDriver: sqlite3Driver, + haveClause: StaticJSONBObjectContains( + data, + map[string]any{ + "a": map[string]any{"b": "x"}, + "e": map[string]any{}, + "n": 1.5, + "i": 2, + "labels": []string{"t"}, + }, + ), + want: "(json_type(data) = 'object' AND " + + "json_type(data, $1) IN ('integer', 'real', 'text') AND " + + "json_extract(data, $2) = $3 AND " + + "json_type(data, $4) = 'object' AND " + + "json_type(data, $5) IN ('integer', 'real', 'text') AND " + + "json_extract(data, $6) = $7 AND " + + "json_type(data, $8) = 'array' AND (SELECT count(DISTINCT " + + "jsonb_want.key) FROM json_each(data, $9) AS jsonb_elem, " + + "json_each($10) AS jsonb_want WHERE " + sqliteJSONBElementMatch + + ") = 1 AND " + + "json_type(data, $11) IN ('integer', 'real', 'text') AND " + + "json_extract(data, $12) = $13)", + wantArgs: []any{ + `$."a"."b"`, `$."a"."b"`, "x", + `$."e"`, + `$."i"`, `$."i"`, int64(2), + `$."labels"`, `$."labels"`, `["t"]`, + `$."n"`, `$."n"`, 1.5, + }, + wantErr: errtest.NoError(), + }, + { + name: "sqlite object contains nothing", + haveDriver: sqlite3Driver, + haveClause: StaticJSONBObjectContains(data, struct{}{}), + want: "(json_type(data) = 'object')", + wantErr: errtest.NoError(), + }, + { + name: "boolean element", + haveDriver: postgresDriver, + haveClause: StaticJSONBArrayContainsAny(tags, []any{"a", true}), + wantErr: errtest.ErrorCause(errInvalidJSONBValue), + }, + { + name: "null element", + haveDriver: sqlite3Driver, + haveClause: StaticJSONBArrayContainsAll(tags, []any{nil}), + wantErr: errtest.ErrorCause(errInvalidJSONBValue), + }, + { + name: "array of objects", + haveDriver: postgresDriver, + haveClause: StaticJSONBArrayContainsAll(tags, []any{map[string]any{}}), + wantErr: errtest.ErrorCause(errInvalidJSONBValue), + }, + { + name: "not an array", + haveDriver: sqlite3Driver, + haveClause: StaticJSONBArrayContainsAny(tags, "a"), + wantErr: errtest.ErrorCause(errInvalidJSONBValue), + }, + { + name: "object holding an array of objects", + haveDriver: postgresDriver, + haveClause: StaticJSONBObjectContains( + data, + map[string]any{"items": []any{map[string]any{"sku": "x"}}}, + ), + wantErr: errtest.ErrorCause(errInvalidJSONBValue), + }, + { + name: "object holding a boolean", + haveDriver: sqlite3Driver, + haveClause: StaticJSONBObjectContains( + data, + map[string]any{"a": map[string]any{"b": false}}, + ), + wantErr: errtest.ErrorCause(errInvalidJSONBValue), + }, + { + name: "not an object", + haveDriver: postgresDriver, + haveClause: StaticJSONBObjectContains(data, []string{"a"}), + wantErr: errtest.ErrorCause(errInvalidJSONBValue), + }, + { + name: "sqlite object key holding a double quote", + haveDriver: sqlite3Driver, + haveClause: StaticJSONBObjectContains(data, map[string]any{`a"b`: 1}), + wantErr: errtest.ErrorCause(errInvalidJSONBKey), + }, + } { + t.Run(tt.name, func(t *testing.T) { + w := queryWriter{driver: tt.haveDriver} + + err := tt.haveClause.WriteTo(&w, nil) + + tt.wantErr.Assert(t, err) + assert.Equal(t, tt.want, w.String()) + assert.Equal(t, tt.wantArgs, w.vs) + }) + } +} + +type driverQueryer struct { + *static.Queryer + + driver string +} + +func (dq driverQueryer) Driver() string { return dq.driver } + +func TestJSONBPredicateClauseDriver(t *testing.T) { + var ( + q static.Queryer + + tags = Column("tags") + qb = QueryBuilder{ + Queryer: driverQueryer{Queryer: &q, driver: sqlite3Driver}, + } + ) + + _, err := qb.PrepareDelete( + DeleteStatement{ + Table: "rows", + WhereClause: JSONBArrayContainsAny(tags), + }, + ).Exec(context.Background(), map[string]any{tags.Binding(): []string{"c"}}) + + require.NoError(t, err) + require.Len(t, q.ExecQueries, 1) + assert.Equal( + t, + static.Query{ + Query: "DELETE FROM rows WHERE (json_type(tags) = 'array' AND " + + "EXISTS (SELECT 1 FROM json_each(tags) AS jsonb_elem, " + + "json_each($1) AS jsonb_want WHERE " + sqliteJSONBElementMatch + + "))", + Args: []any{`["c"]`}, + }, + q.ExecQueries[0], + ) +} From 5616399968809541b413b27e589f12931fa7046a Mon Sep 17 00:00:00 2001 From: Alexis Montagne Date: Fri, 23 Oct 2026 11:23:28 -0700 Subject: [PATCH 4/6] integration: run the JSONB predicates on postgres and sqlite3 --- integration/jsonb_test.go | 256 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 256 insertions(+) create mode 100644 integration/jsonb_test.go diff --git a/integration/jsonb_test.go b/integration/jsonb_test.go new file mode 100644 index 0000000..f0ee6de --- /dev/null +++ b/integration/jsonb_test.go @@ -0,0 +1,256 @@ +package integration + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/upfluence/sql" + "github.com/upfluence/sql/sqltest" + "github.com/upfluence/sql/x/migration" + "github.com/upfluence/sql/x/sqlbuilder" +) + +const jsonbAddressKey = "address" + +func jsonbMigrator(db sql.DB) migration.Migrator { + up := "CREATE TABLE jsonb_rows(id INTEGER PRIMARY KEY, tags TEXT, data TEXT)" + + if db.Driver() == postgresDriver { + up = "CREATE TABLE jsonb_rows(id INTEGER PRIMARY KEY, tags JSONB, data JSONB)" + } + + return migration.NewMigrator( + db, + staticSource{up: up, down: "DROP TABLE jsonb_rows"}, + ) +} + +func TestJSONBPredicateClause(t *testing.T) { + sqltest.NewTestCase( + sqltest.WithMigratorFunc(jsonbMigrator), + ).Run(t, func(t *testing.T, db sql.DB) { + ctx := context.Background() + + for _, row := range []struct { + id int64 + tags, data any + }{ + { + id: 1, + tags: `["a", "b"]`, + data: `{"status": "active", "tier": 2, "address": {"country": "FR"}, "labels": ["x", "y"]}`, + }, + { + id: 2, + tags: `[1, 2.5]`, + data: `{"status": "inactive", "tier": 2.0, "address": {"country": "US"}}`, + }, + { + id: 3, + tags: `["1", true]`, + data: `{"status": "active", "tier": "2", "labels": "x"}`, + }, + {id: 4, tags: `{"a": "a"}`, data: `["status"]`}, + {id: 5, tags: `[]`, data: `{}`}, + {id: 6}, + { + id: 7, + tags: `[["a"]]`, + data: `{"address": {"country": "FR", "city": "Paris"}, "labels": ["y", "x", "z"], "flags": true}`, + }, + } { + _, err := db.Exec( + ctx, + "INSERT INTO jsonb_rows(id, tags, data) VALUES ($1, $2, $3)", + row.id, + row.tags, + row.data, + ) + + require.NoError(t, err) + } + + var ( + tags = sqlbuilder.Column("tags") + data = sqlbuilder.Column("data") + ) + + for _, tt := range []struct { + name string + haveClause sqlbuilder.PredicateClause + want []int64 + }{ + { + name: "contains any string", + haveClause: sqlbuilder.StaticJSONBArrayContainsAny(tags, []string{"a"}), + want: []int64{1}, + }, + { + name: "contains any number", + haveClause: sqlbuilder.StaticJSONBArrayContainsAny(tags, []int{1}), + want: []int64{2}, + }, + { + name: "contains any mixed", + haveClause: sqlbuilder.StaticJSONBArrayContainsAny(tags, []any{2.5, "b"}), + want: []int64{1, 2}, + }, + { + name: "contains any of nothing", + haveClause: sqlbuilder.StaticJSONBArrayContainsAny(tags, []string{}), + }, + { + name: "contains all", + haveClause: sqlbuilder.StaticJSONBArrayContainsAll(tags, []string{"b", "a"}), + want: []int64{1}, + }, + { + name: "contains all duplicated", + haveClause: sqlbuilder.StaticJSONBArrayContainsAll(tags, []string{"a", "a"}), + want: []int64{1}, + }, + { + name: "contains all numbers", + haveClause: sqlbuilder.StaticJSONBArrayContainsAll(tags, []float64{1, 2.5}), + want: []int64{2}, + }, + { + name: "contains all numeric string", + haveClause: sqlbuilder.StaticJSONBArrayContainsAll(tags, []string{"1"}), + want: []int64{3}, + }, + { + name: "contains all of nothing", + haveClause: sqlbuilder.StaticJSONBArrayContainsAll(tags, []string{}), + want: []int64{1, 2, 3, 5, 7}, + }, + { + name: "object contains string", + haveClause: sqlbuilder.StaticJSONBObjectContains( + data, + map[string]any{"status": "active"}, + ), + want: []int64{1, 3}, + }, + { + name: "object contains number", + haveClause: sqlbuilder.StaticJSONBObjectContains( + data, + map[string]any{"tier": 2}, + ), + want: []int64{1, 2}, + }, + { + name: "object contains nested object", + haveClause: sqlbuilder.StaticJSONBObjectContains( + data, + map[string]any{jsonbAddressKey: map[string]any{"country": "FR"}}, + ), + want: []int64{1, 7}, + }, + { + name: "object contains empty object", + haveClause: sqlbuilder.StaticJSONBObjectContains( + data, + map[string]any{jsonbAddressKey: map[string]any{}}, + ), + want: []int64{1, 2, 7}, + }, + { + name: "object contains array", + haveClause: sqlbuilder.StaticJSONBObjectContains( + data, + map[string]any{"labels": []string{"x"}}, + ), + want: []int64{1, 7}, + }, + { + name: "object contains nothing", + haveClause: sqlbuilder.StaticJSONBObjectContains(data, map[string]any{}), + want: []int64{1, 2, 3, 5, 7}, + }, + { + name: "object contains many", + haveClause: sqlbuilder.StaticJSONBObjectContains( + data, + map[string]any{ + "status": "active", + jsonbAddressKey: map[string]any{"country": "FR"}, + "labels": []string{"y", "x"}, + }, + ), + want: []int64{1}, + }, + } { + t.Run(tt.name, func(t *testing.T) { + got, err := selectJSONBRowIDs(ctx, db, tt.haveClause) + + require.NoError(t, err) + assert.Equal(t, tt.want, got) + }) + } + + t.Run("in a transaction", func(t *testing.T) { + var got []int64 + + err := sql.ExecuteTx( + ctx, + db, + sql.TxOptions{}, + func(q sql.Queryer) error { + var err error + + got, err = selectJSONBRowIDs( + ctx, + q, + sqlbuilder.StaticJSONBArrayContainsAll(tags, []string{"a"}), + ) + + return err + }, + ) + + require.NoError(t, err) + assert.Equal(t, []int64{1}, got) + }) + }) +} + +func selectJSONBRowIDs(ctx context.Context, q sql.Queryer, pc sqlbuilder.PredicateClause) ([]int64, error) { + qb := sqlbuilder.QueryBuilder{Queryer: q} + + cur, err := qb.PrepareSelect( + sqlbuilder.SelectStatement{ + Table: "jsonb_rows", + SelectClauses: []sqlbuilder.Marker{sqlbuilder.Column("id")}, + WhereClause: pc, + OrderByClauses: []sqlbuilder.OrderByClause{{Field: sqlbuilder.Column("id")}}, + }, + ).Query(ctx, nil) + + if err != nil { + return nil, err + } + + var ids []int64 + + err = sqlbuilder.ScrollCursor( + cur, + func(sc sqlbuilder.Scanner) error { + var id int64 + + if err := sc.Scan(map[string]any{"id": &id}); err != nil { + return err + } + + ids = append(ids, id) + + return nil + }, + ) + + return ids, err +} From 3be4739eb0a69cf5bf9341c383cdf8c8dfb96956 Mon Sep 17 00:00:00 2001 From: Alexis Montagne Date: Mon, 26 Oct 2026 07:34:15 -0700 Subject: [PATCH 5/6] x/sqlbuilder: support objects and arrays in JSONB predicates Columns holding arrays of objects, such as the bound entities of a conversation, could not be filtered as the predicates only accepted strings and numbers. On SQLite every wanted object or array is now matched with an iteration of its own, as @> does on PostgreSQL. --- x/sqlbuilder/jsonb_predicate_clause.go | 381 ++++++++++++-------- x/sqlbuilder/jsonb_predicate_clause_test.go | 144 +++++++- 2 files changed, 365 insertions(+), 160 deletions(-) diff --git a/x/sqlbuilder/jsonb_predicate_clause.go b/x/sqlbuilder/jsonb_predicate_clause.go index a3d6f49..29e7016 100644 --- a/x/sqlbuilder/jsonb_predicate_clause.go +++ b/x/sqlbuilder/jsonb_predicate_clause.go @@ -24,28 +24,29 @@ const ( ) var ( - errInvalidJSONBValue = errors.New("JSONB predicates only support strings and numbers") + errInvalidJSONBValue = errors.New("JSONB predicates do not support booleans and nulls") errInvalidJSONBKey = errors.New("SQLite does not support object keys holding a double quote") ) // JSONBArrayContainsAny matches the rows whose column, a JSONB array, holds -// at least one of the elements of the slice bound to the marker. The -// elements have to be strings or numbers. +// at least one of the elements of the slice bound to the marker. An object +// or array element matches the elements containing it, as the @> operator +// of PostgreSQL does. The elements can not hold booleans and nulls. func JSONBArrayContainsAny(m Marker) PredicateClause { return &basicClause{m: m, fn: writeJSONBArrayContainsAny} } // JSONBArrayContainsAll matches the rows whose column, a JSONB array, holds -// every element of the slice bound to the marker. The elements have to be -// strings or numbers. +// every element of the slice bound to the marker. An object or array element +// matches the elements containing it, as the @> operator of PostgreSQL does. +// The elements can not hold booleans and nulls. func JSONBArrayContainsAll(m Marker) PredicateClause { return &basicClause{m: m, fn: writeJSONBArrayContainsAll} } // JSONBObjectContains matches the rows whose column, a JSONB object, // contains the object bound to the marker, as the @> operator of -// PostgreSQL does. The object can only hold strings, numbers, objects and -// arrays of strings and numbers. +// PostgreSQL does. The object can not hold booleans and nulls. func JSONBObjectContains(m Marker) PredicateClause { return &basicClause{m: m, fn: writeJSONBObjectContains} } @@ -76,27 +77,22 @@ func writeJSONBArrayContainsAny(w QueryWriter, vv any, k string) error { } if w.Driver() == sqlite3Driver { - want, err := encodeJSONB(es) + sc := sqliteContains{w: w} + + cs, err := sc.containsAny(k, es) if err != nil { return err } - _, err = fmt.Fprintf( - w, - "(json_type(%s) = 'array' AND EXISTS (SELECT 1 FROM json_each(%s) "+ - "AS jsonb_elem, json_each(%s) AS jsonb_want WHERE %s))", - k, - k, - w.RedeemVariable(want), - sqliteJSONBElementMatch, - ) + _, err = fmt.Fprintf(w, "(json_type(%s) = 'array' AND %s)", k, cs) return errors.WithStack(err) } // A disjunction of containments, unlike the ?| operator, matches - // numbers and can use both the jsonb_ops and jsonb_path_ops GIN indexes. + // numbers and objects and can use both the jsonb_ops and jsonb_path_ops + // GIN indexes. cs := make([]string, len(es)) for i, e := range es { @@ -121,19 +117,7 @@ func writeJSONBArrayContainsAll(w QueryWriter, vv any, k string) error { return err } - if w.Driver() == sqlite3Driver { - c, err := sqliteArrayContainsAll(w, k, "", es) - - if err != nil { - return err - } - - _, err = fmt.Fprintf(w, "(%s)", c) - - return errors.WithStack(err) - } - - return writePostgresContains(w, k, es) + return writeJSONBContains(w, k, es) } func writeJSONBObjectContains(w QueryWriter, vv any, k string) error { @@ -143,27 +127,29 @@ func writeJSONBObjectContains(w QueryWriter, vv any, k string) error { return err } - obj, ok := d.(map[string]any) - - if !ok { + if _, ok := d.(map[string]any); !ok { return errors.Wrapf(errInvalidJSONBValue, "%T is not an object", vv) } - if err := checkJSONBObject(obj); err != nil { - return err - } + return writeJSONBContains(w, k, d) +} +func writeJSONBContains(w QueryWriter, k string, want any) error { if w.Driver() != sqlite3Driver { - return writePostgresContains(w, k, obj) + v, err := encodeJSONB(want) + + if err != nil { + return err + } + + _, err = fmt.Fprintf(w, "%s @> %s::jsonb", k, w.RedeemVariable(v)) + + return errors.WithStack(err) } - cs, err := sqliteObjectContains( - w, - k, - "$", - obj, - []string{fmt.Sprintf("json_type(%s) = 'object'", k)}, - ) + sc := sqliteContains{w: w} + + cs, err := sc.contains(k, "", want) if err != nil { return err @@ -174,21 +160,55 @@ func writeJSONBObjectContains(w QueryWriter, vv any, k string) error { return errors.WithStack(err) } -func writePostgresContains(w QueryWriter, k string, v any) error { - want, err := encodeJSONB(v) +// sqliteContains writes the SQLite conditions matching a JSON document +// containing a wanted value, as the @> operator of PostgreSQL does. +type sqliteContains struct { + w QueryWriter - if err != nil { - return err + // items is the number of aliases given to the iterations of the arrays + // holding objects or arrays, each one is unique so that the nested + // iterations never shadow each other. + items int +} + +// contains returns the conditions matching when the JSON value at path in +// doc, or doc itself when path is empty, contains want. +func (sc *sqliteContains) contains(doc, path string, want any) ([]string, error) { + switch want := want.(type) { + case map[string]any: + return sc.containsObject(doc, path, want) + case []any: + return sc.containsArray(doc, path, want) + } + + if path == "" { + path = "$" } - _, err = fmt.Fprintf(w, "%s @> %s::jsonb", k, w.RedeemVariable(want)) + var ( + typ = sc.args(doc, path) + value = sc.args(doc, path) + ) - return errors.WithStack(err) + return []string{ + fmt.Sprintf( + "json_type(%s) IN ('integer', 'real', 'text') AND json_extract(%s) = %s", + typ, + value, + sc.w.RedeemVariable(sqliteScalar(want)), + ), + }, nil } -// sqliteObjectContains appends to cs the conditions matching when the JSON -// object at path in k contains obj: one per leaf of obj. -func sqliteObjectContains(w QueryWriter, k, path string, obj map[string]any, cs []string) ([]string, error) { +func (sc *sqliteContains) containsObject(doc, path string, obj map[string]any) ([]string, error) { + var cs []string + + // The type of a non empty object below the root is checked by the + // conditions on its keys, which never match a value of another type. + if path == "" || len(obj) == 0 { + cs = append(cs, fmt.Sprintf("json_type(%s) = 'object'", sc.args(doc, path))) + } + keys := make([]string, 0, len(obj)) for key := range obj { @@ -202,92 +222,172 @@ func sqliteObjectContains(w QueryWriter, k, path string, obj map[string]any, cs return nil, errors.Wrapf(errInvalidJSONBKey, "%q", key) } - p := fmt.Sprintf(`%s."%s"`, path, key) + parent := path - switch v := obj[key].(type) { - case map[string]any: - if len(v) > 0 { - var err error + if parent == "" { + parent = "$" + } - cs, err = sqliteObjectContains(w, k, p, v, cs) + kcs, err := sc.contains(doc, fmt.Sprintf(`%s."%s"`, parent, key), obj[key]) - if err != nil { - return nil, err - } + if err != nil { + return nil, err + } - continue - } + cs = append(cs, kcs...) + } - cs = append( - cs, - fmt.Sprintf("json_type(%s, %s) = 'object'", k, w.RedeemVariable(p)), - ) - case []any: - c, err := sqliteArrayContainsAll(w, k, p, v) + return cs, nil +} - if err != nil { - return nil, err - } +// containsArray matches the scalars of es with a single iteration of the +// array, counting the ones it holds, and every object or array of es with +// an iteration of its own. +func (sc *sqliteContains) containsArray(doc, path string, es []any) ([]string, error) { + cs := []string{fmt.Sprintf("json_type(%s) = 'array'", sc.args(doc, path))} - cs = append(cs, c) - default: - var ( - typ = w.RedeemVariable(p) - value = w.RedeemVariable(p) - ) - - cs = append( - cs, - fmt.Sprintf( - "json_type(%s, %s) IN ('integer', 'real', 'text') AND "+ - "json_extract(%s, %s) = %s", - k, - typ, - k, - value, - w.RedeemVariable(sqliteScalar(v)), - ), - ) + scalars, others := splitJSONBScalars(es) + + if len(scalars) > 0 { + want, err := encodeJSONB(scalars) + + if err != nil { + return nil, err } + + elem := sc.args(doc, path) + + cs = append( + cs, + fmt.Sprintf( + "(SELECT count(DISTINCT jsonb_want.key) FROM json_each(%s) AS "+ + "jsonb_elem, json_each(%s) AS jsonb_want WHERE %s) = %d", + elem, + sc.w.RedeemVariable(want), + sqliteJSONBElementMatch, + len(scalars), + ), + ) + } + + for _, e := range others { + c, err := sc.existsItem(doc, path, e) + + if err != nil { + return nil, err + } + + cs = append(cs, c) } return cs, nil } -// sqliteArrayContainsAll returns the condition matching when the JSON array -// at path in k, or k itself when path is empty, holds every element of es. -// The column is iterated first so that its name is never resolved to one -// of the columns of the iteration of the wanted values. -func sqliteArrayContainsAll(w QueryWriter, k, path string, es []any) (string, error) { - want, err := encodeJSONB(es) +// containsAny returns the condition matching when the JSON array doc holds +// one of the elements of es. +func (sc *sqliteContains) containsAny(doc string, es []any) (string, error) { + var ( + cs []string - if err != nil { - return "", err + scalars, others = splitJSONBScalars(es) + ) + + if len(scalars) > 0 { + want, err := encodeJSONB(scalars) + + if err != nil { + return "", err + } + + cs = append( + cs, + fmt.Sprintf( + "EXISTS (SELECT 1 FROM json_each(%s) AS jsonb_elem, json_each(%s) "+ + "AS jsonb_want WHERE %s)", + doc, + sc.w.RedeemVariable(want), + sqliteJSONBElementMatch, + ), + ) } + for _, e := range others { + c, err := sc.existsItem(doc, "", e) + + if err != nil { + return "", err + } + + cs = append(cs, c) + } + + if len(cs) == 1 { + return cs[0], nil + } + + return "(" + strings.Join(cs, " OR ") + ")", nil +} + +// existsItem returns the condition matching when the JSON array at path in +// doc holds an element containing want, an object or an array. +func (sc *sqliteContains) existsItem(doc, path string, want any) (string, error) { + sc.items++ + var ( - typ = sqliteJSONArgs(w, k, path) - elem = sqliteJSONArgs(w, k, path) + array = sc.args(doc, path) + alias = fmt.Sprintf("jsonb_item_%d", sc.items) + typ = "array" ) + if _, ok := want.(map[string]any); ok { + typ = "object" + } + + // json_each returns the strings of an array without their quotes, which + // the JSON functions reject as malformed: the other elements are only + // read as NULL, as the order the terms of a WHERE clause are evaluated + // in is not guaranteed. + cs, err := sc.contains( + fmt.Sprintf("(CASE WHEN %[1]s.type = '%[2]s' THEN %[1]s.value END)", alias, typ), + "", + want, + ) + + if err != nil { + return "", err + } + return fmt.Sprintf( - "json_type(%s) = 'array' AND (SELECT count(DISTINCT jsonb_want.key) "+ - "FROM json_each(%s) AS jsonb_elem, json_each(%s) AS jsonb_want "+ - "WHERE %s) = %d", - typ, - elem, - w.RedeemVariable(want), - sqliteJSONBElementMatch, - len(es), + "EXISTS (SELECT 1 FROM json_each(%s) AS %s WHERE %s)", + array, + alias, + strings.Join(cs, " AND "), ), nil } -func sqliteJSONArgs(w QueryWriter, k, path string) string { +// args returns the arguments of a JSON function reading the value at path +// in doc, or doc itself when path is empty. +func (sc *sqliteContains) args(doc, path string) string { if path == "" { - return k + return doc + } + + return doc + ", " + sc.w.RedeemVariable(path) +} + +func splitJSONBScalars(es []any) ([]any, []any) { + var scalars, others []any + + for _, e := range es { + switch e.(type) { + case map[string]any, []any: + others = append(others, e) + default: + scalars = append(scalars, e) + } } - return k + ", " + w.RedeemVariable(path) + return scalars, others } // sqliteScalar converts a string or a JSON number to the value json_extract @@ -308,6 +408,8 @@ func sqliteScalar(v any) any { return f } +// decodeJSONB returns the JSON representation of v, decoded with its +// numbers as json.Number, after checking it holds no booleans and nulls. func decodeJSONB(v any) (any, error) { buf, err := json.Marshal(v) @@ -327,7 +429,11 @@ func decodeJSONB(v any) (any, error) { return nil, errors.Wrap(err, "cant decode the JSONB value") } - return res, nil + if res == nil { + return nil, nil + } + + return res, checkJSONB(res) } func decodeJSONBArray(v any) ([]any, error) { @@ -341,50 +447,35 @@ func decodeJSONBArray(v any) ([]any, error) { case nil: return []any{}, nil case []any: - for _, e := range d { - if err := checkJSONBScalar(e); err != nil { - return nil, err - } - } - return d, nil } return nil, errors.Wrapf(errInvalidJSONBValue, "%T is not an array", v) } -func checkJSONBScalar(v any) error { - switch v.(type) { +func checkJSONB(v any) error { + switch v := v.(type) { case string, json.Number: return nil - } - - return errors.Wrapf(errInvalidJSONBValue, "unexpected %T", v) -} - -func checkJSONBObject(obj map[string]any) error { - for _, v := range obj { - var err error - - switch v := v.(type) { - case map[string]any: - err = checkJSONBObject(v) - case []any: - for _, e := range v { - if err = checkJSONBScalar(e); err != nil { - break - } + case map[string]any: + for _, e := range v { + if err := checkJSONB(e); err != nil { + return err } - default: - err = checkJSONBScalar(v) } - if err != nil { - return err + return nil + case []any: + for _, e := range v { + if err := checkJSONB(e); err != nil { + return err + } } + + return nil } - return nil + return errors.Wrapf(errInvalidJSONBValue, "unexpected %T", v) } func encodeJSONB(v any) (string, error) { diff --git a/x/sqlbuilder/jsonb_predicate_clause_test.go b/x/sqlbuilder/jsonb_predicate_clause_test.go index 7364fc6..9c0ce8a 100644 --- a/x/sqlbuilder/jsonb_predicate_clause_test.go +++ b/x/sqlbuilder/jsonb_predicate_clause_test.go @@ -2,6 +2,7 @@ package sqlbuilder import ( "context" + "fmt" "testing" "github.com/stretchr/testify/assert" @@ -11,12 +12,53 @@ import ( "github.com/upfluence/sql/backend/static" ) -const postgresDriver = "postgres" +const ( + postgresDriver = "postgres" + + skuKey = "sku" + skuPath = `$."sku"` + idPath = `$."id"` + typePath = `$."type"` +) + +type testBoundEntity struct { + Type string `json:"type"` + ID int64 `json:"id"` +} + +// sqliteBoundEntityMatch returns the condition matching an element of the +// entities column containing a testBoundEntity, its arguments being +// numbered from first. +func sqliteBoundEntityMatch(alias string, first int) string { + return fmt.Sprintf( + "EXISTS (SELECT 1 FROM json_each(entities) AS %[1]s WHERE "+ + "json_type((CASE WHEN %[1]s.type = 'object' THEN %[1]s.value END)) = 'object' AND "+ + "json_type((CASE WHEN %[1]s.type = 'object' THEN %[1]s.value END), $%[2]d) IN ('integer', 'real', 'text') AND "+ + "json_extract((CASE WHEN %[1]s.type = 'object' THEN %[1]s.value END), $%[3]d) = $%[4]d AND "+ + "json_type((CASE WHEN %[1]s.type = 'object' THEN %[1]s.value END), $%[5]d) IN ('integer', 'real', 'text') AND "+ + "json_extract((CASE WHEN %[1]s.type = 'object' THEN %[1]s.value END), $%[6]d) = $%[7]d)", + alias, + first, + first+1, + first+2, + first+3, + first+4, + first+5, + ) +} + +func sqliteBoundEntityArgs(e testBoundEntity) []any { + return []any{idPath, idPath, e.ID, typePath, typePath, e.Type} +} func TestJSONBPredicateClause(t *testing.T) { var ( - tags = Column("tags") - data = Column("data") + tags = Column("tags") + data = Column("data") + entities = Column("entities") + + campaign = testBoundEntity{Type: "campaign", ID: 12} + contribution = testBoundEntity{Type: "contribution", ID: 14} ) for _, tt := range []struct { @@ -149,6 +191,84 @@ func TestJSONBPredicateClause(t *testing.T) { want: "(json_type(data) = 'object')", wantErr: errtest.NoError(), }, + { + name: "postgres contains any object", + haveDriver: postgresDriver, + haveClause: StaticJSONBArrayContainsAny( + entities, + []testBoundEntity{campaign, contribution}, + ), + want: "(entities @> $1::jsonb OR entities @> $2::jsonb)", + wantArgs: []any{ + `[{"id":12,"type":"campaign"}]`, + `[{"id":14,"type":"contribution"}]`, + }, + wantErr: errtest.NoError(), + }, + { + name: "sqlite contains any object", + haveDriver: sqlite3Driver, + haveClause: StaticJSONBArrayContainsAny( + entities, + []testBoundEntity{campaign, contribution}, + ), + want: "(json_type(entities) = 'array' AND (" + + sqliteBoundEntityMatch("jsonb_item_1", 1) + " OR " + + sqliteBoundEntityMatch("jsonb_item_2", 7) + "))", + wantArgs: append( + sqliteBoundEntityArgs(campaign), + sqliteBoundEntityArgs(contribution)..., + ), + wantErr: errtest.NoError(), + }, + { + name: "sqlite contains all of scalars, objects and arrays", + haveDriver: sqlite3Driver, + haveClause: StaticJSONBArrayContainsAll( + tags, + []any{"a", map[string]any{skuKey: "x"}, []any{1}}, + ), + want: "(json_type(tags) = 'array' AND (SELECT count(DISTINCT " + + "jsonb_want.key) FROM json_each(tags) AS jsonb_elem, " + + "json_each($1) AS jsonb_want WHERE " + sqliteJSONBElementMatch + + ") = 1 AND EXISTS (SELECT 1 FROM json_each(tags) AS jsonb_item_1 " + + "WHERE json_type((CASE WHEN jsonb_item_1.type = 'object' THEN jsonb_item_1.value END)) = 'object' AND " + + "json_type((CASE WHEN jsonb_item_1.type = 'object' THEN jsonb_item_1.value END), $2) IN ('integer', 'real', 'text') " + + "AND json_extract((CASE WHEN jsonb_item_1.type = 'object' THEN jsonb_item_1.value END), $3) = $4) AND EXISTS " + + "(SELECT 1 FROM json_each(tags) AS jsonb_item_2 WHERE " + + "json_type((CASE WHEN jsonb_item_2.type = 'array' THEN jsonb_item_2.value END)) = 'array' AND (SELECT " + + "count(DISTINCT jsonb_want.key) FROM json_each((CASE WHEN jsonb_item_2.type = 'array' THEN jsonb_item_2.value END)) " + + "AS jsonb_elem, json_each($5) AS jsonb_want WHERE " + + sqliteJSONBElementMatch + ") = 1))", + wantArgs: []any{`["a"]`, skuPath, skuPath, "x", `[1]`}, + wantErr: errtest.NoError(), + }, + { + name: "postgres object holding an array of objects", + haveDriver: postgresDriver, + haveClause: StaticJSONBObjectContains( + data, + map[string]any{"items": []any{map[string]any{skuKey: "x"}}}, + ), + want: "data @> $1::jsonb", + wantArgs: []any{`{"items":[{"sku":"x"}]}`}, + wantErr: errtest.NoError(), + }, + { + name: "sqlite object holding an array of objects", + haveDriver: sqlite3Driver, + haveClause: StaticJSONBObjectContains( + data, + map[string]any{"items": []any{map[string]any{skuKey: "x"}}}, + ), + want: "(json_type(data) = 'object' AND json_type(data, $1) = 'array' " + + "AND EXISTS (SELECT 1 FROM json_each(data, $2) AS jsonb_item_1 " + + "WHERE json_type((CASE WHEN jsonb_item_1.type = 'object' THEN jsonb_item_1.value END)) = 'object' AND " + + "json_type((CASE WHEN jsonb_item_1.type = 'object' THEN jsonb_item_1.value END), $3) IN ('integer', 'real', 'text') " + + "AND json_extract((CASE WHEN jsonb_item_1.type = 'object' THEN jsonb_item_1.value END), $4) = $5))", + wantArgs: []any{`$."items"`, `$."items"`, skuPath, skuPath, "x"}, + wantErr: errtest.NoError(), + }, { name: "boolean element", haveDriver: postgresDriver, @@ -162,10 +282,13 @@ func TestJSONBPredicateClause(t *testing.T) { wantErr: errtest.ErrorCause(errInvalidJSONBValue), }, { - name: "array of objects", + name: "object holding a boolean in an array", haveDriver: postgresDriver, - haveClause: StaticJSONBArrayContainsAll(tags, []any{map[string]any{}}), - wantErr: errtest.ErrorCause(errInvalidJSONBValue), + haveClause: StaticJSONBArrayContainsAny( + entities, + []any{map[string]any{"active": true}}, + ), + wantErr: errtest.ErrorCause(errInvalidJSONBValue), }, { name: "not an array", @@ -173,15 +296,6 @@ func TestJSONBPredicateClause(t *testing.T) { haveClause: StaticJSONBArrayContainsAny(tags, "a"), wantErr: errtest.ErrorCause(errInvalidJSONBValue), }, - { - name: "object holding an array of objects", - haveDriver: postgresDriver, - haveClause: StaticJSONBObjectContains( - data, - map[string]any{"items": []any{map[string]any{"sku": "x"}}}, - ), - wantErr: errtest.ErrorCause(errInvalidJSONBValue), - }, { name: "object holding a boolean", haveDriver: sqlite3Driver, From 6a4cdb3953db7c8b6d76245365674ac8cf6d72d5 Mon Sep 17 00:00:00 2001 From: Alexis Montagne Date: Mon, 26 Oct 2026 07:34:16 -0700 Subject: [PATCH 6/6] integration: run the JSONB predicates on arrays of objects --- integration/jsonb_test.go | 113 +++++++++++++++++++++++++++++++------- 1 file changed, 92 insertions(+), 21 deletions(-) diff --git a/integration/jsonb_test.go b/integration/jsonb_test.go index f0ee6de..b8c73c1 100644 --- a/integration/jsonb_test.go +++ b/integration/jsonb_test.go @@ -15,11 +15,16 @@ import ( const jsonbAddressKey = "address" +type jsonbBoundEntity struct { + Type string `json:"type"` + ID int64 `json:"id"` +} + func jsonbMigrator(db sql.DB) migration.Migrator { - up := "CREATE TABLE jsonb_rows(id INTEGER PRIMARY KEY, tags TEXT, data TEXT)" + up := "CREATE TABLE jsonb_rows(id INTEGER PRIMARY KEY, tags TEXT, data TEXT, entities TEXT)" if db.Driver() == postgresDriver { - up = "CREATE TABLE jsonb_rows(id INTEGER PRIMARY KEY, tags JSONB, data JSONB)" + up = "CREATE TABLE jsonb_rows(id INTEGER PRIMARY KEY, tags JSONB, data JSONB, entities JSONB)" } return migration.NewMigrator( @@ -35,47 +40,62 @@ func TestJSONBPredicateClause(t *testing.T) { ctx := context.Background() for _, row := range []struct { - id int64 - tags, data any + id int64 + tags, data, entities any }{ { - id: 1, - tags: `["a", "b"]`, - data: `{"status": "active", "tier": 2, "address": {"country": "FR"}, "labels": ["x", "y"]}`, + id: 1, + tags: `["a", "b"]`, + data: `{"status": "active", "tier": 2, "address": {"country": "FR"}, "labels": ["x", "y"]}`, + entities: `[{"type": "campaign", "id": 12}, {"type": "contribution", "id": 14}]`, + }, + { + id: 2, + tags: `[1, 2.5]`, + data: `{"status": "inactive", "tier": 2.0, "address": {"country": "US"}}`, + entities: `[{"type": "campaign", "id": 13}]`, }, { - id: 2, - tags: `[1, 2.5]`, - data: `{"status": "inactive", "tier": 2.0, "address": {"country": "US"}}`, + id: 3, + tags: `["1", true]`, + data: `{"status": "active", "tier": "2", "labels": "x"}`, + entities: `[{"type": "campaign", "id": 12, "extra": true}]`, }, { - id: 3, - tags: `["1", true]`, - data: `{"status": "active", "tier": "2", "labels": "x"}`, + id: 4, + tags: `{"a": "a"}`, + data: `["status"]`, + entities: `{"type": "campaign", "id": 12}`, }, - {id: 4, tags: `{"a": "a"}`, data: `["status"]`}, - {id: 5, tags: `[]`, data: `{}`}, + {id: 5, tags: `[]`, data: `{}`, entities: `[]`}, {id: 6}, { - id: 7, - tags: `[["a"]]`, - data: `{"address": {"country": "FR", "city": "Paris"}, "labels": ["y", "x", "z"], "flags": true}`, + id: 7, + tags: `[["a"]]`, + data: `{"address": {"country": "FR", "city": "Paris"}, "labels": ["y", "x", "z"], "flags": true, "items": [{"sku": "x", "qty": 1}, {"sku": "y"}]}`, + entities: `[{"type": "contribution", "id": 12}, {"type": "campaign", "id": "12"}]`, }, } { _, err := db.Exec( ctx, - "INSERT INTO jsonb_rows(id, tags, data) VALUES ($1, $2, $3)", + "INSERT INTO jsonb_rows(id, tags, data, entities) VALUES ($1, $2, $3, $4)", row.id, row.tags, row.data, + row.entities, ) require.NoError(t, err) } var ( - tags = sqlbuilder.Column("tags") - data = sqlbuilder.Column("data") + tags = sqlbuilder.Column("tags") + data = sqlbuilder.Column("data") + entities = sqlbuilder.Column("entities") + + campaign12 = jsonbBoundEntity{Type: "campaign", ID: 12} + campaign13 = jsonbBoundEntity{Type: "campaign", ID: 13} + contribution14 = jsonbBoundEntity{Type: "contribution", ID: 14} ) for _, tt := range []struct { @@ -184,6 +204,57 @@ func TestJSONBPredicateClause(t *testing.T) { ), want: []int64{1}, }, + { + name: "object contains array of objects", + haveClause: sqlbuilder.StaticJSONBObjectContains( + data, + map[string]any{"items": []any{map[string]any{"sku": "x"}}}, + ), + want: []int64{7}, + }, + { + name: "contains all nested array", + haveClause: sqlbuilder.StaticJSONBArrayContainsAll(tags, []any{[]string{"a"}}), + want: []int64{7}, + }, + { + name: "contains any object", + haveClause: sqlbuilder.StaticJSONBArrayContainsAny( + entities, + []jsonbBoundEntity{campaign12}, + ), + want: []int64{1, 3}, + }, + { + name: "contains any objects", + haveClause: sqlbuilder.StaticJSONBArrayContainsAny( + entities, + []jsonbBoundEntity{ + campaign13, + contribution14, + }, + ), + want: []int64{1, 2}, + }, + { + name: "contains all objects", + haveClause: sqlbuilder.StaticJSONBArrayContainsAll( + entities, + []jsonbBoundEntity{ + campaign12, + contribution14, + }, + ), + want: []int64{1}, + }, + { + name: "contains all object", + haveClause: sqlbuilder.StaticJSONBArrayContainsAll( + entities, + []jsonbBoundEntity{campaign12}, + ), + want: []int64{1, 3}, + }, } { t.Run(tt.name, func(t *testing.T) { got, err := selectJSONBRowIDs(ctx, db, tt.haveClause)