From 9cee7b1b0f0e975027563a507ca1746bb25b2cb8 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Tue, 14 Dec 2021 16:40:04 -0500 Subject: [PATCH 1/5] Support identifiers with `.` in them --- datafusion/src/sql/planner.rs | 17 +++++++--- datafusion/tests/sql.rs | 61 +++++++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+), 5 deletions(-) diff --git a/datafusion/src/sql/planner.rs b/datafusion/src/sql/planner.rs index 3558d6ca4e231..621c952f98edb 100644 --- a/datafusion/src/sql/planner.rs +++ b/datafusion/src/sql/planner.rs @@ -1062,8 +1062,10 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { } let field = schema.field(field_index - 1); - let col_ident = SQLExpr::Identifier(Ident::new(field.qualified_name())); - self.sql_expr_to_logical_expr(&col_ident, schema)? + Expr::Column(Column { + relation: field.qualifier().cloned(), + name: field.name().clone(), + }) } e => self.sql_expr_to_logical_expr(e, schema)?, }; @@ -1323,9 +1325,14 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { let var_names = vec![id.value.clone()]; Ok(Expr::ScalarVariable(var_names)) } else { - // create a column expression based on raw user input, this column will be - // normalized with qualifer later by the SQL planner. - Ok(col(&id.value)) + // Don't use `col()` here because it will try to + // interpret names with '.' as if they were + // compound indenfiers, but this is not a compound + // identifier. (e.g. it is "foo.bar" not foo.bar) + Ok(Expr::Column(Column { + relation: None, + name: id.value.clone(), + })) } } diff --git a/datafusion/tests/sql.rs b/datafusion/tests/sql.rs index 945bb7ebc2eb8..1505d8c9dadef 100644 --- a/datafusion/tests/sql.rs +++ b/datafusion/tests/sql.rs @@ -5426,6 +5426,67 @@ async fn qualified_table_references() -> Result<()> { Ok(()) } +#[tokio::test] +async fn qualified_table_references_and_fields() -> Result<()> { + let mut ctx = ExecutionContext::new(); + + let c1: StringArray = vec!["foofoo", "foobar", "foobaz"] + .into_iter() + .map(Some) + .collect(); + let c2: Int64Array = vec![1, 2, 3].into_iter().map(Some).collect(); + + let batch = RecordBatch::try_from_iter(vec![ + ("f.c1", Arc::new(c1) as ArrayRef), + // evil -- use the same name as the table + ("test.c2", Arc::new(c2) as ArrayRef), + ])?; + + let table = MemTable::try_new(batch.schema(), vec![vec![batch]])?; + ctx.register_table("test", Arc::new(table))?; + + // referring to the unquoted column is an error + let sql = format!(r#"SELECT f1.c1 from test"#); + let error = ctx.create_logical_plan(&sql).unwrap_err(); + assert_contains!( + error.to_string(), + "No field named 'f1.c1'. Valid fields are 'test.f.c1', 'test.test.c2'" + ); + + // however, enclosing it in double quotes is ok + let sql = format!(r#"SELECT "f.c1" from test"#); + let actual = execute_to_batches(&mut ctx, &sql).await; + let expected = vec![ + "+--------+", + "| f.c1 |", + "+--------+", + "| foofoo |", + "| foobar |", + "| foobaz |", + "+--------+", + ]; + assert_batches_eq!(expected, &actual); + // Works fully qualified too + let sql = format!(r#"SELECT test."f.c1" from test"#); + let actual = execute_to_batches(&mut ctx, &sql).await; + assert_batches_eq!(expected, &actual); + + // check that duplicated table name and column name are ok + let sql = format!(r#"SELECT "test.c2" as expr1, test."test.c2" as expr2 from test"#); + let actual = execute_to_batches(&mut ctx, &sql).await; + let expected = vec![ + "+-------+-------+", + "| expr1 | expr2 |", + "+-------+-------+", + "| 1 | 1 |", + "| 2 | 2 |", + "| 3 | 3 |", + "+-------+-------+", + ]; + assert_batches_eq!(expected, &actual); + Ok(()) +} + #[tokio::test] async fn invalid_qualified_table_references() -> Result<()> { let mut ctx = ExecutionContext::new(); From 712497f268b6a3504e78646fdb47638b9d381905 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Tue, 14 Dec 2021 17:50:54 -0500 Subject: [PATCH 2/5] simplify --- datafusion/src/sql/planner.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/datafusion/src/sql/planner.rs b/datafusion/src/sql/planner.rs index 621c952f98edb..ac6d58033734b 100644 --- a/datafusion/src/sql/planner.rs +++ b/datafusion/src/sql/planner.rs @@ -1062,10 +1062,7 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { } let field = schema.field(field_index - 1); - Expr::Column(Column { - relation: field.qualifier().cloned(), - name: field.name().clone(), - }) + Expr::Column(field.qualified_column()) } e => self.sql_expr_to_logical_expr(e, schema)?, }; From 44f49ad700fa545e0dcd3f089714a60bdf6ddc4c Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Wed, 15 Dec 2021 14:49:57 -0500 Subject: [PATCH 3/5] fix: clippy --- datafusion/tests/sql.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/datafusion/tests/sql.rs b/datafusion/tests/sql.rs index 1505d8c9dadef..2260776e24f38 100644 --- a/datafusion/tests/sql.rs +++ b/datafusion/tests/sql.rs @@ -5446,7 +5446,7 @@ async fn qualified_table_references_and_fields() -> Result<()> { ctx.register_table("test", Arc::new(table))?; // referring to the unquoted column is an error - let sql = format!(r#"SELECT f1.c1 from test"#); + let sql = r#"SELECT f1.c1 from test"#; let error = ctx.create_logical_plan(&sql).unwrap_err(); assert_contains!( error.to_string(), @@ -5454,7 +5454,7 @@ async fn qualified_table_references_and_fields() -> Result<()> { ); // however, enclosing it in double quotes is ok - let sql = format!(r#"SELECT "f.c1" from test"#); + let sql = r#"SELECT "f.c1" from test"#; let actual = execute_to_batches(&mut ctx, &sql).await; let expected = vec![ "+--------+", @@ -5467,12 +5467,12 @@ async fn qualified_table_references_and_fields() -> Result<()> { ]; assert_batches_eq!(expected, &actual); // Works fully qualified too - let sql = format!(r#"SELECT test."f.c1" from test"#); + let sql = r#"SELECT test."f.c1" from test"#; let actual = execute_to_batches(&mut ctx, &sql).await; assert_batches_eq!(expected, &actual); // check that duplicated table name and column name are ok - let sql = format!(r#"SELECT "test.c2" as expr1, test."test.c2" as expr2 from test"#); + let sql = r#"SELECT "test.c2" as expr1, test."test.c2" as expr2 from test"#; let actual = execute_to_batches(&mut ctx, &sql).await; let expected = vec![ "+-------+-------+", From d35c44a64fc26fa5af4b586eeb7ff5c30bcbd7e7 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Wed, 15 Dec 2021 14:50:13 -0500 Subject: [PATCH 4/5] fix: more clippy --- datafusion/tests/sql.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/datafusion/tests/sql.rs b/datafusion/tests/sql.rs index 2260776e24f38..87150e8a5ee4a 100644 --- a/datafusion/tests/sql.rs +++ b/datafusion/tests/sql.rs @@ -5447,7 +5447,7 @@ async fn qualified_table_references_and_fields() -> Result<()> { // referring to the unquoted column is an error let sql = r#"SELECT f1.c1 from test"#; - let error = ctx.create_logical_plan(&sql).unwrap_err(); + let error = ctx.create_logical_plan(sql).unwrap_err(); assert_contains!( error.to_string(), "No field named 'f1.c1'. Valid fields are 'test.f.c1', 'test.test.c2'" @@ -5455,7 +5455,7 @@ async fn qualified_table_references_and_fields() -> Result<()> { // however, enclosing it in double quotes is ok let sql = r#"SELECT "f.c1" from test"#; - let actual = execute_to_batches(&mut ctx, &sql).await; + let actual = execute_to_batches(&mut ctx, sql).await; let expected = vec![ "+--------+", "| f.c1 |", @@ -5468,12 +5468,12 @@ async fn qualified_table_references_and_fields() -> Result<()> { assert_batches_eq!(expected, &actual); // Works fully qualified too let sql = r#"SELECT test."f.c1" from test"#; - let actual = execute_to_batches(&mut ctx, &sql).await; + let actual = execute_to_batches(&mut ctx, sql).await; assert_batches_eq!(expected, &actual); // check that duplicated table name and column name are ok let sql = r#"SELECT "test.c2" as expr1, test."test.c2" as expr2 from test"#; - let actual = execute_to_batches(&mut ctx, &sql).await; + let actual = execute_to_batches(&mut ctx, sql).await; let expected = vec![ "+-------+-------+", "| expr1 | expr2 |", From 3e4418c67a8b8684c262b48fc635cebb5da9af9b Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Wed, 15 Dec 2021 14:54:46 -0500 Subject: [PATCH 5/5] Add test for "...." --- datafusion/tests/sql.rs | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/datafusion/tests/sql.rs b/datafusion/tests/sql.rs index 87150e8a5ee4a..481186b5e12e5 100644 --- a/datafusion/tests/sql.rs +++ b/datafusion/tests/sql.rs @@ -5435,11 +5435,14 @@ async fn qualified_table_references_and_fields() -> Result<()> { .map(Some) .collect(); let c2: Int64Array = vec![1, 2, 3].into_iter().map(Some).collect(); + let c3: Int64Array = vec![10, 20, 30].into_iter().map(Some).collect(); let batch = RecordBatch::try_from_iter(vec![ ("f.c1", Arc::new(c1) as ArrayRef), // evil -- use the same name as the table ("test.c2", Arc::new(c2) as ArrayRef), + // more evil still + ("....", Arc::new(c3) as ArrayRef), ])?; let table = MemTable::try_new(batch.schema(), vec![vec![batch]])?; @@ -5484,6 +5487,22 @@ async fn qualified_table_references_and_fields() -> Result<()> { "+-------+-------+", ]; assert_batches_eq!(expected, &actual); + + // check that '....' is also an ok column name (in the sense that + // datafusion should run the query, not that someone should write + // this + let sql = r#"SELECT "....", "...." as c3 from test order by "....""#; + let actual = execute_to_batches(&mut ctx, sql).await; + let expected = vec![ + "+------+----+", + "| .... | c3 |", + "+------+----+", + "| 10 | 10 |", + "| 20 | 20 |", + "| 30 | 30 |", + "+------+----+", + ]; + assert_batches_eq!(expected, &actual); Ok(()) }