From df617c914e09f1443b4ccea76f9ad4218bc6aa86 Mon Sep 17 00:00:00 2001 From: "Heres, Daniel" Date: Tue, 4 May 2021 22:13:49 +0200 Subject: [PATCH 01/10] Implement select distinct --- datafusion/src/sql/planner.rs | 14 +++++++-- datafusion/tests/sql.rs | 55 +++++++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 3 deletions(-) diff --git a/datafusion/src/sql/planner.rs b/datafusion/src/sql/planner.rs index a40d0becdcb4b..83c715e418fa6 100644 --- a/datafusion/src/sql/planner.rs +++ b/datafusion/src/sql/planner.rs @@ -17,9 +17,9 @@ //! SQL Query Planner (produces logical plan from SQL AST) -use std::convert::TryInto; use std::str::FromStr; use std::sync::Arc; +use std::{convert::TryInto, vec}; use crate::catalog::TableReference; use crate::datasource::TableProvider; @@ -621,7 +621,7 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { plan }; - self.project(&plan, select_exprs_post_aggr, false) + self.project(&plan, select_exprs_post_aggr, false, select.distinct) } /// Returns the `Expr`'s corresponding to a SQL query's SELECT expressions. @@ -654,9 +654,17 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { input: &LogicalPlan, expr: Vec, force: bool, + distinct: bool, ) -> Result { self.validate_schema_satisfies_exprs(&input.schema(), &expr)?; - let plan = LogicalPlanBuilder::from(input).project(expr)?.build()?; + let plan = if distinct { + LogicalPlanBuilder::from(input) + .aggregate(expr.clone(), vec![])? + .project(expr)? + .build()? + } else { + LogicalPlanBuilder::from(input).project(expr)?.build()? + }; let project = force || match input { diff --git a/datafusion/tests/sql.rs b/datafusion/tests/sql.rs index 716929405c3a1..858e53d9493df 100644 --- a/datafusion/tests/sql.rs +++ b/datafusion/tests/sql.rs @@ -372,6 +372,61 @@ async fn csv_query_group_by_float32() -> Result<()> { Ok(()) } +#[tokio::test] +async fn select_distinct_all() -> Result<()> { + let mut ctx = ExecutionContext::new(); + register_aggregate_simple_csv(&mut ctx)?; + + let sql = "SELECT DISTINCT * FROM aggregate_simple"; + let mut actual = execute(&mut ctx, sql).await; + actual.sort(); + + let mut dedup = actual.clone(); + dedup.dedup(); + + assert_eq!(actual, dedup); + + Ok(()) +} + +#[tokio::test] +async fn select_distinct_simple() -> Result<()> { + let mut ctx = ExecutionContext::new(); + register_aggregate_simple_csv(&mut ctx)?; + + let sql = "SELECT DISTINCT c1 FROM aggregate_simple order by c1"; + let actual = execute(&mut ctx, sql).await; + + let expected = vec![ + vec!["0.00001"], + vec!["0.00002"], + vec!["0.00003"], + vec!["0.00004"], + vec!["0.00005"], + ]; + assert_eq!(actual, expected); + + let sql = "SELECT DISTINCT c1, c2 FROM aggregate_simple order by c1"; + let actual = execute(&mut ctx, sql).await; + + let expected = vec![ + vec!["0.00001", "0.000000000001"], + vec!["0.00002", "0.000000000002"], + vec!["0.00003", "0.000000000003"], + vec!["0.00004", "0.000000000004"], + vec!["0.00005", "0.000000000005"], + ]; + assert_eq!(actual, expected); + + let sql = "SELECT distinct c3 FROM aggregate_simple order by c3"; + let actual = execute(&mut ctx, sql).await; + + let expected = vec![vec!["false"], vec!["true"]]; + assert_eq!(actual, expected); + + Ok(()) +} + #[tokio::test] async fn csv_query_group_by_float64() -> Result<()> { let mut ctx = ExecutionContext::new(); From 46e0eb9feaa7cb576361d4a11fc72d9da7fac189 Mon Sep 17 00:00:00 2001 From: "Heres, Daniel" Date: Tue, 4 May 2021 22:29:32 +0200 Subject: [PATCH 02/10] Simplify implementation --- datafusion/src/sql/planner.rs | 24 ++++++++++++------------ datafusion/tests/sql.rs | 4 ++++ 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/datafusion/src/sql/planner.rs b/datafusion/src/sql/planner.rs index 83c715e418fa6..37459e990256b 100644 --- a/datafusion/src/sql/planner.rs +++ b/datafusion/src/sql/planner.rs @@ -621,7 +621,7 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { plan }; - self.project(&plan, select_exprs_post_aggr, false, select.distinct) + self.project(&plan, select_exprs_post_aggr, select.distinct) } /// Returns the `Expr`'s corresponding to a SQL query's SELECT expressions. @@ -653,24 +653,24 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { &self, input: &LogicalPlan, expr: Vec, - force: bool, distinct: bool, ) -> Result { self.validate_schema_satisfies_exprs(&input.schema(), &expr)?; - let plan = if distinct { - LogicalPlanBuilder::from(input) + + let input = if distinct { + return LogicalPlanBuilder::from(input) .aggregate(expr.clone(), vec![])? - .project(expr)? - .build()? + .build(); } else { - LogicalPlanBuilder::from(input).project(expr)?.build()? + input }; - let project = force - || match input { - LogicalPlan::TableScan { .. } => true, - _ => plan.schema().fields() != input.schema().fields(), - }; + let plan = LogicalPlanBuilder::from(input).project(expr)?.build()?; + + let project = match input { + LogicalPlan::TableScan { .. } => true, + _ => plan.schema().fields() != input.schema().fields(), + }; if project { Ok(plan) diff --git a/datafusion/tests/sql.rs b/datafusion/tests/sql.rs index 858e53d9493df..612a1307021ae 100644 --- a/datafusion/tests/sql.rs +++ b/datafusion/tests/sql.rs @@ -424,6 +424,10 @@ async fn select_distinct_simple() -> Result<()> { let expected = vec![vec!["false"], vec!["true"]]; assert_eq!(actual, expected); + let sql = "SELECT distinct c1+c2 as a FROM aggregate_simple"; + let actual = execute(&mut ctx, sql).await; + + assert_eq!(actual.len(), 5); Ok(()) } From ada560bbb5b19cb189c7143c9f45639463561d39 Mon Sep 17 00:00:00 2001 From: "Heres, Daniel" Date: Tue, 4 May 2021 22:33:14 +0200 Subject: [PATCH 03/10] Move out of project fn --- datafusion/src/sql/planner.rs | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/datafusion/src/sql/planner.rs b/datafusion/src/sql/planner.rs index 37459e990256b..25fb940e8197d 100644 --- a/datafusion/src/sql/planner.rs +++ b/datafusion/src/sql/planner.rs @@ -621,6 +621,14 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { plan }; + let plan = if select.distinct { + return LogicalPlanBuilder::from(&plan) + .aggregate(select_exprs_post_aggr.clone(), vec![])? + .build(); + } else { + plan + }; + self.project(&plan, select_exprs_post_aggr, select.distinct) } @@ -649,22 +657,9 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { /// necessary, i.e., when the input fields are different than the /// projection. Note that if the input fields are the same, but out of /// order, the projection will be applied. - fn project( - &self, - input: &LogicalPlan, - expr: Vec, - distinct: bool, - ) -> Result { + fn project(&self, input: &LogicalPlan, expr: Vec) -> Result { self.validate_schema_satisfies_exprs(&input.schema(), &expr)?; - let input = if distinct { - return LogicalPlanBuilder::from(input) - .aggregate(expr.clone(), vec![])? - .build(); - } else { - input - }; - let plan = LogicalPlanBuilder::from(input).project(expr)?.build()?; let project = match input { From a30dd40dddb1d161e6148c1a0f11f4e11b1d20c4 Mon Sep 17 00:00:00 2001 From: "Heres, Daniel" Date: Tue, 4 May 2021 22:34:22 +0200 Subject: [PATCH 04/10] Fix call --- datafusion/src/sql/planner.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datafusion/src/sql/planner.rs b/datafusion/src/sql/planner.rs index 25fb940e8197d..a53401b105d12 100644 --- a/datafusion/src/sql/planner.rs +++ b/datafusion/src/sql/planner.rs @@ -629,7 +629,7 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { plan }; - self.project(&plan, select_exprs_post_aggr, select.distinct) + self.project(&plan, select_exprs_post_aggr) } /// Returns the `Expr`'s corresponding to a SQL query's SELECT expressions. From 32a9c95fe5e9f1ddb7cd84f841eea3ce6a6430ad Mon Sep 17 00:00:00 2001 From: "Heres, Daniel" Date: Tue, 4 May 2021 22:35:29 +0200 Subject: [PATCH 05/10] Fix docs --- datafusion/src/sql/planner.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/datafusion/src/sql/planner.rs b/datafusion/src/sql/planner.rs index a53401b105d12..9a75fa75a0fec 100644 --- a/datafusion/src/sql/planner.rs +++ b/datafusion/src/sql/planner.rs @@ -653,8 +653,8 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { /// Wrap a plan in a projection /// - /// If the `force` argument is `false`, the projection is applied only when - /// necessary, i.e., when the input fields are different than the + /// The projection is applied only when necessary, + /// i.e., when the input fields are different than the /// projection. Note that if the input fields are the same, but out of /// order, the projection will be applied. fn project(&self, input: &LogicalPlan, expr: Vec) -> Result { From 217d786086f82b355895fbd02cc0bfdca5b80533 Mon Sep 17 00:00:00 2001 From: "Heres, Daniel" Date: Tue, 4 May 2021 22:47:18 +0200 Subject: [PATCH 06/10] Add to docs --- docs/user-guide/src/sql/select.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/user-guide/src/sql/select.md b/docs/user-guide/src/sql/select.md index 777b4ff61e5d0..a27bfe262cfbc 100644 --- a/docs/user-guide/src/sql/select.md +++ b/docs/user-guide/src/sql/select.md @@ -26,7 +26,7 @@ DataFusion supports the following syntax for queries: [ [WITH](#with-clause) with_query [, ...] ]
-[SELECT](#select-clause) select_expr [, ...]
+[SELECT](#select-clause) [ ALL | DISTINCT ] select_expr [, ...]
[ [FROM](#from-clause) from_item [, ...] ]
[ [WHERE](#where-clause) condition ]
[ [GROUP BY](#group-by-clause) grouping_element [, ...] ]
@@ -55,6 +55,12 @@ Example: SELECT a, b, a + b FROM table ``` +The `DISTINCT` quantifier can be added to make the query return all distinct rows. + + +```sql +SELECT DISTINCT person, age FROM employees +``` # FROM clause From 6da57f9756ab145b0663461b39706b62a8443e1d Mon Sep 17 00:00:00 2001 From: "Heres, Daniel" Date: Tue, 4 May 2021 22:48:35 +0200 Subject: [PATCH 07/10] Add note of ALL support --- docs/user-guide/src/sql/select.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user-guide/src/sql/select.md b/docs/user-guide/src/sql/select.md index a27bfe262cfbc..78d0cb58531d4 100644 --- a/docs/user-guide/src/sql/select.md +++ b/docs/user-guide/src/sql/select.md @@ -56,7 +56,7 @@ SELECT a, b, a + b FROM table ``` The `DISTINCT` quantifier can be added to make the query return all distinct rows. - +By default `ALL` will be used, which returns all the rows. ```sql SELECT DISTINCT person, age FROM employees From 8c15380891e9f54e34323d4f6a1d0558362dee44 Mon Sep 17 00:00:00 2001 From: "Heres, Daniel" Date: Tue, 4 May 2021 22:50:32 +0200 Subject: [PATCH 08/10] Add test for SELECT ALL --- datafusion/tests/sql.rs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/datafusion/tests/sql.rs b/datafusion/tests/sql.rs index 612a1307021ae..f70973bb1edd6 100644 --- a/datafusion/tests/sql.rs +++ b/datafusion/tests/sql.rs @@ -372,6 +372,22 @@ async fn csv_query_group_by_float32() -> Result<()> { Ok(()) } +#[tokio::test] +async fn select_all() -> Result<()> { + let mut ctx = ExecutionContext::new(); + register_aggregate_simple_csv(&mut ctx)?; + + let sql = "SELECT c1 FROM aggregate_simple order by c1"; + let actual_no_all = execute(&mut ctx, sql).await; + + let sql_all = "SELECT ALL c1 FROM aggregate_simple order by c1"; + let actual_all = execute(&mut ctx, sql_all).await; + + assert_eq!(actual_no_all, actual_all); + + Ok(()) +} + #[tokio::test] async fn select_distinct_all() -> Result<()> { let mut ctx = ExecutionContext::new(); From 316c73431efd568d09b6e3f181a0f6a693633e7d Mon Sep 17 00:00:00 2001 From: "Heres, Daniel" Date: Tue, 4 May 2021 23:06:39 +0200 Subject: [PATCH 09/10] Clippy --- datafusion/src/sql/planner.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datafusion/src/sql/planner.rs b/datafusion/src/sql/planner.rs index 9a75fa75a0fec..48900f56aad5e 100644 --- a/datafusion/src/sql/planner.rs +++ b/datafusion/src/sql/planner.rs @@ -623,7 +623,7 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> { let plan = if select.distinct { return LogicalPlanBuilder::from(&plan) - .aggregate(select_exprs_post_aggr.clone(), vec![])? + .aggregate(select_exprs_post_aggr, vec![])? .build(); } else { plan From 6ea9c7728f95c6de3fdb7f7a43c0bcf15cbac239 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20Heres?= Date: Wed, 5 May 2021 06:25:13 +0200 Subject: [PATCH 10/10] Update datafusion/tests/sql.rs Co-authored-by: Andrew Lamb --- datafusion/tests/sql.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datafusion/tests/sql.rs b/datafusion/tests/sql.rs index f70973bb1edd6..bf28525ad437f 100644 --- a/datafusion/tests/sql.rs +++ b/datafusion/tests/sql.rs @@ -389,7 +389,7 @@ async fn select_all() -> Result<()> { } #[tokio::test] -async fn select_distinct_all() -> Result<()> { +async fn select_distinct() -> Result<()> { let mut ctx = ExecutionContext::new(); register_aggregate_simple_csv(&mut ctx)?;