Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 18 additions & 15 deletions datafusion/src/sql/planner.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,9 @@

//! SQL Query Planner (produces logical plan from SQL AST)

use std::convert::TryInto;
use std::str::FromStr;
use std::sync::Arc;
use std::{convert::TryInto, vec};

use crate::catalog::TableReference;
use crate::datasource::TableProvider;
Expand Down Expand Up @@ -621,7 +621,15 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> {
plan
};

self.project(&plan, select_exprs_post_aggr, false)
let plan = if select.distinct {
return LogicalPlanBuilder::from(&plan)
.aggregate(select_exprs_post_aggr, vec![])?
.build();
} else {
plan
};

self.project(&plan, select_exprs_post_aggr)
}

/// Returns the `Expr`'s corresponding to a SQL query's SELECT expressions.
Expand All @@ -645,24 +653,19 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> {

/// Wrap a plan in a projection
///
/// If the `force` argument is `false`, the projection is applied only when
/// necessary, i.e., when the input fields are different than the
/// The projection is applied only when necessary,
/// i.e., when the input fields are different than the
/// projection. Note that if the input fields are the same, but out of
/// order, the projection will be applied.
fn project(
&self,
input: &LogicalPlan,
expr: Vec<Expr>,
force: bool,

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

small drive-by simplification, an unused option

) -> Result<LogicalPlan> {
fn project(&self, input: &LogicalPlan, expr: Vec<Expr>) -> Result<LogicalPlan> {
self.validate_schema_satisfies_exprs(&input.schema(), &expr)?;

let plan = LogicalPlanBuilder::from(input).project(expr)?.build()?;

let project = force
|| match input {
LogicalPlan::TableScan { .. } => true,
_ => plan.schema().fields() != input.schema().fields(),
};
let project = match input {
LogicalPlan::TableScan { .. } => true,
_ => plan.schema().fields() != input.schema().fields(),
};

if project {
Ok(plan)
Expand Down
75 changes: 75 additions & 0 deletions datafusion/tests/sql.rs
Original file line number Diff line number Diff line change
Expand Up @@ -372,6 +372,81 @@ async fn csv_query_group_by_float32() -> Result<()> {
Ok(())
}

#[tokio::test]
async fn select_all() -> Result<()> {
let mut ctx = ExecutionContext::new();
register_aggregate_simple_csv(&mut ctx)?;

let sql = "SELECT c1 FROM aggregate_simple order by c1";
let actual_no_all = execute(&mut ctx, sql).await;

let sql_all = "SELECT ALL c1 FROM aggregate_simple order by c1";
let actual_all = execute(&mut ctx, sql_all).await;

assert_eq!(actual_no_all, actual_all);

Ok(())
}

#[tokio::test]
async fn select_distinct() -> Result<()> {
let mut ctx = ExecutionContext::new();
register_aggregate_simple_csv(&mut ctx)?;

let sql = "SELECT DISTINCT * FROM aggregate_simple";
let mut actual = execute(&mut ctx, sql).await;
actual.sort();

let mut dedup = actual.clone();
dedup.dedup();

assert_eq!(actual, dedup);

Ok(())
}

#[tokio::test]
async fn select_distinct_simple() -> Result<()> {
let mut ctx = ExecutionContext::new();
register_aggregate_simple_csv(&mut ctx)?;

let sql = "SELECT DISTINCT c1 FROM aggregate_simple order by c1";
let actual = execute(&mut ctx, sql).await;

let expected = vec![

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I doubled checked that the input has duplicates (to show that the distinct has removed them): https://github.com/apache/arrow-datafusion/blob/master/datafusion/tests/aggregate_simple.csv

👍

vec!["0.00001"],
vec!["0.00002"],
vec!["0.00003"],
vec!["0.00004"],
vec!["0.00005"],
];
assert_eq!(actual, expected);

let sql = "SELECT DISTINCT c1, c2 FROM aggregate_simple order by c1";
let actual = execute(&mut ctx, sql).await;

let expected = vec![
vec!["0.00001", "0.000000000001"],
vec!["0.00002", "0.000000000002"],
vec!["0.00003", "0.000000000003"],
vec!["0.00004", "0.000000000004"],
vec!["0.00005", "0.000000000005"],
];
assert_eq!(actual, expected);

let sql = "SELECT distinct c3 FROM aggregate_simple order by c3";
let actual = execute(&mut ctx, sql).await;

let expected = vec![vec!["false"], vec!["true"]];
assert_eq!(actual, expected);

let sql = "SELECT distinct c1+c2 as a FROM aggregate_simple";
let actual = execute(&mut ctx, sql).await;

assert_eq!(actual.len(), 5);
Ok(())
}

#[tokio::test]
async fn csv_query_group_by_float64() -> Result<()> {
let mut ctx = ExecutionContext::new();
Expand Down
8 changes: 7 additions & 1 deletion docs/user-guide/src/sql/select.md
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ DataFusion supports the following syntax for queries:
<code class="language-sql hljs">

[ [WITH](#with-clause) with_query [, ...] ] <br/>
[SELECT](#select-clause) select_expr [, ...] <br/>
[SELECT](#select-clause) [ ALL | DISTINCT ] select_expr [, ...] <br/>
[ [FROM](#from-clause) from_item [, ...] ] <br/>
[ [WHERE](#where-clause) condition ] <br/>
[ [GROUP BY](#group-by-clause) grouping_element [, ...] ] <br/>
Expand Down Expand Up @@ -55,6 +55,12 @@ Example:
SELECT a, b, a + b FROM table
```

The `DISTINCT` quantifier can be added to make the query return all distinct rows.
By default `ALL` will be used, which returns all the rows.

```sql
SELECT DISTINCT person, age FROM employees
```

# FROM clause

Expand Down