diff --git a/datafusion/core/tests/data/partitioned_table/c_date=2018-11-13/timestamps.csv b/datafusion/core/tests/data/partitioned_table/c_date=2018-11-13/timestamps.csv new file mode 100644 index 0000000000000..a23ab2af429eb --- /dev/null +++ b/datafusion/core/tests/data/partitioned_table/c_date=2018-11-13/timestamps.csv @@ -0,0 +1,2 @@ +Jorge,2018-12-13T12:12:10.011Z +Andrew,2018-11-13T17:11:10.011Z \ No newline at end of file diff --git a/datafusion/core/tests/data/partitioned_table/c_date=2018-12-13/timestamps.csv b/datafusion/core/tests/data/partitioned_table/c_date=2018-12-13/timestamps.csv new file mode 100644 index 0000000000000..a23ab2af429eb --- /dev/null +++ b/datafusion/core/tests/data/partitioned_table/c_date=2018-12-13/timestamps.csv @@ -0,0 +1,2 @@ +Jorge,2018-12-13T12:12:10.011Z +Andrew,2018-11-13T17:11:10.011Z \ No newline at end of file diff --git a/datafusion/core/tests/data/timestamps.csv b/datafusion/core/tests/data/timestamps.csv new file mode 100644 index 0000000000000..a23ab2af429eb --- /dev/null +++ b/datafusion/core/tests/data/timestamps.csv @@ -0,0 +1,2 @@ +Jorge,2018-12-13T12:12:10.011Z +Andrew,2018-11-13T17:11:10.011Z \ No newline at end of file diff --git a/datafusion/core/tests/sql/create_drop.rs b/datafusion/core/tests/sql/create_drop.rs index fa2cbc00ddb81..1d32d5e0d93cc 100644 --- a/datafusion/core/tests/sql/create_drop.rs +++ b/datafusion/core/tests/sql/create_drop.rs @@ -15,427 +15,11 @@ // specific language governing permissions and limitations // under the License. -use std::fs::create_dir_all; -use std::io::Write; - use datafusion::execution::runtime_env::{RuntimeConfig, RuntimeEnv}; use datafusion::test_util::TestTableFactory; -use tempfile::TempDir; use super::*; -#[tokio::test] -async fn create_table_as() -> Result<()> { - let ctx = SessionContext::new(); - register_aggregate_simple_csv(&ctx).await?; - - let sql = "CREATE TABLE my_table AS SELECT * FROM aggregate_simple"; - ctx.sql(sql).await.unwrap(); - - let sql_all = "SELECT * FROM my_table order by c1 LIMIT 1"; - let results_all = execute_to_batches(&ctx, sql_all).await; - - let expected = vec![ - "+---------+----------------+------+", - "| c1 | c2 | c3 |", - "+---------+----------------+------+", - "| 0.00001 | 0.000000000001 | true |", - "+---------+----------------+------+", - ]; - - assert_batches_eq!(expected, &results_all); - - Ok(()) -} - -#[tokio::test] -async fn create_table_with_schema_as_select() -> Result<()> { - let ctx = SessionContext::new(); - register_aggregate_simple_csv(&ctx).await?; - - let sql = "CREATE TABLE my_table(c1 float, c2 double, c3 boolean, c4 varchar) \ - AS SELECT *,c3 as c4_tmp FROM aggregate_simple"; - ctx.sql(sql).await.unwrap(); - - let sql_all = "SELECT * FROM my_table order by c1 LIMIT 1"; - let results_all = execute_to_batches(&ctx, sql_all).await; - - let expected = vec![ - "+---------+----------------+------+----+", - "| c1 | c2 | c3 | c4 |", - "+---------+----------------+------+----+", - "| 0.00001 | 0.000000000001 | true | 1 |", - "+---------+----------------+------+----+", - ]; - - assert_batches_eq!(expected, &results_all); - - Ok(()) -} - -#[tokio::test] -async fn create_table_with_schema_as_select_mismatch() -> Result<()> { - let ctx = SessionContext::new(); - register_aggregate_simple_csv(&ctx).await?; - - let sql = "CREATE TABLE my_table(c1 float, c2 double, c3 boolean, c4 varchar) \ - AS SELECT * FROM aggregate_simple"; - let expected_err = ctx.sql(sql).await.unwrap_err(); - assert_contains!( - expected_err.to_string(), - "Mismatch: 4 columns specified, but result has 3 columns" - ); - Ok(()) -} - -#[tokio::test] -async fn create_table_with_schema_as_values() -> Result<()> { - let ctx = SessionContext::new(); - register_aggregate_simple_csv(&ctx).await?; - - let sql = - "CREATE TABLE my_table(c1 int, c2 float, c3 varchar) AS VALUES(1, 2, 'hello')"; - ctx.sql(sql).await.unwrap(); - - let sql_all = "SELECT * FROM my_table"; - let results_all = execute_to_batches(&ctx, sql_all).await; - - let expected = vec![ - "+----+----+-------+", - "| c1 | c2 | c3 |", - "+----+----+-------+", - "| 1 | 2 | hello |", - "+----+----+-------+", - ]; - - assert_batches_eq!(expected, &results_all); - - Ok(()) -} - -#[tokio::test] -async fn create_or_replace_table_as() -> Result<()> { - // the information schema used to introduce cyclic Arcs - let ctx = - SessionContext::with_config(SessionConfig::new().with_information_schema(true)); - - // Create table - let result = ctx - .sql("CREATE TABLE y AS VALUES (1,2),(3,4)") - .await - .unwrap() - .collect() - .await - .unwrap(); - assert!(result.is_empty()); - - // Replace table - ctx.sql("CREATE OR REPLACE TABLE y AS VALUES (5,6)") - .await - .unwrap() - .collect() - .await - .unwrap(); - - let sql_all = "SELECT * FROM y"; - let results_all = execute_to_batches(&ctx, sql_all).await; - - let expected = vec![ - "+---------+---------+", - "| column1 | column2 |", - "+---------+---------+", - "| 5 | 6 |", - "+---------+---------+", - ]; - - assert_batches_eq!(expected, &results_all); - - // 'IF NOT EXISTS' cannot coexist with 'REPLACE' - let result = ctx - .sql("CREATE OR REPLACE TABLE if not exists y AS VALUES (7,8)") - .await; - assert!( - result.is_err(), - "'IF NOT EXISTS' cannot coexist with 'REPLACE'" - ); - - Ok(()) -} - -#[tokio::test] -async fn drop_table() -> Result<()> { - let ctx = SessionContext::new(); - register_aggregate_simple_csv(&ctx).await?; - - let sql = "CREATE TABLE my_table AS SELECT * FROM aggregate_simple"; - ctx.sql(sql).await.unwrap(); - - let sql = "DROP TABLE my_table"; - ctx.sql(sql).await.unwrap(); - - let result = ctx.table("my_table"); - assert!(result.is_err(), "drop table should deregister table."); - - let sql = "DROP TABLE IF EXISTS my_table"; - ctx.sql(sql).await.unwrap(); - - Ok(()) -} - -#[tokio::test] -async fn drop_view() -> Result<()> { - let ctx = - SessionContext::with_config(SessionConfig::new().with_information_schema(true)); - plan_and_collect(&ctx, "CREATE VIEW v AS SELECT 1").await?; - let rb = plan_and_collect( - &ctx, - "select * from information_schema.tables where table_name = 'v' and table_type = 'VIEW'", - ) - .await?; - assert_eq!(rb[0].num_rows(), 1); - - plan_and_collect(&ctx, "DROP VIEW v").await?; - let rb = plan_and_collect( - &ctx, - "select * from information_schema.tables where table_name = 'v' and table_type = 'VIEW'", - ) - .await?; - assert!(rb.is_empty()); - Ok(()) -} - -#[tokio::test] -#[should_panic(expected = "doesn't exist")] -async fn drop_view_nonexistent() { - let ctx = SessionContext::new(); - ctx.sql("DROP VIEW non_existent_view") - .await - .unwrap() - .collect() - .await - .unwrap(); -} - -#[tokio::test] -#[should_panic(expected = "doesn't exist")] -async fn drop_view_cant_drop_table() { - let ctx = SessionContext::new(); - ctx.sql("CREATE TABLE t AS SELECT 1") - .await - .unwrap() - .collect() - .await - .unwrap(); - ctx.sql("DROP VIEW t") - .await - .unwrap() - .collect() - .await - .unwrap(); -} - -#[tokio::test] -#[should_panic(expected = "doesn't exist")] -async fn drop_table_cant_drop_view() { - let ctx = SessionContext::new(); - ctx.sql("CREATE VIEW v AS SELECT 1") - .await - .unwrap() - .collect() - .await - .unwrap(); - ctx.sql("DROP TABLE v") - .await - .unwrap() - .collect() - .await - .unwrap(); -} - -#[tokio::test] -async fn drop_view_if_exists() -> Result<()> { - let ctx = - SessionContext::with_config(SessionConfig::new().with_information_schema(true)); - plan_and_collect(&ctx, "CREATE VIEW v AS SELECT 1").await?; - let rb = plan_and_collect(&ctx, "DROP VIEW IF EXISTS non_existent_view").await?; - // make sure we get an empty response back - assert!(rb.is_empty()); - - let rb = plan_and_collect( - &ctx, - "select * from information_schema.views where table_name = 'v'", - ) - .await?; - // confirm view exists - assert_eq!(rb[0].num_rows(), 1); - - plan_and_collect(&ctx, "DROP VIEW IF EXISTS v").await?; - let rb = plan_and_collect( - &ctx, - "select * from information_schema.views where table_name = 'v'", - ) - .await?; - // confirm view is gone - assert!(rb.is_empty()); - Ok(()) -} - -#[tokio::test] -async fn csv_query_create_external_table() { - let ctx = SessionContext::new(); - register_aggregate_csv_by_sql(&ctx).await; - let sql = "SELECT c1, c2, c3, c4, c5, c6, c7, c8, c9, 10, c11, c12, c13 FROM aggregate_test_100 LIMIT 1"; - let actual = execute_to_batches(&ctx, sql).await; - let expected = vec![ - "+----+----+----+-------+------------+----------------------+----+-------+------------+-----------+-------------+--------------------+--------------------------------+", - "| c1 | c2 | c3 | c4 | c5 | c6 | c7 | c8 | c9 | Int64(10) | c11 | c12 | c13 |", - "+----+----+----+-------+------------+----------------------+----+-------+------------+-----------+-------------+--------------------+--------------------------------+", - "| c | 2 | 1 | 18109 | 2033001162 | -6513304855495910254 | 25 | 43062 | 1491205016 | 10 | 0.110830784 | 0.9294097332465232 | 6WfVFBVGJSQb7FhA7E0lBwdvjfZnSW |", - "+----+----+----+-------+------------+----------------------+----+-------+------------+-----------+-------------+--------------------+--------------------------------+", - ]; - assert_batches_eq!(expected, &actual); -} - -#[tokio::test] -async fn create_external_table_with_timestamps() { - let ctx = SessionContext::new(); - - let data = "Jorge,2018-12-13T12:12:10.011Z\n\ - Andrew,2018-11-13T17:11:10.011Z"; - - let tmp_dir = TempDir::new().unwrap(); - let file_path = tmp_dir.path().join("timestamps.csv"); - - // scope to ensure the file is closed and written - { - std::fs::File::create(&file_path) - .expect("creating temp file") - .write_all(data.as_bytes()) - .expect("writing data"); - } - - let sql = format!( - "CREATE EXTERNAL TABLE csv_with_timestamps ( - name VARCHAR, - ts TIMESTAMP - ) - STORED AS CSV - LOCATION '{}' - ", - file_path.to_str().expect("path is utf8") - ); - - plan_and_collect(&ctx, &sql) - .await - .expect("Executing CREATE EXTERNAL TABLE"); - - let sql = "SELECT * from csv_with_timestamps"; - let result = plan_and_collect(&ctx, sql).await.unwrap(); - let expected = vec![ - "+--------+-------------------------+", - "| name | ts |", - "+--------+-------------------------+", - "| Andrew | 2018-11-13T17:11:10.011 |", - "| Jorge | 2018-12-13T12:12:10.011 |", - "+--------+-------------------------+", - ]; - assert_batches_sorted_eq!(expected, &result); -} - -#[tokio::test] -async fn create_partitioned_external_table() -> Result<()> { - let ctx = SessionContext::new(); - - let data0 = "Jorge,2018-12-13T12:12:10.011Z"; - let data1 = "Andrew,2018-11-13T17:11:10.011Z"; - - let tmp_dir = TempDir::new().unwrap(); - - // scope to ensure the file is closed and written - { - let part0 = tmp_dir.path().join("c_date=2018-12-13"); - let file_path0 = part0.join("timestamps.csv"); - create_dir_all(part0).expect("creating part dir"); - File::options() - .read(true) - .write(true) - .create(true) - .open(&file_path0) - .expect("creating temp file") - .write_all(data0.as_bytes()) - .expect("writing data"); - - let part1 = tmp_dir.path().join("c_date=2018-11-13"); - let file_path1 = part1.join("timestamps.csv"); - create_dir_all(part1).expect("creating part dir"); - File::options() - .read(true) - .write(true) - .create(true) - .open(&file_path1) - .expect("creating temp file") - .write_all(data1.as_bytes()) - .expect("writing data"); - } - - let sql = format!( - "CREATE EXTERNAL TABLE csv_with_timestamps ( - name VARCHAR, - ts TIMESTAMP, - c_date DATE, - ) - STORED AS CSV - PARTITIONED BY (c_date) - LOCATION '{}' - ", - tmp_dir.path().to_str().expect("path is utf8") - ); - - plan_and_collect(&ctx, &sql) - .await - .expect("Executing CREATE EXTERNAL TABLE"); - - let sql = "SELECT * from csv_with_timestamps where c_date='2018-11-13'"; - let result = plan_and_collect(&ctx, sql).await.unwrap(); - let expected = vec![ - "+--------+-------------------------+------------+", - "| name | ts | c_date |", - "+--------+-------------------------+------------+", - "| Andrew | 2018-11-13T17:11:10.011 | 2018-11-13 |", - "+--------+-------------------------+------------+", - ]; - assert_batches_sorted_eq!(expected, &result); - - Ok(()) -} - -#[tokio::test] -#[should_panic(expected = "already exists")] -async fn sql_create_duplicate_table() { - // the information schema used to introduce cyclic Arcs - let ctx = - SessionContext::with_config(SessionConfig::new().with_information_schema(true)); - - // Create table - ctx.sql("CREATE TABLE y AS VALUES (1,2,3)") - .await - .unwrap() - .collect() - .await - .unwrap(); - - // Create table again - let result = ctx - .sql("CREATE TABLE y AS VALUES (1,2,3)") - .await - .unwrap() - .collect() - .await - .unwrap(); - - assert_eq!(result, Vec::new()); -} - #[tokio::test] async fn sql_create_table_if_not_exists() -> Result<()> { // the information schema used to introduce cyclic Arcs @@ -474,29 +58,6 @@ async fn sql_create_table_if_not_exists() -> Result<()> { Ok(()) } -#[tokio::test] -async fn create_pipe_delimited_csv_table() -> Result<()> { - let ctx = SessionContext::new(); - - let sql = "CREATE EXTERNAL TABLE aggregate_simple STORED AS CSV WITH HEADER ROW DELIMITER '|' LOCATION 'tests/aggregate_simple_pipe.csv'"; - ctx.sql(sql).await.unwrap(); - - let sql_all = "SELECT * FROM aggregate_simple order by c1 LIMIT 1"; - let results_all = execute_to_batches(&ctx, sql_all).await; - - let expected = vec![ - "+---------+----------------+------+", - "| c1 | c2 | c3 |", - "+---------+----------------+------+", - "| 0.00001 | 0.000000000001 | true |", - "+---------+----------------+------+", - ]; - - assert_batches_eq!(expected, &results_all); - - Ok(()) -} - #[tokio::test] async fn create_custom_table() -> Result<()> { let mut cfg = RuntimeConfig::new(); @@ -588,8 +149,3 @@ async fn create_csv_table_empty_file() -> Result<()> { Ok(()) } - -/// Execute SQL and return results -async fn plan_and_collect(ctx: &SessionContext, sql: &str) -> Result> { - ctx.sql(sql).await?.collect().await -} diff --git a/datafusion/core/tests/sqllogictests/test_files/ddl.slt b/datafusion/core/tests/sqllogictests/test_files/ddl.slt index 07e71ddc428b5..20dca0a0a354f 100644 --- a/datafusion/core/tests/sqllogictests/test_files/ddl.slt +++ b/datafusion/core/tests/sqllogictests/test_files/ddl.slt @@ -55,15 +55,15 @@ statement ok DROP TABLE users; # Table is gone -statement error +statement error Error during planning: table 'datafusion.public.users' not found select * from users; # Can not drop it again -statement error +statement error Table Bare \{ table: "user" \} doesn't exist. DROP TABLE user; # Can not insert into a undefined table -statement error +statement error No table named 'user' insert into user values(1, 20); ########## @@ -83,11 +83,11 @@ select * from "foo.bar"; 2 3 # Can not select from non existent foo.bar table -statement error +statement error failed to resolve schema: foo select * from foo.bar; # Error if wrong capitalization to select -statement error +statement error table 'datafusion.public.Foo.bar' not found select * from "Foo.bar"; # Should be able to drop the table @@ -125,11 +125,11 @@ select * from "foo.bar"; 2 3 # Can not select from non existent foo.bar view -statement error +statement error failed to resolve schema: foo select * from foo.bar; # Error if wrong capitalization to select -statement error +statement error table 'datafusion.public.Foo.bar' not found select * from "Foo.bar"; # Should be able to drop the view @@ -175,12 +175,242 @@ select * from "bar"; 4 5 # multiple drops not supported -statement error +statement error Multiple objects not supported DROP VIEW foo, bar; # multiple drops not supported statement ok DROP VIEW foo; +# Ok to drop statement ok DROP VIEW bar; + +# not ok to drop after already dropped +statement error View Bare \{ table: "bar" \} doesn't exist. +DROP VIEW bar; + +# ok to drop with IF EXISTS after already dropped +statement ok +DROP VIEW IF EXISTS bar; + +# can't drop non existent view +statement error View Bare \{ table: "non_existent_view" \} doesn't exist. +DROP VIEW non_existent_view + +########## +# Create / drop views agaist tables +########## + +statement ok +CREATE external table aggregate_simple(c1 real, c2 double, c3 boolean) STORED as CSV WITH HEADER ROW LOCATION 'tests/aggregate_simple.csv'; + +# create_table_as +statement ok +CREATE TABLE my_table AS SELECT * FROM aggregate_simple + +query II rowsort +SELECT * FROM my_table order by c1 LIMIT 1 +---- +0.00001 1.0e-12 true + +statement ok +DROP TABLE my_table; + +# not ok to drop after already dropped +statement error Table Bare \{ table: "my_table" \} doesn't exist. +DROP TABLE my_table; + +# ok to drop after already dropped with IF EXISTS +statement ok +DROP TABLE IF EXISTS my_table; + +# Can't drop non existent table +statement error Table Bare \{ table: "non_existent_table" \} doesn't exist. +DROP TABLE non_existent_table; + + +# create_table_with_schema_as_select +statement ok +CREATE TABLE my_table(c1 float, c2 double, c3 boolean, c4 varchar) AS SELECT *,c3 as c4_tmp FROM aggregate_simple; + +query II rowsort +SELECT * FROM my_table order by c1 LIMIT 1 +---- +0.00001 1.0e-12 true 1 + +statement ok +DROP TABLE my_table; + +statement ok +DROP TABLE aggregate_simple + +# create_table_with_schema_as_select_mismatch +statement error table 'datafusion.public.aggregate_simple' not found +CREATE TABLE my_table(c1 float, c2 double, c3 boolean, c4 varchar) AS SELECT * FROM aggregate_simple; + + +# create_table_with_schema_as_values +statement ok +CREATE TABLE my_table(c1 int, c2 float, c3 varchar) AS VALUES(1, 2, 'hello') + +query II rowsort +SELECT * FROM my_table; +---- +1 2.0 hello + +statement ok +DROP TABLE my_table; + + +# TODO enable information schema + +statement ok +CREATE TABLE y AS VALUES (1,2),(3,4); + +statement ok +CREATE OR REPLACE TABLE y AS VALUES (5,6); + +query II +SELECT * FROM y +---- +5 6 + +# 'IF NOT EXISTS' cannot coexist with 'REPLACE' +# TODO this shoudn't be an internal error +statement error Internal error: 'IF NOT EXISTS' cannot coexist with 'REPLACE' +CREATE OR REPLACE TABLE if not exists y AS VALUES (7,8); + +statement ok +DROP TABLE y + + +# drop_view_cant_drop_table +statement ok +CREATE TABLE t AS SELECT 1 + +statement error View Bare \{ table: "t" \} doesn't exist. +DROP VIEW t + +statement ok +DROP TABLE t + +# drop_table_cant_drop_view +statement ok +CREATE VIEW v AS SELECT 1; + +statement error Table Bare \{ table: "v" \} doesn't exist. +DROP TABLE v; + +statement ok +DROP VIEW v; + + +# csv_query_create_external_table +statement ok +CREATE EXTERNAL TABLE aggregate_test_100 ( + c1 VARCHAR NOT NULL, + c2 TINYINT NOT NULL, + c3 SMALLINT NOT NULL, + c4 SMALLINT NOT NULL, + c5 INTEGER NOT NULL, + c6 BIGINT NOT NULL, + c7 SMALLINT NOT NULL, + c8 INT NOT NULL, + c9 INT UNSIGNED NOT NULL, + c10 BIGINT UNSIGNED NOT NULL, + c11 FLOAT NOT NULL, + c12 DOUBLE NOT NULL, + c13 VARCHAR NOT NULL + ) +STORED AS CSV +WITH HEADER ROW +LOCATION '../../testing/data/csv/aggregate_test_100.csv'; + +query IIIIIIIIIIIII +SELECT c1, c2, c3, c4, c5, c6, c7, c8, c9, 10, c11, c12, c13 FROM aggregate_test_100 LIMIT 1; +---- +c 2 1 18109 2033001162 -6513304855495910254 25 43062 1491205016 10 0.110830784 0.9294097332465232 6WfVFBVGJSQb7FhA7E0lBwdvjfZnSW + + + +# create_external_table_with_timestamps +statement ok +CREATE EXTERNAL TABLE csv_with_timestamps ( +name VARCHAR, +ts TIMESTAMP +) STORED AS CSV LOCATION 'tests/data/timestamps.csv'; + +query CC +SELECT * from csv_with_timestamps +---- +Jorge 2018-12-13T12:12:10.011000000 +Andrew 2018-11-13T17:11:10.011000000 + +statement ok +DROP TABLE csv_with_timestamps + +# create_partitioned_external_table +# +# this test covers creating a partitioned file. The input files are in this structure: +# +# tests/data/partitioned_table/c_date=2018-12-13/timestamps.csv +# tests/data/partitioned_table/c_date=2018-11-13/timestamps.csv + +statement ok +CREATE EXTERNAL TABLE csv_with_timestamps ( + name VARCHAR, + ts TIMESTAMP, + c_date DATE, +) +STORED AS CSV +PARTITIONED BY (c_date) +LOCATION 'tests/data/partitioned_table'; + +query CCC +SELECT * from csv_with_timestamps where c_date='2018-11-13' +---- +Jorge 2018-12-13T12:12:10.011000000 2018-11-13 +Andrew 2018-11-13T17:11:10.011000000 2018-11-13 + +statement ok +DROP TABLE csv_with_timestamps + + +# sql_create_duplicate_table +# Can not recreate the same table +statement ok +CREATE TABLE y AS VALUES (1,2,3); + +statement error Table 'Bare \{ table: "y" \}' already exists +CREATE TABLE y AS VALUES (1,2,3); + +statement ok +DROP TABLE y; + + +# Can not recreate the same view +statement ok +CREATE VIEW y AS VALUES (1,2,3); + +statement error Table 'Bare \{ table: "y" \}' already exists +CREATE VIEW y AS VALUES (1,2,3); + +statement ok +DROP VIEW y; + + + +# create_pipe_delimited_csv_table() + +statement ok +CREATE EXTERNAL TABLE aggregate_simple STORED AS CSV WITH HEADER ROW DELIMITER '|' LOCATION 'tests/aggregate_simple_pipe.csv'; + + +query CCC +SELECT * FROM aggregate_simple order by c1 LIMIT 1; +---- +0.00001 1.0e-12 true + +statement ok +DROP TABLE aggregate_simple