From f34d198c3bae22fc879315cc60f6e7fc37fc0fea Mon Sep 17 00:00:00 2001 From: "Heres, Daniel" Date: Sat, 1 May 2021 13:08:27 +0200 Subject: [PATCH 01/12] Move datafusion-cli to own crate --- Cargo.toml | 1 + datafusion-cli/Cargo.toml | 16 +++++ datafusion-cli/src/main.rs | 140 +++++++++++++++++++++++++++++++++++++ datafusion/Cargo.toml | 5 +- datafusion/src/bin/repl.rs | 139 ------------------------------------ 5 files changed, 158 insertions(+), 143 deletions(-) create mode 100644 datafusion-cli/Cargo.toml create mode 100644 datafusion-cli/src/main.rs diff --git a/Cargo.toml b/Cargo.toml index 2f34babdb247b..fa36a0c0fed7c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,6 +18,7 @@ [workspace] members = [ "datafusion", + "datafusion-cli", "datafusion-examples", "benchmarks", "ballista/rust/client", diff --git a/datafusion-cli/Cargo.toml b/datafusion-cli/Cargo.toml new file mode 100644 index 0000000000000..a125d68652bd2 --- /dev/null +++ b/datafusion-cli/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "datafusion-cli" +version = "0.1.0" +authors = ["Heres, Daniel "] +edition = "2018" +keywords = [ "arrow", "query", "sql", "cli", "repl" ] +homepage = "https://github.com/apache/arrow-datafusion" +repository = "https://github.com/apache/arrow-datafusion" + + +[dependencies] +clap = "2.33" +rustyline ="8.0" +tokio = { version = "1.0", features = ["macros", "rt", "rt-multi-thread", "sync"] } +arrow = { git = "https://github.com/apache/arrow-rs", rev = "d008f31b107c1030a1f5144c164e8ca8bf543576", features = ["prettyprint"] } +datafusion = { path = "../datafusion" } diff --git a/datafusion-cli/src/main.rs b/datafusion-cli/src/main.rs new file mode 100644 index 0000000000000..634a3d879c884 --- /dev/null +++ b/datafusion-cli/src/main.rs @@ -0,0 +1,140 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#![allow(bare_trait_objects)] + +use arrow::util::pretty; +use clap::{crate_version, App, Arg}; +use datafusion::error::Result; +use datafusion::execution::context::{ExecutionConfig, ExecutionContext}; +use rustyline::Editor; +use std::env; +use std::path::Path; +use std::time::Instant; + +#[tokio::main] +pub async fn main() { + let matches = App::new("DataFusion") + .version(crate_version!()) + .about( + "DataFusion is an in-memory query engine that uses Apache Arrow \ + as the memory model. It supports executing SQL queries against CSV and \ + Parquet files as well as querying directly against in-memory data.", + ) + .arg( + Arg::with_name("data-path") + .help("Path to your data, default to current directory") + .short("p") + .long("data-path") + .takes_value(true), + ) + .arg( + Arg::with_name("batch-size") + .help("The batch size of each query, default value is 8192") + .short("c") + .long("batch-size") + .takes_value(true), + ) + .get_matches(); + + if let Some(path) = matches.value_of("data-path") { + let p = Path::new(path); + env::set_current_dir(&p).unwrap(); + }; + + let batch_size = matches + .value_of("batch-size") + .map(|size| size.parse::().unwrap()) + .unwrap_or(8192); + + let mut ctx = ExecutionContext::with_config( + ExecutionConfig::new() + .with_batch_size(batch_size) + .with_information_schema(true), + ); + + let mut rl = Editor::<()>::new(); + rl.load_history(".history").ok(); + + let mut query = "".to_owned(); + loop { + let readline = rl.readline("> "); + match readline { + Ok(ref line) if is_exit_command(line) && query.is_empty() => { + break; + } + Ok(ref line) if line.trim_end().ends_with(';') => { + query.push_str(line.trim_end()); + rl.add_history_entry(query.clone()); + match exec_and_print(&mut ctx, query).await { + Ok(_) => {} + Err(err) => println!("{:?}", err), + } + query = "".to_owned(); + } + Ok(ref line) => { + query.push_str(line); + query.push(' '); + } + Err(_) => { + break; + } + } + } + + rl.save_history(".history").ok(); +} + +fn is_exit_command(line: &str) -> bool { + let line = line.trim_end().to_lowercase(); + line == "quit" || line == "exit" +} + +async fn exec_and_print(ctx: &mut ExecutionContext, sql: String) -> Result<()> { + let now = Instant::now(); + + let df = ctx.sql(&sql)?; + let results = df.collect().await?; + + if results.is_empty() { + println!( + "0 rows in set. Query took {} seconds.", + now.elapsed().as_secs() + ); + return Ok(()); + } + + pretty::print_batches(&results)?; + + let row_count: usize = results.iter().map(|b| b.num_rows()).sum(); + + if row_count > 1 { + println!( + "{} row in set. Query took {} seconds.", + row_count, + now.elapsed().as_secs() + ); + } else { + println!( + "{} rows in set. Query took {} seconds.", + row_count, + now.elapsed().as_secs() + ); + } + + Ok(()) +} diff --git a/datafusion/Cargo.toml b/datafusion/Cargo.toml index 5f743f6f559ef..3d5d79e3810f1 100644 --- a/datafusion/Cargo.toml +++ b/datafusion/Cargo.toml @@ -41,8 +41,7 @@ name = "datafusion-cli" path = "src/bin/main.rs" [features] -default = ["cli", "crypto_expressions", "regex_expressions", "unicode_expressions"] -cli = ["rustyline"] +default = ["crypto_expressions", "regex_expressions", "unicode_expressions"] simd = ["arrow/simd"] crypto_expressions = ["md-5", "sha2"] regex_expressions = ["regex", "lazy_static"] @@ -54,8 +53,6 @@ hashbrown = "0.11" arrow = { git = "https://github.com/apache/arrow-rs", rev = "d008f31b107c1030a1f5144c164e8ca8bf543576", features = ["prettyprint"] } parquet = { git = "https://github.com/apache/arrow-rs", rev = "d008f31b107c1030a1f5144c164e8ca8bf543576", features = ["arrow"] } sqlparser = "0.9.0" -clap = "2.33" -rustyline = {version = "7.0", optional = true} paste = "^1.0" num_cpus = "1.13.0" chrono = "0.4" diff --git a/datafusion/src/bin/repl.rs b/datafusion/src/bin/repl.rs index a6aec204c0d3b..8b137891791fe 100644 --- a/datafusion/src/bin/repl.rs +++ b/datafusion/src/bin/repl.rs @@ -1,140 +1 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -#![allow(bare_trait_objects)] - -use arrow::util::pretty; -use clap::{crate_version, App, Arg}; -use datafusion::error::Result; -use datafusion::execution::context::{ExecutionConfig, ExecutionContext}; -use rustyline::Editor; -use std::env; -use std::path::Path; -use std::time::Instant; - -#[tokio::main] -pub async fn main() { - let matches = App::new("DataFusion") - .version(crate_version!()) - .about( - "DataFusion is an in-memory query engine that uses Apache Arrow \ - as the memory model. It supports executing SQL queries against CSV and \ - Parquet files as well as querying directly against in-memory data.", - ) - .arg( - Arg::with_name("data-path") - .help("Path to your data, default to current directory") - .short("p") - .long("data-path") - .takes_value(true), - ) - .arg( - Arg::with_name("batch-size") - .help("The batch size of each query, default value is 1048576") - .short("c") - .long("batch-size") - .takes_value(true), - ) - .get_matches(); - - if let Some(path) = matches.value_of("data-path") { - let p = Path::new(path); - env::set_current_dir(&p).unwrap(); - }; - - let batch_size = matches - .value_of("batch-size") - .map(|size| size.parse::().unwrap()) - .unwrap_or(1_048_576); - - let mut ctx = ExecutionContext::with_config( - ExecutionConfig::new() - .with_batch_size(batch_size) - .with_information_schema(true), - ); - - let mut rl = Editor::<()>::new(); - rl.load_history(".history").ok(); - - let mut query = "".to_owned(); - loop { - let readline = rl.readline("> "); - match readline { - Ok(ref line) if is_exit_command(line) && query.is_empty() => { - break; - } - Ok(ref line) if line.trim_end().ends_with(';') => { - query.push_str(line.trim_end()); - rl.add_history_entry(query.clone()); - match exec_and_print(&mut ctx, query).await { - Ok(_) => {} - Err(err) => println!("{:?}", err), - } - query = "".to_owned(); - } - Ok(ref line) => { - query.push_str(line); - query.push(' '); - } - Err(_) => { - break; - } - } - } - - rl.save_history(".history").ok(); -} - -fn is_exit_command(line: &str) -> bool { - let line = line.trim_end().to_lowercase(); - line == "quit" || line == "exit" -} - -async fn exec_and_print(ctx: &mut ExecutionContext, sql: String) -> Result<()> { - let now = Instant::now(); - - let df = ctx.sql(&sql)?; - let results = df.collect().await?; - - if results.is_empty() { - println!( - "0 rows in set. Query took {} seconds.", - now.elapsed().as_secs() - ); - return Ok(()); - } - - pretty::print_batches(&results)?; - - let row_count: usize = results.iter().map(|b| b.num_rows()).sum(); - - if row_count > 1 { - println!( - "{} row in set. Query took {} seconds.", - row_count, - now.elapsed().as_secs() - ); - } else { - println!( - "{} rows in set. Query took {} seconds.", - row_count, - now.elapsed().as_secs() - ); - } - - Ok(()) -} From 5fc2d74a9a3bcb100941000640e84c9a393ad9af Mon Sep 17 00:00:00 2001 From: "Heres, Daniel" Date: Sat, 1 May 2021 13:11:54 +0200 Subject: [PATCH 02/12] Cargo.toml fixes, remove repl code --- datafusion-cli/Cargo.toml | 5 +++-- datafusion/src/bin/main.rs | 25 ------------------------- datafusion/src/bin/repl.rs | 1 - 3 files changed, 3 insertions(+), 28 deletions(-) delete mode 100644 datafusion/src/bin/main.rs delete mode 100644 datafusion/src/bin/repl.rs diff --git a/datafusion-cli/Cargo.toml b/datafusion-cli/Cargo.toml index a125d68652bd2..57408c1e96647 100644 --- a/datafusion-cli/Cargo.toml +++ b/datafusion-cli/Cargo.toml @@ -1,9 +1,10 @@ [package] name = "datafusion-cli" -version = "0.1.0" -authors = ["Heres, Daniel "] +version = "4.0.0-SNAPSHOT" +authors = ["Apache Arrow "] edition = "2018" keywords = [ "arrow", "query", "sql", "cli", "repl" ] +license = "Apache-2.0" homepage = "https://github.com/apache/arrow-datafusion" repository = "https://github.com/apache/arrow-datafusion" diff --git a/datafusion/src/bin/main.rs b/datafusion/src/bin/main.rs deleted file mode 100644 index deb5b796b2d69..0000000000000 --- a/datafusion/src/bin/main.rs +++ /dev/null @@ -1,25 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -// Only bring in dependencies for the repl when the cli feature is enabled. -#[cfg(feature = "cli")] -mod repl; - -pub fn main() { - #[cfg(feature = "cli")] - repl::main() -} diff --git a/datafusion/src/bin/repl.rs b/datafusion/src/bin/repl.rs deleted file mode 100644 index 8b137891791fe..0000000000000 --- a/datafusion/src/bin/repl.rs +++ /dev/null @@ -1 +0,0 @@ - From 9ac80d7fdb60f24d0ccde62f542c5666e9db5d6c Mon Sep 17 00:00:00 2001 From: "Heres, Daniel" Date: Sat, 1 May 2021 13:15:26 +0200 Subject: [PATCH 03/12] Remove bin option --- datafusion/Cargo.toml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/datafusion/Cargo.toml b/datafusion/Cargo.toml index 3d5d79e3810f1..e154fbe3fe9d3 100644 --- a/datafusion/Cargo.toml +++ b/datafusion/Cargo.toml @@ -36,10 +36,6 @@ edition = "2018" name = "datafusion" path = "src/lib.rs" -[[bin]] -name = "datafusion-cli" -path = "src/bin/main.rs" - [features] default = ["crypto_expressions", "regex_expressions", "unicode_expressions"] simd = ["arrow/simd"] From b53442fa13d11d19a49e9e6799b7577807683e81 Mon Sep 17 00:00:00 2001 From: "Heres, Daniel" Date: Sat, 1 May 2021 13:27:54 +0200 Subject: [PATCH 04/12] License and doc updates --- datafusion-cli/Cargo.toml | 17 +++++++++++++++++ datafusion/docs/cli.md | 10 +++++----- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/datafusion-cli/Cargo.toml b/datafusion-cli/Cargo.toml index 57408c1e96647..0d241b1875fea 100644 --- a/datafusion-cli/Cargo.toml +++ b/datafusion-cli/Cargo.toml @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + [package] name = "datafusion-cli" version = "4.0.0-SNAPSHOT" diff --git a/datafusion/docs/cli.md b/datafusion/docs/cli.md index aeacdeee04a18..5c9ca6616de5b 100644 --- a/datafusion/docs/cli.md +++ b/datafusion/docs/cli.md @@ -26,9 +26,9 @@ The DataFusion CLI is a command-line interactive SQL utility that allows queries Use the following commands to clone this repository and run the CLI. This will require the Rust toolchain to be installed. Rust can be installed from [https://rustup.rs/](https://rustup.rs/). ```sh -git clone https://github.com/apache/arrow -cd arrow/rust/datafusion -cargo run --bin datafusion-cli --release +git clone https://github.com/apache/arrow-datafusion +cd arrow/datafusion-cli +cargo run --release ``` ## Run using Docker @@ -36,9 +36,9 @@ cargo run --bin datafusion-cli --release Use the following commands to clone this repository and build a Docker image containing the CLI tool. Note that there is `.dockerignore` file in the root of the repository that may need to be deleted in order for this to work. ```sh -git clone https://github.com/apache/arrow +git clone https://github.com/apache/arrow-datafusion cd arrow -docker build -f rust/datafusion/Dockerfile . --tag datafusion-cli +docker build -f datafusion/Dockerfile . --tag datafusion-cli docker run -it -v $(your_data_location):/data datafusion-cli ``` From eba4d22b151d05d69b6125add1ba74dfdefd7481 Mon Sep 17 00:00:00 2001 From: "Heres, Daniel" Date: Sat, 1 May 2021 13:29:36 +0200 Subject: [PATCH 05/12] Fix link to CLI docs in readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f6ef7d176686e..783bdd655cbb6 100644 --- a/README.md +++ b/README.md @@ -132,7 +132,7 @@ datafusion = "4.0.0-SNAPSHOT" ## Using DataFusion as a binary -DataFusion also includes a simple command-line interactive SQL utility. See the [CLI reference](docs/cli.md) for more information. +DataFusion also includes a simple command-line interactive SQL utility. See the [CLI reference](datafusion/docs/cli.md) for more information. # Status From 2eb0fab340929b8c9c6d3e86d735cf5a8518bc6b Mon Sep 17 00:00:00 2001 From: "Heres, Daniel" Date: Sat, 1 May 2021 13:33:36 +0200 Subject: [PATCH 06/12] Use re-exported arrow --- datafusion-cli/Cargo.toml | 1 - datafusion-cli/src/main.rs | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/datafusion-cli/Cargo.toml b/datafusion-cli/Cargo.toml index 0d241b1875fea..7bc14361f7a95 100644 --- a/datafusion-cli/Cargo.toml +++ b/datafusion-cli/Cargo.toml @@ -30,5 +30,4 @@ repository = "https://github.com/apache/arrow-datafusion" clap = "2.33" rustyline ="8.0" tokio = { version = "1.0", features = ["macros", "rt", "rt-multi-thread", "sync"] } -arrow = { git = "https://github.com/apache/arrow-rs", rev = "d008f31b107c1030a1f5144c164e8ca8bf543576", features = ["prettyprint"] } datafusion = { path = "../datafusion" } diff --git a/datafusion-cli/src/main.rs b/datafusion-cli/src/main.rs index 634a3d879c884..d5c9c794d675d 100644 --- a/datafusion-cli/src/main.rs +++ b/datafusion-cli/src/main.rs @@ -17,7 +17,7 @@ #![allow(bare_trait_objects)] -use arrow::util::pretty; +use datafusion::arrow::util::pretty; use clap::{crate_version, App, Arg}; use datafusion::error::Result; use datafusion::execution::context::{ExecutionConfig, ExecutionContext}; From 05537205d63f026b2406738dc38b185fcadb58d1 Mon Sep 17 00:00:00 2001 From: "Heres, Daniel" Date: Sat, 1 May 2021 13:33:49 +0200 Subject: [PATCH 07/12] fmt --- datafusion-cli/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datafusion-cli/src/main.rs b/datafusion-cli/src/main.rs index d5c9c794d675d..5179db117b188 100644 --- a/datafusion-cli/src/main.rs +++ b/datafusion-cli/src/main.rs @@ -17,8 +17,8 @@ #![allow(bare_trait_objects)] -use datafusion::arrow::util::pretty; use clap::{crate_version, App, Arg}; +use datafusion::arrow::util::pretty; use datafusion::error::Result; use datafusion::execution::context::{ExecutionConfig, ExecutionContext}; use rustyline::Editor; From 280d0a4cea33bcfe43d0e19140f0021969487d40 Mon Sep 17 00:00:00 2001 From: "Heres, Daniel" Date: Sat, 1 May 2021 13:36:15 +0200 Subject: [PATCH 08/12] Spacing --- datafusion-cli/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datafusion-cli/Cargo.toml b/datafusion-cli/Cargo.toml index 7bc14361f7a95..883d0f2f4c66b 100644 --- a/datafusion-cli/Cargo.toml +++ b/datafusion-cli/Cargo.toml @@ -28,6 +28,6 @@ repository = "https://github.com/apache/arrow-datafusion" [dependencies] clap = "2.33" -rustyline ="8.0" +rustyline = "8.0" tokio = { version = "1.0", features = ["macros", "rt", "rt-multi-thread", "sync"] } datafusion = { path = "../datafusion" } From 4a6ec49313ca3a7258417765b36355226a9563ee Mon Sep 17 00:00:00 2001 From: "Heres, Daniel" Date: Sat, 1 May 2021 15:38:55 +0200 Subject: [PATCH 09/12] Inherit datafusion default --- datafusion-cli/src/main.rs | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/datafusion-cli/src/main.rs b/datafusion-cli/src/main.rs index 5179db117b188..dd7265e1f707e 100644 --- a/datafusion-cli/src/main.rs +++ b/datafusion-cli/src/main.rs @@ -44,7 +44,7 @@ pub async fn main() { ) .arg( Arg::with_name("batch-size") - .help("The batch size of each query, default value is 8192") + .help("The batch size of each query, or use DataFusion default") .short("c") .long("batch-size") .takes_value(true), @@ -56,16 +56,17 @@ pub async fn main() { env::set_current_dir(&p).unwrap(); }; - let batch_size = matches + let mut execution_config = ExecutionConfig::new().with_information_schema(true); + + if let Some(batch_size) = matches .value_of("batch-size") - .map(|size| size.parse::().unwrap()) - .unwrap_or(8192); - - let mut ctx = ExecutionContext::with_config( - ExecutionConfig::new() - .with_batch_size(batch_size) - .with_information_schema(true), - ); + .and_then(|size| size.parse::().ok()) + { + execution_config = execution_config.with_batch_size(batch_size); + }; + + let mut ctx = + ExecutionContext::with_config(execution_config.with_information_schema(true)); let mut rl = Editor::<()>::new(); rl.load_history(".history").ok(); From 1044fa9e9df7b849823c6d5ad4dfd50a66e6ccc2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20Heres?= Date: Sat, 1 May 2021 15:40:58 +0200 Subject: [PATCH 10/12] Update datafusion/docs/cli.md Co-authored-by: Andy Grove --- datafusion/docs/cli.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datafusion/docs/cli.md b/datafusion/docs/cli.md index 5c9ca6616de5b..8f937b2ff7a70 100644 --- a/datafusion/docs/cli.md +++ b/datafusion/docs/cli.md @@ -27,7 +27,7 @@ Use the following commands to clone this repository and run the CLI. This will r ```sh git clone https://github.com/apache/arrow-datafusion -cd arrow/datafusion-cli +cd arrow-datafusion/datafusion-cli cargo run --release ``` From 27419be824976b8a8e57057f1b008532459c3c69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20Heres?= Date: Sat, 1 May 2021 15:41:05 +0200 Subject: [PATCH 11/12] Update datafusion/docs/cli.md Co-authored-by: Andy Grove --- datafusion/docs/cli.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datafusion/docs/cli.md b/datafusion/docs/cli.md index 8f937b2ff7a70..c8d9023127ebe 100644 --- a/datafusion/docs/cli.md +++ b/datafusion/docs/cli.md @@ -37,7 +37,7 @@ Use the following commands to clone this repository and build a Docker image con ```sh git clone https://github.com/apache/arrow-datafusion -cd arrow +cd arrow-datafusion docker build -f datafusion/Dockerfile . --tag datafusion-cli docker run -it -v $(your_data_location):/data datafusion-cli ``` From 220fcf0ddf1de94aed19c3fcbad5aff79452b80c Mon Sep 17 00:00:00 2001 From: "Heres, Daniel" Date: Sat, 1 May 2021 16:24:58 +0200 Subject: [PATCH 12/12] Update docker setup --- .dockerignore | 30 ++--------------------- {datafusion => datafusion-cli}/Dockerfile | 13 ++++++---- datafusion/docs/cli.md | 2 +- 3 files changed, 11 insertions(+), 34 deletions(-) rename {datafusion => datafusion-cli}/Dockerfile (83%) diff --git a/.dockerignore b/.dockerignore index 8cd6a89645c3b..533221e2b4a69 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,28 +1,2 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Turn .dockerignore to .dockerallow by excluding everything and explicitly -# allowing specific files and directories. This enables us to quickly add -# dependency files to the docker content without scanning the whole directory. -# This setup requires to all of our docker containers have arrow's source -# as a mounted directory. - -ci -dev -testing -parquet-testing -**/target/* +.git +**target diff --git a/datafusion/Dockerfile b/datafusion-cli/Dockerfile similarity index 83% rename from datafusion/Dockerfile rename to datafusion-cli/Dockerfile index 97e82b4bbca20..32bd38b9419dc 100644 --- a/datafusion/Dockerfile +++ b/datafusion-cli/Dockerfile @@ -15,11 +15,14 @@ # specific language governing permissions and limitations # under the License. -FROM rustlang/rust:nightly +FROM rust:latest + + +COPY ./datafusion ./usr/src/datafusion +COPY ./datafusion-cli ./usr/src/datafusion-cli + +WORKDIR /usr/src/datafusion-cli +RUN cargo install --path . -COPY format /arrow/format/ -COPY rust /arrow/rust/ -WORKDIR /arrow/rust/datafusion -RUN cargo install --bin datafusion-cli --path . CMD ["datafusion-cli", "--data-path", "/data"] diff --git a/datafusion/docs/cli.md b/datafusion/docs/cli.md index c8d9023127ebe..27605b2e98c0f 100644 --- a/datafusion/docs/cli.md +++ b/datafusion/docs/cli.md @@ -38,7 +38,7 @@ Use the following commands to clone this repository and build a Docker image con ```sh git clone https://github.com/apache/arrow-datafusion cd arrow-datafusion -docker build -f datafusion/Dockerfile . --tag datafusion-cli +docker build -f datafusion-cli/Dockerfile . --tag datafusion-cli docker run -it -v $(your_data_location):/data datafusion-cli ```