From dddaefe0ff0de1a6098ad12839973e02be347475 Mon Sep 17 00:00:00 2001 From: comphead Date: Mon, 5 Sep 2022 09:38:55 -0700 Subject: [PATCH 1/9] removed panics from proto crate --- datafusion/proto/build.rs | 9 ++++----- datafusion/proto/src/bytes/mod.rs | 2 +- datafusion/proto/src/from_proto.rs | 13 ++++++++----- datafusion/proto/src/logical_plan.rs | 1 + datafusion/proto/src/to_proto.rs | 24 +++++++++++------------- 5 files changed, 25 insertions(+), 24 deletions(-) diff --git a/datafusion/proto/build.rs b/datafusion/proto/build.rs index e13ffa86a53ce..7c20b8852a52a 100644 --- a/datafusion/proto/build.rs +++ b/datafusion/proto/build.rs @@ -30,8 +30,8 @@ fn main() -> Result<(), String> { #[cfg(feature = "json")] fn build() -> Result<(), String> { - let descriptor_path = std::path::PathBuf::from(std::env::var("OUT_DIR").unwrap()) - .join("proto_descriptor.bin"); + let descriptor_path = + std::path::PathBuf::from(std::env::var("OUT_DIR")?).join("proto_descriptor.bin"); prost_build::Config::new() .file_descriptor_set_path(&descriptor_path) @@ -40,10 +40,9 @@ fn build() -> Result<(), String> { .compile_protos(&["proto/datafusion.proto"], &["proto"]) .map_err(|e| format!("protobuf compilation failed: {}", e))?; - let descriptor_set = std::fs::read(descriptor_path).unwrap(); + let descriptor_set = std::fs::read(descriptor_path)?; pbjson_build::Builder::new() - .register_descriptors(&descriptor_set) - .unwrap() + .register_descriptors(&descriptor_set)? .build(&[".datafusion"]) .map_err(|e| format!("pbjson compilation failed: {}", e))?; diff --git a/datafusion/proto/src/bytes/mod.rs b/datafusion/proto/src/bytes/mod.rs index 6ddaecae098f0..9e815e052053c 100644 --- a/datafusion/proto/src/bytes/mod.rs +++ b/datafusion/proto/src/bytes/mod.rs @@ -137,7 +137,7 @@ pub fn logical_plan_to_bytes_with_extension_codec( /// Deserialize a LogicalPlan from json #[cfg(feature = "json")] pub fn logical_plan_from_json(json: &str, ctx: &SessionContext) -> Result { - let back: protobuf::LogicalPlanNode = serde_json::from_str(json).unwrap(); + let back: protobuf::LogicalPlanNode = serde_json::from_str(json)?; let extension_codec = DefaultExtensionCodec {}; back.try_into_logical_plan(ctx, &extension_codec) } diff --git a/datafusion/proto/src/from_proto.rs b/datafusion/proto/src/from_proto.rs index 0a4e0e68bbe1a..f40045766a0b4 100644 --- a/datafusion/proto/src/from_proto.rs +++ b/datafusion/proto/src/from_proto.rs @@ -378,11 +378,13 @@ impl From<&protobuf::StringifiedPlan> for StringifiedPlan { plan_type: match stringified_plan .plan_type .as_ref() - .unwrap() - .plan_type_enum - .as_ref() - .unwrap() - { + .and_then(|pt| pt.plan_type_enum.as_ref()) + .unwrap_or_else(|| { + panic!( + "Cannot create protobuf::StringifiedPlan from {:?}", + stringified_plan + ) + }) { InitialLogicalPlan(_) => PlanType::InitialLogicalPlan, OptimizedLogicalPlan(OptimizedLogicalPlanType { optimizer_name }) => { PlanType::OptimizedLogicalPlan { @@ -1341,6 +1343,7 @@ impl From for IntervalUnit { } } +// panic here bcos no better way to convert from Vec to Array fn vec_to_array(v: Vec) -> [T; N] { v.try_into().unwrap_or_else(|v: Vec| { panic!("Expected a Vec of length {} but it was {}", N, v.len()) diff --git a/datafusion/proto/src/logical_plan.rs b/datafusion/proto/src/logical_plan.rs index bcdb9796f17ee..857fdcb20d853 100644 --- a/datafusion/proto/src/logical_plan.rs +++ b/datafusion/proto/src/logical_plan.rs @@ -653,6 +653,7 @@ impl AsLogicalPlan for LogicalPlanNode { ))); } + // unwrap is safe here because of len check above let mut builder = LogicalPlanBuilder::from(input_plans.pop().unwrap()); for plan in input_plans { builder = builder.union(plan)?; diff --git a/datafusion/proto/src/to_proto.rs b/datafusion/proto/src/to_proto.rs index e64096a4e482d..a090ef16ae55d 100644 --- a/datafusion/proto/src/to_proto.rs +++ b/datafusion/proto/src/to_proto.rs @@ -224,10 +224,12 @@ impl From<&DataType> for protobuf::arrow_type::ArrowTypeEnum { fractional: *fractional as u64, }), DataType::Decimal256(_, _) => { - unimplemented!("The Decimal256 data type is not yet supported") + unimplemented!("Proto serialization error: The Decimal256 data type is not yet supported") } DataType::Map(_, _) => { - unimplemented!("The Map data type is not yet supported") + unimplemented!( + "Proto serialization error: The Map data type is not yet supported" + ) } } } @@ -612,7 +614,7 @@ impl TryFrom<&Expr> for protobuf::LogicalExprNode { expr_type: Some(ExprType::AggregateExpr(aggregate_expr)), } } - Expr::ScalarVariable(_, _) => unimplemented!(), + Expr::ScalarVariable(_, _) => return Err(Error::General("Proto serialization error: Scalar Variable not supported".to_string())), Expr::ScalarFunction { ref fun, ref args } => { let fun: protobuf::ScalarFunction = fun.try_into()?; let args: Vec = args @@ -820,7 +822,7 @@ impl TryFrom<&Expr> for protobuf::LogicalExprNode { Expr::ScalarSubquery(_) | Expr::InSubquery { .. } | Expr::Exists { .. } => { // we would need to add logical plan operators to datafusion.proto to support this // see discussion in https://github.com/apache/arrow-datafusion/issues/2565 - unimplemented!("subquery expressions are not supported yet") + return Err(Error::General("Proto serialization error: Expr::ScalarSubquery(_) | Expr::InSubquery { .. } | Expr::Exists { .. } not supported".to_string())) } Expr::GetIndexedField { key, expr } => Self { expr_type: Some(ExprType::GetIndexedField(Box::new( @@ -865,7 +867,8 @@ impl TryFrom<&Expr> for protobuf::LogicalExprNode { })), }, - Expr::QualifiedWildcard { .. } | Expr::TryCast { .. } => unimplemented!(), + Expr::QualifiedWildcard { .. } | Expr::TryCast { .. } => + return Err(Error::General("Proto serialization error: Expr::QualifiedWildcard { .. } | Expr::TryCast { .. }".to_string())), }; Ok(expr_node) @@ -946,7 +949,7 @@ impl TryFrom<&ScalarValue> for protobuf::ScalarValue { }) } scalar::ScalarValue::List(value, boxed_field) => { - println!("Current field of list: {:?}", boxed_field); + //println!("Current field of list: {:?}", boxed_field); match value { Some(values) => { if values.is_empty() { @@ -959,13 +962,8 @@ impl TryFrom<&ScalarValue> for protobuf::ScalarValue { )), } } else { - let scalar_type = match boxed_field.data_type() { - DataType::List(field) => field.as_ref().data_type(), - unsupported => { - todo!("Proper error handling {}", unsupported) - } - }; - println!("Current scalar type for list: {:?}", scalar_type); + let scalar_type = boxed_field.data_type(); + //println!("Current scalar type for list: {:?}", scalar_type); let type_checked_values: Vec = values .iter() From 0c6c5937cd5179dfd67092ed85f07e5e92010644 Mon Sep 17 00:00:00 2001 From: comphead Date: Mon, 5 Sep 2022 11:16:56 -0700 Subject: [PATCH 2/9] fixes --- datafusion/proto/src/logical_plan.rs | 3 ++- datafusion/proto/src/to_proto.rs | 9 +++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/datafusion/proto/src/logical_plan.rs b/datafusion/proto/src/logical_plan.rs index 480950d5cc8a2..03b35381ea5e1 100644 --- a/datafusion/proto/src/logical_plan.rs +++ b/datafusion/proto/src/logical_plan.rs @@ -385,7 +385,8 @@ impl AsLogicalPlan for LogicalPlanNode { let table_paths = &scan .paths .iter() - .map(|p| ListingTableUrl::parse(p).unwrap()) + .filter_map(|p| ListingTableUrl::parse(p).ok()) + //.map(|p| ListingTableUrl::parse(p).unwrap()) .collect::>(); let options = ListingOptions { file_extension: scan.file_extension.clone(), diff --git a/datafusion/proto/src/to_proto.rs b/datafusion/proto/src/to_proto.rs index a090ef16ae55d..d05654a0fa633 100644 --- a/datafusion/proto/src/to_proto.rs +++ b/datafusion/proto/src/to_proto.rs @@ -868,7 +868,7 @@ impl TryFrom<&Expr> for protobuf::LogicalExprNode { }, Expr::QualifiedWildcard { .. } | Expr::TryCast { .. } => - return Err(Error::General("Proto serialization error: Expr::QualifiedWildcard { .. } | Expr::TryCast { .. }".to_string())), + return Err(Error::General("Proto serialization error: Expr::QualifiedWildcard { .. } | Expr::TryCast { .. } not supported".to_string())), }; Ok(expr_node) @@ -962,7 +962,12 @@ impl TryFrom<&ScalarValue> for protobuf::ScalarValue { )), } } else { - let scalar_type = boxed_field.data_type(); + let scalar_type = match boxed_field.data_type() { + DataType::List(field) => field.as_ref().data_type(), + unsupported => { + return Err(Error::General(format!("Proto serialization error: {:?} not supported to convert to DataType::List", unsupported))); + } + }; //println!("Current scalar type for list: {:?}", scalar_type); let type_checked_values: Vec = values From 227e0e0f6c4b094f6b867b5aa4b4fdaf072ac9a5 Mon Sep 17 00:00:00 2001 From: comphead Date: Tue, 6 Sep 2022 17:13:38 -0700 Subject: [PATCH 3/9] fixed comments --- datafusion/proto/build.rs | 28 ++-- datafusion/proto/src/logical_plan.rs | 7 +- datafusion/proto/src/to_proto.rs | 210 +++++++++++++-------------- 3 files changed, 113 insertions(+), 132 deletions(-) diff --git a/datafusion/proto/build.rs b/datafusion/proto/build.rs index 7511e511d24c3..893f65fc79483 100644 --- a/datafusion/proto/build.rs +++ b/datafusion/proto/build.rs @@ -30,9 +30,9 @@ fn main() -> Result<(), String> { #[cfg(feature = "json")] fn build() -> Result<(), String> { - use std::io::Write; - - let out = std::path::PathBuf::from(std::env::var("OUT_DIR")?); + let out = std::path::PathBuf::from( + std::env::var("OUT_DIR").expect("Cannot find OUT_DIR environment vairable"), + ); let descriptor_path = out.join("proto_descriptor.bin"); prost_build::Config::new() @@ -42,30 +42,24 @@ fn build() -> Result<(), String> { .compile_protos(&["proto/datafusion.proto"], &["proto"]) .map_err(|e| format!("protobuf compilation failed: {}", e))?; - let descriptor_set = std::fs::read(descriptor_path)?; + let descriptor_set = std::fs::read(&descriptor_path) + .expect(&*format!("Cannot read {:?}", &descriptor_path)); + pbjson_build::Builder::new() - .register_descriptors(&descriptor_set)? + .register_descriptors(&descriptor_set) + .expect(&*format!( + "Cannot register descriptors {:?}", + &descriptor_set + )) .build(&[".datafusion"]) .map_err(|e| format!("pbjson compilation failed: {}", e))?; - // .serde.rs is not a valid package name, so append to datafusion.rs so we can treat it normally - let proto = std::fs::read_to_string(out.join("datafusion.rs")).unwrap(); - let json = std::fs::read_to_string(out.join("datafusion.serde.rs")).unwrap(); - let mut file = std::fs::OpenOptions::new() - .write(true) - .create(true) - .open("src/generated/datafusion.rs") - .unwrap(); - file.write(proto.as_str().as_ref()).unwrap(); - file.write(json.as_str().as_ref()).unwrap(); - Ok(()) } #[cfg(not(feature = "json"))] fn build() -> Result<(), String> { prost_build::Config::new() - .out_dir("src/generated") .compile_protos(&["proto/datafusion.proto"], &["proto"]) .map_err(|e| format!("protobuf compilation failed: {}", e)) } diff --git a/datafusion/proto/src/logical_plan.rs b/datafusion/proto/src/logical_plan.rs index 03b35381ea5e1..6f7646267bd10 100644 --- a/datafusion/proto/src/logical_plan.rs +++ b/datafusion/proto/src/logical_plan.rs @@ -381,13 +381,12 @@ impl AsLogicalPlan for LogicalPlanNode { FileFormatType::Avro(..) => Arc::new(AvroFormat::default()), }; - // let table_path = ListingTableUrl::parse(&scan.paths)?; let table_paths = &scan .paths .iter() - .filter_map(|p| ListingTableUrl::parse(p).ok()) - //.map(|p| ListingTableUrl::parse(p).unwrap()) - .collect::>(); + .map(ListingTableUrl::parse) + .collect::, _>>()?; + let options = ListingOptions { file_extension: scan.file_extension.clone(), format: file_format, diff --git a/datafusion/proto/src/to_proto.rs b/datafusion/proto/src/to_proto.rs index d05654a0fa633..ef7e5bf219f59 100644 --- a/datafusion/proto/src/to_proto.rs +++ b/datafusion/proto/src/to_proto.rs @@ -948,127 +948,115 @@ impl TryFrom<&ScalarValue> for protobuf::ScalarValue { Value::LargeUtf8Value(s.to_owned()) }) } - scalar::ScalarValue::List(value, boxed_field) => { - //println!("Current field of list: {:?}", boxed_field); - match value { - Some(values) => { - if values.is_empty() { - protobuf::ScalarValue { - value: Some(protobuf::scalar_value::Value::ListValue( - protobuf::ScalarListValue { - field: Some(boxed_field.as_ref().into()), - values: Vec::new(), - }, - )), + scalar::ScalarValue::List(value, boxed_field) => match value { + Some(values) => { + if values.is_empty() { + protobuf::ScalarValue { + value: Some(protobuf::scalar_value::Value::ListValue( + protobuf::ScalarListValue { + field: Some(boxed_field.as_ref().into()), + values: Vec::new(), + }, + )), + } + } else { + let scalar_type = match boxed_field.data_type() { + DataType::List(field) => field.as_ref().data_type(), + unsupported => { + return Err(Error::General(format!("Proto serialization error: {:?} not supported to convert to DataType::List", unsupported))); } - } else { - let scalar_type = match boxed_field.data_type() { - DataType::List(field) => field.as_ref().data_type(), - unsupported => { - return Err(Error::General(format!("Proto serialization error: {:?} not supported to convert to DataType::List", unsupported))); - } - }; - //println!("Current scalar type for list: {:?}", scalar_type); + }; - let type_checked_values: Vec = values - .iter() - .map(|scalar| match (scalar, scalar_type) { - ( - scalar::ScalarValue::List(_, list_type), - DataType::List(field), - ) => { - if let DataType::List(list_field) = - list_type.data_type() + let type_checked_values: Vec = values + .iter() + .map(|scalar| match (scalar, scalar_type) { + ( + scalar::ScalarValue::List(_, list_type), + DataType::List(field), + ) => { + if let DataType::List(list_field) = + list_type.data_type() + { + let scalar_datatype = field.data_type(); + let list_datatype = list_field.data_type(); + if std::mem::discriminant(list_datatype) + != std::mem::discriminant(scalar_datatype) { - let scalar_datatype = field.data_type(); - let list_datatype = list_field.data_type(); - if std::mem::discriminant(list_datatype) - != std::mem::discriminant(scalar_datatype) - { - return Err( - Error::inconsistent_list_typing( - list_datatype, - scalar_datatype, - ), - ); - } - scalar.try_into() - } else { - Err(Error::inconsistent_list_designated( - scalar, - boxed_field.data_type(), - )) + return Err(Error::inconsistent_list_typing( + list_datatype, + scalar_datatype, + )); } - } - ( - scalar::ScalarValue::Boolean(_), - DataType::Boolean, - ) => scalar.try_into(), - ( - scalar::ScalarValue::Float32(_), - DataType::Float32, - ) => scalar.try_into(), - ( - scalar::ScalarValue::Float64(_), - DataType::Float64, - ) => scalar.try_into(), - (scalar::ScalarValue::Int8(_), DataType::Int8) => { - scalar.try_into() - } - (scalar::ScalarValue::Int16(_), DataType::Int16) => { - scalar.try_into() - } - (scalar::ScalarValue::Int32(_), DataType::Int32) => { - scalar.try_into() - } - (scalar::ScalarValue::Int64(_), DataType::Int64) => { - scalar.try_into() - } - (scalar::ScalarValue::UInt8(_), DataType::UInt8) => { - scalar.try_into() - } - ( - scalar::ScalarValue::UInt16(_), - DataType::UInt16, - ) => scalar.try_into(), - ( - scalar::ScalarValue::UInt32(_), - DataType::UInt32, - ) => scalar.try_into(), - ( - scalar::ScalarValue::UInt64(_), - DataType::UInt64, - ) => scalar.try_into(), - (scalar::ScalarValue::Utf8(_), DataType::Utf8) => { scalar.try_into() + } else { + Err(Error::inconsistent_list_designated( + scalar, + boxed_field.data_type(), + )) } - ( - scalar::ScalarValue::LargeUtf8(_), - DataType::LargeUtf8, - ) => scalar.try_into(), - _ => Err(Error::inconsistent_list_designated( - scalar, - boxed_field.data_type(), - )), - }) - .collect::, _>>()?; - protobuf::ScalarValue { - value: Some(protobuf::scalar_value::Value::ListValue( - protobuf::ScalarListValue { - field: Some(boxed_field.as_ref().into()), - values: type_checked_values, - }, + } + (scalar::ScalarValue::Boolean(_), DataType::Boolean) => { + scalar.try_into() + } + (scalar::ScalarValue::Float32(_), DataType::Float32) => { + scalar.try_into() + } + (scalar::ScalarValue::Float64(_), DataType::Float64) => { + scalar.try_into() + } + (scalar::ScalarValue::Int8(_), DataType::Int8) => { + scalar.try_into() + } + (scalar::ScalarValue::Int16(_), DataType::Int16) => { + scalar.try_into() + } + (scalar::ScalarValue::Int32(_), DataType::Int32) => { + scalar.try_into() + } + (scalar::ScalarValue::Int64(_), DataType::Int64) => { + scalar.try_into() + } + (scalar::ScalarValue::UInt8(_), DataType::UInt8) => { + scalar.try_into() + } + (scalar::ScalarValue::UInt16(_), DataType::UInt16) => { + scalar.try_into() + } + (scalar::ScalarValue::UInt32(_), DataType::UInt32) => { + scalar.try_into() + } + (scalar::ScalarValue::UInt64(_), DataType::UInt64) => { + scalar.try_into() + } + (scalar::ScalarValue::Utf8(_), DataType::Utf8) => { + scalar.try_into() + } + ( + scalar::ScalarValue::LargeUtf8(_), + DataType::LargeUtf8, + ) => scalar.try_into(), + _ => Err(Error::inconsistent_list_designated( + scalar, + boxed_field.data_type(), )), - } + }) + .collect::, _>>()?; + protobuf::ScalarValue { + value: Some(protobuf::scalar_value::Value::ListValue( + protobuf::ScalarListValue { + field: Some(boxed_field.as_ref().into()), + values: type_checked_values, + }, + )), } } - None => protobuf::ScalarValue { - value: Some(protobuf::scalar_value::Value::NullListValue( - boxed_field.as_ref().try_into()?, - )), - }, } - } + None => protobuf::ScalarValue { + value: Some(protobuf::scalar_value::Value::NullListValue( + boxed_field.as_ref().try_into()?, + )), + }, + }, datafusion::scalar::ScalarValue::Date32(val) => { create_proto_scalar(val, PrimitiveScalarType::Date32, |s| { Value::Date32Value(*s) From 713c5a0048b04679eb8b095f9f68c404353dbe2e Mon Sep 17 00:00:00 2001 From: comphead Date: Tue, 6 Sep 2022 17:19:35 -0700 Subject: [PATCH 4/9] fix build --- datafusion/proto/build.rs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/datafusion/proto/build.rs b/datafusion/proto/build.rs index 893f65fc79483..333d8ceb4d9f7 100644 --- a/datafusion/proto/build.rs +++ b/datafusion/proto/build.rs @@ -30,6 +30,8 @@ fn main() -> Result<(), String> { #[cfg(feature = "json")] fn build() -> Result<(), String> { + use std::io::Write; + let out = std::path::PathBuf::from( std::env::var("OUT_DIR").expect("Cannot find OUT_DIR environment vairable"), ); @@ -54,12 +56,24 @@ fn build() -> Result<(), String> { .build(&[".datafusion"]) .map_err(|e| format!("pbjson compilation failed: {}", e))?; + // .serde.rs is not a valid package name, so append to datafusion.rs so we can treat it normally + let proto = std::fs::read_to_string(out.join("datafusion.rs")).unwrap(); + let json = std::fs::read_to_string(out.join("datafusion.serde.rs")).unwrap(); + let mut file = std::fs::OpenOptions::new() + .write(true) + .create(true) + .open("src/generated/datafusion.rs") + .unwrap(); + file.write(proto.as_str().as_ref()).unwrap(); + file.write(json.as_str().as_ref()).unwrap(); + Ok(()) } #[cfg(not(feature = "json"))] fn build() -> Result<(), String> { prost_build::Config::new() + .out_dir("src/generated") .compile_protos(&["proto/datafusion.proto"], &["proto"]) .map_err(|e| format!("protobuf compilation failed: {}", e)) } From 38afffd30559ec45461ee51f2ef78bde2a31e57c Mon Sep 17 00:00:00 2001 From: comphead Date: Tue, 6 Sep 2022 17:33:52 -0700 Subject: [PATCH 5/9] fixed comments --- datafusion/proto/build.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datafusion/proto/build.rs b/datafusion/proto/build.rs index 333d8ceb4d9f7..c6e9595d41ec2 100644 --- a/datafusion/proto/build.rs +++ b/datafusion/proto/build.rs @@ -31,7 +31,7 @@ fn main() -> Result<(), String> { #[cfg(feature = "json")] fn build() -> Result<(), String> { use std::io::Write; - + let out = std::path::PathBuf::from( std::env::var("OUT_DIR").expect("Cannot find OUT_DIR environment vairable"), ); From 3512cdd6c1b938d089d2ac0aea8fa361a60e1b2b Mon Sep 17 00:00:00 2001 From: comphead Date: Wed, 7 Sep 2022 10:44:32 -0700 Subject: [PATCH 6/9] Fixing proto json --- datafusion/proto/src/bytes/mod.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/datafusion/proto/src/bytes/mod.rs b/datafusion/proto/src/bytes/mod.rs index 9e815e052053c..ea56776014f30 100644 --- a/datafusion/proto/src/bytes/mod.rs +++ b/datafusion/proto/src/bytes/mod.rs @@ -137,7 +137,10 @@ pub fn logical_plan_to_bytes_with_extension_codec( /// Deserialize a LogicalPlan from json #[cfg(feature = "json")] pub fn logical_plan_from_json(json: &str, ctx: &SessionContext) -> Result { - let back: protobuf::LogicalPlanNode = serde_json::from_str(json)?; + let back: protobuf::LogicalPlanNode = match serde_json::from_str(json) { + Ok(res) => res, + Err(e) => return Err(DataFusionError::Plan(e.to_string())) + }; let extension_codec = DefaultExtensionCodec {}; back.try_into_logical_plan(ctx, &extension_codec) } From 8138d9ad1a31e1754dad20adc45a81cbfbf9c08b Mon Sep 17 00:00:00 2001 From: comphead Date: Wed, 7 Sep 2022 12:47:36 -0700 Subject: [PATCH 7/9] formatting --- datafusion/proto/src/bytes/mod.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/datafusion/proto/src/bytes/mod.rs b/datafusion/proto/src/bytes/mod.rs index ea56776014f30..47ab5e3d31f1f 100644 --- a/datafusion/proto/src/bytes/mod.rs +++ b/datafusion/proto/src/bytes/mod.rs @@ -137,10 +137,8 @@ pub fn logical_plan_to_bytes_with_extension_codec( /// Deserialize a LogicalPlan from json #[cfg(feature = "json")] pub fn logical_plan_from_json(json: &str, ctx: &SessionContext) -> Result { - let back: protobuf::LogicalPlanNode = match serde_json::from_str(json) { - Ok(res) => res, - Err(e) => return Err(DataFusionError::Plan(e.to_string())) - }; + let back: protobuf::LogicalPlanNode = serde_json::from_str(json) + .map_err(|e| DataFusionError::Plan(format!("Error deserializing plan: {}", e))); let extension_codec = DefaultExtensionCodec {}; back.try_into_logical_plan(ctx, &extension_codec) } From 4e6e92036dedb5d1a1c16d39355e5a4e3821b0ae Mon Sep 17 00:00:00 2001 From: comphead Date: Wed, 7 Sep 2022 13:24:25 -0700 Subject: [PATCH 8/9] fix build --- datafusion/proto/src/bytes/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datafusion/proto/src/bytes/mod.rs b/datafusion/proto/src/bytes/mod.rs index 47ab5e3d31f1f..1dd8a5c1a4035 100644 --- a/datafusion/proto/src/bytes/mod.rs +++ b/datafusion/proto/src/bytes/mod.rs @@ -138,7 +138,7 @@ pub fn logical_plan_to_bytes_with_extension_codec( #[cfg(feature = "json")] pub fn logical_plan_from_json(json: &str, ctx: &SessionContext) -> Result { let back: protobuf::LogicalPlanNode = serde_json::from_str(json) - .map_err(|e| DataFusionError::Plan(format!("Error deserializing plan: {}", e))); + .map_err(|e| DataFusionError::Plan(format!("Error serializing plan: {}", e)))?; let extension_codec = DefaultExtensionCodec {}; back.try_into_logical_plan(ctx, &extension_codec) } From 625fcf0d8faae7eeacd862ac7453c61fdd068efd Mon Sep 17 00:00:00 2001 From: comphead Date: Thu, 8 Sep 2022 10:06:17 -0700 Subject: [PATCH 9/9] addresed comments --- datafusion/proto/src/from_proto.rs | 2 +- datafusion/proto/src/logical_plan.rs | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/datafusion/proto/src/from_proto.rs b/datafusion/proto/src/from_proto.rs index f40045766a0b4..00ab7b16cb231 100644 --- a/datafusion/proto/src/from_proto.rs +++ b/datafusion/proto/src/from_proto.rs @@ -1343,7 +1343,7 @@ impl From for IntervalUnit { } } -// panic here bcos no better way to convert from Vec to Array +// panic here because no better way to convert from Vec to Array fn vec_to_array(v: Vec) -> [T; N] { v.try_into().unwrap_or_else(|v: Vec| { panic!("Expected a Vec of length {} but it was {}", N, v.len()) diff --git a/datafusion/proto/src/logical_plan.rs b/datafusion/proto/src/logical_plan.rs index 6f7646267bd10..3d6164e547aa5 100644 --- a/datafusion/proto/src/logical_plan.rs +++ b/datafusion/proto/src/logical_plan.rs @@ -632,8 +632,10 @@ impl AsLogicalPlan for LogicalPlanNode { ))); } - // unwrap is safe here because of len check above - let mut builder = LogicalPlanBuilder::from(input_plans.pop().unwrap()); + let first = input_plans.pop().ok_or_else(|| DataFusionError::Internal(String::from( + "Protobuf deserialization error, Union was require at least two input.", + )))?; + let mut builder = LogicalPlanBuilder::from(first); for plan in input_plans { builder = builder.union(plan)?; }