Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 9 additions & 32 deletions datafusion/core/src/physical_plan/file_format/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ use crate::{
scalar::ScalarValue,
};
use arrow::array::{new_null_array, UInt16BufferBuilder};
use arrow::record_batch::RecordBatchOptions;
use datafusion_data_access::object_store::ObjectStore;
use lazy_static::lazy_static;
use log::info;
Expand Down Expand Up @@ -275,9 +276,15 @@ impl SchemaAdapter {

let projected_schema = Arc::new(self.table_schema.clone().project(projections)?);

let merged_batch = RecordBatch::try_new(projected_schema, cols)?;
// Necessary to handle empty batches
let mut options = RecordBatchOptions::default();
options.row_count = Some(batch.num_rows());

Ok(merged_batch)
Ok(RecordBatch::try_new_with_options(
projected_schema,
cols,
&options,
)?)
}
}

Expand Down Expand Up @@ -322,36 +329,6 @@ impl PartitionColumnProjector {
}
}

// Creates a RecordBatch with values from the partition_values. Used when no non-partition values are read
fn project_from_size(
&mut self,
batch_size: usize,
partition_values: &[ScalarValue],
) -> ArrowResult<RecordBatch> {
let expected_cols = self.projected_schema.fields().len();
if expected_cols != self.projected_partition_indexes.len() {
return Err(ArrowError::SchemaError(format!(
"Unexepected number of partition values, expected {} but got {}",
expected_cols,
partition_values.len()
)));
}
//The destination index is not needed. Since there are no non-partition columns it will simply be equivalent to
//the index that would be provided by .enumerate()
let cols = self
.projected_partition_indexes
.iter()
.map(|(pidx, _)| {
create_dict_array(
&mut self.key_buffer_cache,
&partition_values[*pidx],
batch_size,
)
})
.collect();
RecordBatch::try_new(Arc::clone(&self.projected_schema), cols)
}

// Transform the batch read from the file by inserting the partitioning columns
// to the right positions as deduced from `projected_schema`
// - file_batch: batch read from the file, with internal projection applied
Expand Down
70 changes: 16 additions & 54 deletions datafusion/core/src/physical_plan/file_format/parquet.rs
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,6 @@ use parquet::arrow::{
arrow_reader::ParquetRecordBatchReader, ArrowReader, ArrowWriter,
ParquetFileArrowReader,
};
use parquet::file::reader::FileReader;
use parquet::file::{
metadata::RowGroupMetaData, properties::WriterProperties,
reader::SerializedFileReader, serialized_reader::ReadOptionsBuilder,
Expand Down Expand Up @@ -285,19 +284,6 @@ impl ExecutionPlan for ParquetExec {
}
}

/// Special-case empty column projection
///
/// This is a workaround for https://github.com/apache/arrow-rs/issues/1537
enum ProjectedReader {
Reader {
reader: ParquetRecordBatchReader,
},
EmptyProjection {
remaining_rows: usize,
batch_size: usize,
},
}

/// Implements [`RecordBatchStream`] for a collection of [`PartitionedFile`]
///
/// NB: This will perform blocking IO synchronously without yielding which may
Expand All @@ -313,14 +299,17 @@ struct ParquetExecStream {
schema: SchemaRef,
projection: Vec<usize>,
remaining_rows: Option<usize>,
reader: Option<(ProjectedReader, PartitionedFile)>,
reader: Option<(ParquetRecordBatchReader, PartitionedFile)>,
files: VecDeque<PartitionedFile>,
projector: PartitionColumnProjector,
adapter: SchemaAdapter,
}

impl ParquetExecStream {
fn create_reader(&mut self, file: &PartitionedFile) -> Result<ProjectedReader> {
fn create_reader(
&mut self,
file: &PartitionedFile,
) -> Result<ParquetRecordBatchReader> {
let file_metrics = ParquetFileMetrics::new(
self.partition_index,
file.file_meta.path(),
Expand Down Expand Up @@ -351,20 +340,6 @@ impl ParquetExecStream {
opt.build(),
)?;

if self.projection.is_empty() {
let remaining_rows = file_reader
.metadata()
.file_metadata()
.num_rows()
.try_into()
.expect("Row count should always be greater than or equal to 0 and less than usize::MAX");

return Ok(ProjectedReader::EmptyProjection {
remaining_rows,
batch_size: self.batch_size,
});
}

let mut arrow_reader = ParquetFileArrowReader::new(Arc::new(file_reader));

let adapted_projections = self
Expand All @@ -374,7 +349,7 @@ impl ParquetExecStream {
let reader = arrow_reader
.get_record_reader_by_columns(adapted_projections, self.batch_size)?;

Ok(ProjectedReader::Reader { reader })
Ok(reader)
}
}

Expand Down Expand Up @@ -402,30 +377,17 @@ impl Iterator for ParquetExecStream {
},
};

let result = match reader {
ProjectedReader::Reader { reader } => reader.next().map(|result| {
result
.and_then(|batch| {
self.adapter
.adapt_batch(batch, &self.projection)
.map_err(|e| ArrowError::ExternalError(Box::new(e)))
})
.and_then(|batch| {
self.projector.project(batch, &file.partition_values)
})
}),
ProjectedReader::EmptyProjection {
remaining_rows,
batch_size,
} => {
let size = *remaining_rows.min(batch_size);
*remaining_rows -= size;
(size != 0).then(|| {
self.projector
.project_from_size(size, &file.partition_values)
let result = reader.next().map(|result| {
result
.and_then(|batch| {
self.adapter
.adapt_batch(batch, &self.projection)
.map_err(|e| ArrowError::ExternalError(Box::new(e)))
})
}
};
.and_then(|batch| {
self.projector.project(batch, &file.partition_values)
})
});

let result = match result {
Some(result) => result,
Expand Down