From b546cea92b3149cee619a9ad251a9b57045e99a8 Mon Sep 17 00:00:00 2001 From: srib Date: Sat, 3 Sep 2022 15:42:10 -0400 Subject: [PATCH 1/3] Replace panic with Result in reader.rs --- .../src/physical_plan/aggregates/row_hash.rs | 15 +++-- datafusion/row/src/reader.rs | 64 ++++++++++++++----- 2 files changed, 58 insertions(+), 21 deletions(-) diff --git a/datafusion/core/src/physical_plan/aggregates/row_hash.rs b/datafusion/core/src/physical_plan/aggregates/row_hash.rs index 2ef1d16d4fab4..e839e022bf361 100644 --- a/datafusion/core/src/physical_plan/aggregates/row_hash.rs +++ b/datafusion/core/src/physical_plan/aggregates/row_hash.rs @@ -440,14 +440,15 @@ fn create_batch_from_map( .unzip(); let mut columns: Vec = - read_as_batch(&group_buffers, group_schema, RowType::Compact); + read_as_batch(&group_buffers, group_schema, RowType::Compact) + .map_err(|e| ArrowError::ParseError(e.to_string()))?; match mode { AggregateMode::Partial => columns.extend(read_as_batch( &state_buffers, aggr_schema, RowType::WordAligned, - )), + )?), AggregateMode::Final | AggregateMode::FinalPartitioned => { let mut results: Vec> = vec![vec![]; accumulators.len()]; for buffer in state_buffers.iter_mut() { @@ -474,15 +475,19 @@ fn create_batch_from_map( RecordBatch::try_new(Arc::new(output_schema.to_owned()), columns) } -fn read_as_batch(rows: &[Vec], schema: &Schema, row_type: RowType) -> Vec { +fn read_as_batch( + rows: &[Vec], + schema: &Schema, + row_type: RowType, +) -> Result> { let row_num = rows.len(); let mut output = MutableRecordBatch::new(row_num, Arc::new(schema.clone())); let mut row = RowReader::new(schema, row_type); for data in rows { row.point_to(0, data); - read_row(&row, &mut output, schema); + read_row(&row, &mut output, schema)?; } - output.output_as_columns() + Ok(output.output_as_columns()) } diff --git a/datafusion/row/src/reader.rs b/datafusion/row/src/reader.rs index 634b814ad35ae..33dda4690ed86 100644 --- a/datafusion/row/src/reader.rs +++ b/datafusion/row/src/reader.rs @@ -55,7 +55,7 @@ pub fn read_as_batch( for offset in offsets.iter().take(row_num) { row.point_to(*offset, data); - read_row(&row, &mut output, &schema); + read_row(&row, &mut output, &schema)?; } output.output().map_err(DataFusionError::ArrowError) @@ -270,7 +270,11 @@ impl<'a> RowReader<'a> { } /// Read the row currently pointed by RowWriter to the output columnar batch buffer -pub fn read_row(row: &RowReader, batch: &mut MutableRecordBatch, schema: &Schema) { +pub fn read_row( + row: &RowReader, + batch: &mut MutableRecordBatch, + schema: &Schema, +) -> Result<()> { if row.all_valid() { for ((col_idx, to), field) in batch .arrays @@ -278,8 +282,9 @@ pub fn read_row(row: &RowReader, batch: &mut MutableRecordBatch, schema: &Schema .enumerate() .zip(schema.fields().iter()) { - read_field_null_free(to, field.data_type(), col_idx, row) + read_field_null_free(to, field.data_type(), col_idx, row)?; } + Ok(()) } else { for ((col_idx, to), field) in batch .arrays @@ -287,28 +292,39 @@ pub fn read_row(row: &RowReader, batch: &mut MutableRecordBatch, schema: &Schema .enumerate() .zip(schema.fields().iter()) { - read_field(to, field.data_type(), col_idx, row) + read_field(to, field.data_type(), col_idx, row)?; } + Ok(()) } } macro_rules! fn_read_field { ($NATIVE: ident, $ARRAY: ident) => { paste::item! { - pub(crate) fn [](to: &mut Box, col_idx: usize, row: &RowReader) { + pub(crate) fn [](to: &mut Box, col_idx: usize, row: &RowReader) -> Result<()> { let to = to .as_any_mut() .downcast_mut::<$ARRAY>() - .unwrap(); + .ok_or( + DataFusionError::Internal( + format!("Error downcasting ArrayBuilder to {:?}", stringify!($ARRAY)), + ), + )?; to.append_option(row.[](col_idx)); + Ok(()) } - pub(crate) fn [](to: &mut Box, col_idx: usize, row: &RowReader) { + pub(crate) fn [](to: &mut Box, col_idx: usize, row: &RowReader) -> Result<()> { let to = to .as_any_mut() .downcast_mut::<$ARRAY>() - .unwrap(); + .ok_or( + DataFusionError::Internal( + format!("Error downcasting ArrayBuilder to {:?}", stringify!($ARRAY)), + ), + )?; to.append_value(row.[](col_idx)); + Ok(()) } } }; @@ -333,22 +349,32 @@ pub(crate) fn read_field_binary( to: &mut Box, col_idx: usize, row: &RowReader, -) { - let to = to.as_any_mut().downcast_mut::().unwrap(); +) -> Result<()> { + let to = to.as_any_mut().downcast_mut::().ok_or( + DataFusionError::Internal( + "Error downcasting ArrayBuilder to BinaryBuilder".into(), + ), + )?; if row.is_valid_at(col_idx) { to.append_value(row.get_binary(col_idx)); } else { to.append_null(); } + Ok(()) } pub(crate) fn read_field_binary_null_free( to: &mut Box, col_idx: usize, row: &RowReader, -) { - let to = to.as_any_mut().downcast_mut::().unwrap(); +) -> Result<()> { + let to = to.as_any_mut().downcast_mut::().ok_or( + DataFusionError::Internal( + "Error downcasting ArrayBuilder to BinaryBuilder".into(), + ), + )?; to.append_value(row.get_binary(col_idx)); + Ok(()) } fn read_field( @@ -356,7 +382,7 @@ fn read_field( dt: &DataType, col_idx: usize, row: &RowReader, -) { +) -> Result<()> { use DataType::*; match dt { Boolean => read_field_bool(to, col_idx, row), @@ -374,7 +400,10 @@ fn read_field( Date64 => read_field_date64(to, col_idx, row), Utf8 => read_field_utf8(to, col_idx, row), Binary => read_field_binary(to, col_idx, row), - _ => unimplemented!(), + _ => Err(DataFusionError::NotImplemented(format!( + "The data type {:?} is not recognized", + dt + ))), } } @@ -383,7 +412,7 @@ fn read_field_null_free( dt: &DataType, col_idx: usize, row: &RowReader, -) { +) -> Result<()> { use DataType::*; match dt { Boolean => read_field_bool_null_free(to, col_idx, row), @@ -401,6 +430,9 @@ fn read_field_null_free( Date64 => read_field_date64_null_free(to, col_idx, row), Utf8 => read_field_utf8_null_free(to, col_idx, row), Binary => read_field_binary_null_free(to, col_idx, row), - _ => unimplemented!(), + _ => Err(DataFusionError::NotImplemented(format!( + "The data type {:?} is not recognized", + dt + ))), } } From 26ef014d6be706ebc2ff96caba1a26dda2f68d60 Mon Sep 17 00:00:00 2001 From: srib Date: Sat, 3 Sep 2022 16:53:46 -0400 Subject: [PATCH 2/3] Remove redundant error conversion --- datafusion/core/src/physical_plan/aggregates/row_hash.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/datafusion/core/src/physical_plan/aggregates/row_hash.rs b/datafusion/core/src/physical_plan/aggregates/row_hash.rs index e839e022bf361..9efa2a5a6cc6b 100644 --- a/datafusion/core/src/physical_plan/aggregates/row_hash.rs +++ b/datafusion/core/src/physical_plan/aggregates/row_hash.rs @@ -440,8 +440,7 @@ fn create_batch_from_map( .unzip(); let mut columns: Vec = - read_as_batch(&group_buffers, group_schema, RowType::Compact) - .map_err(|e| ArrowError::ParseError(e.to_string()))?; + read_as_batch(&group_buffers, group_schema, RowType::Compact)?; match mode { AggregateMode::Partial => columns.extend(read_as_batch( From e239bedebe1204a8ab1167c7754116b2cd0bcd6a Mon Sep 17 00:00:00 2001 From: srib Date: Sun, 4 Sep 2022 06:45:37 -0400 Subject: [PATCH 3/3] fix clippy warnings --- datafusion/row/src/reader.rs | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/datafusion/row/src/reader.rs b/datafusion/row/src/reader.rs index 33dda4690ed86..5188e853fb659 100644 --- a/datafusion/row/src/reader.rs +++ b/datafusion/row/src/reader.rs @@ -305,7 +305,7 @@ macro_rules! fn_read_field { let to = to .as_any_mut() .downcast_mut::<$ARRAY>() - .ok_or( + .ok_or_else(|| DataFusionError::Internal( format!("Error downcasting ArrayBuilder to {:?}", stringify!($ARRAY)), ), @@ -318,7 +318,7 @@ macro_rules! fn_read_field { let to = to .as_any_mut() .downcast_mut::<$ARRAY>() - .ok_or( + .ok_or_else(|| DataFusionError::Internal( format!("Error downcasting ArrayBuilder to {:?}", stringify!($ARRAY)), ), @@ -350,11 +350,14 @@ pub(crate) fn read_field_binary( col_idx: usize, row: &RowReader, ) -> Result<()> { - let to = to.as_any_mut().downcast_mut::().ok_or( - DataFusionError::Internal( - "Error downcasting ArrayBuilder to BinaryBuilder".into(), - ), - )?; + let to = to + .as_any_mut() + .downcast_mut::() + .ok_or_else(|| { + DataFusionError::Internal( + "Error downcasting ArrayBuilder to BinaryBuilder".into(), + ) + })?; if row.is_valid_at(col_idx) { to.append_value(row.get_binary(col_idx)); } else { @@ -368,11 +371,14 @@ pub(crate) fn read_field_binary_null_free( col_idx: usize, row: &RowReader, ) -> Result<()> { - let to = to.as_any_mut().downcast_mut::().ok_or( - DataFusionError::Internal( - "Error downcasting ArrayBuilder to BinaryBuilder".into(), - ), - )?; + let to = to + .as_any_mut() + .downcast_mut::() + .ok_or_else(|| { + DataFusionError::Internal( + "Error downcasting ArrayBuilder to BinaryBuilder".into(), + ) + })?; to.append_value(row.get_binary(col_idx)); Ok(()) }