diff --git a/docs/source/user-guide/latest/tuning/operators.md b/docs/source/user-guide/latest/tuning/operators.md index 38fb80f22ec..f557a5e631b 100644 --- a/docs/source/user-guide/latest/tuning/operators.md +++ b/docs/source/user-guide/latest/tuning/operators.md @@ -46,13 +46,13 @@ projects the file schema. The original null checks and residual runtime filter r The original join still verifies matches, including any hash collisions admitted by the filter. Standalone projections, other filter expressions, and limits prevent reader attachment. -To preserve schema-conversion and timestamp-overflow errors, runtime reader pruning is disabled for -each file whose projected or statically filtered columns require schema adaptations beyond direct -column mappings or literal values. This conservative check also disables reader pruning for allowed -`INT32` to `BIGINT` promotion and for projecting a subset of a struct's fields, even when those -adaptations cannot fail. Nested column pruning still reads only the requested struct fields. Scans -with supplied file statistics also skip reader attachment. These cases still use runtime filtering -on decoded batches. +To preserve schema-conversion and timestamp-overflow errors, runtime reader pruning requires every +projected or statically filtered column's schema adaptation to be proven infallible for that file. +Direct column mappings, missing/default literals, permitted `INT32` to `BIGINT` promotion, and +struct subset projections with unchanged leaf types and compatible nullability can retain reader +pruning. Other adaptations conservatively disable it. Nested column pruning still reads only the +requested struct fields. Scans with supplied file statistics also skip reader attachment. These +fallbacks still use runtime filtering on decoded batches. Filters stay within the task's native plan and do not propagate across Spark exchanges or JVM/Arrow boundaries. A shuffled hash join can still filter probe batches after shuffle, but it cannot send diff --git a/native/core/src/execution/operators/dynamic_filter/join/tests.rs b/native/core/src/execution/operators/dynamic_filter/join/tests.rs index 5e2a4f39235..cda7decf5e5 100644 --- a/native/core/src/execution/operators/dynamic_filter/join/tests.rs +++ b/native/core/src/execution/operators/dynamic_filter/join/tests.rs @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +mod benchmark; +mod read_adaptations; mod schema_errors; mod timestamp_errors; diff --git a/native/core/src/execution/operators/dynamic_filter/join/tests/benchmark.rs b/native/core/src/execution/operators/dynamic_filter/join/tests/benchmark.rs new file mode 100644 index 00000000000..f9a34ed5186 --- /dev/null +++ b/native/core/src/execution/operators/dynamic_filter/join/tests/benchmark.rs @@ -0,0 +1,139 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Reproduce the five schema-guard fixtures from PR #6067 with fresh native plans. +//! +//! Run from native/ using the standard optimized release profile: +//! COMET_SCHEMA_GUARD_BENCH_DIR=/tmp/comet-schema-guard cargo test --release \ +//! -p datafusion-comet --lib schema_guard_benchmark -- --ignored --nocapture +//! +//! Build and archive each revision's test binary separately, then run them in +//! base/head/head/base order against the same COMET_SCHEMA_GUARD_BENCH_DIR. Each +//! process performs two warmups and seven measured samples per case/mode, +//! alternating the ON/OFF execution order. Combine the fourteen measured samples +//! per revision/mode when comparing medians; do not average process medians. +//! Fixture creation/validation and session construction are outside the timer. +//! Timing includes scan/join planning and execution with warm local caches. +//! Fixture names are ASCII, so case-insensitive resolution needs no initialized +//! JVM or Java classes. The binary may still link libjvm; include the JDK's +//! lib/server directory in LD_LIBRARY_PATH when running an archived binary. + +use super::read_adaptations::{ParquetJoinCase, ParquetJoinFixture}; +use super::*; +use std::time::{Duration, Instant}; + +struct Measurement { + elapsed: Duration, + data_bytes: usize, + metadata_bytes: usize, + row_groups_read: usize, +} + +impl Measurement { + fn print(&self, case: ParquetJoinCase, enabled: bool, sample: &str) { + println!( + "{},{enabled},{sample},{:.3},{},{},{}", + case.name(), + self.elapsed.as_secs_f64() * 1_000.0, + self.data_bytes, + self.row_groups_read, + self.metadata_bytes, + ); + } +} + +async fn measure( + fixture: &ParquetJoinFixture, + session: &Arc, + enabled: bool, +) -> Measurement { + let started = Instant::now(); + let (scan, plan) = fixture.plan(session, enabled); + let output = collect(Arc::clone(&plan), session.task_ctx()) + .await + .unwrap(); + let elapsed = started.elapsed(); + assert_eq!(row_count(&output), fixture.output_rows); + if enabled { + assert_eq!(metric(&plan, "dynamic_filter_join_filters_attached"), 1); + } + let row_groups_pruned = pruning_metric(&scan, "row_groups_pruned_statistics"); + assert!(row_groups_pruned <= fixture.row_groups); + Measurement { + elapsed, + data_bytes: ParquetJoinFixture::data_bytes(&scan), + metadata_bytes: scan + .metrics() + .unwrap() + .sum_by_name("scan_io_metadata_bytes") + .unwrap() + .as_usize(), + row_groups_read: fixture.row_groups - row_groups_pruned, + } +} + +#[tokio::test] +#[ignore = "manual release benchmark; creates the five Parquet fixtures from PR #6067"] +#[allow(clippy::assertions_on_constants)] // Reject debug runs, not ordinary debug test builds. +async fn schema_guard_benchmark() { + assert!(!cfg!(debug_assertions), "run this benchmark with --release"); + let temporary_directory = tempfile::tempdir().unwrap(); + let directory = std::env::var_os("COMET_SCHEMA_GUARD_BENCH_DIR") + .map(std::path::PathBuf::from) + .unwrap_or_else(|| temporary_directory.path().to_path_buf()); + std::fs::create_dir_all(&directory).unwrap(); + let directory = directory.canonicalize().unwrap(); + eprintln!( + "Parquet schema-guard benchmark fixtures: {}", + directory.display() + ); + println!("case,enabled,sample,elapsed_ms,data_bytes,row_groups_read,metadata_bytes"); + for case in [ + ParquetJoinCase::Direct, + ParquetJoinCase::Promotion, + ParquetJoinCase::StructSubset, + ParquetJoinCase::WideSelective, + ParquetJoinCase::WideNonPruning, + ] { + let (files, rows_per_file, rows_per_group) = match case { + ParquetJoinCase::WideSelective | ParquetJoinCase::WideNonPruning => (128, 2048, 1024), + _ => (16, 65_536, 8192), + }; + let fixture = + ParquetJoinFixture::new(&directory, case, files, rows_per_file, rows_per_group); + let session = ParquetJoinFixture::session(); + let mut samples: [Vec; 2] = [Vec::new(), Vec::new()]; + for iteration in 0..9 { + for enabled in [iteration % 2 != 0, iteration % 2 == 0] { + let measurement = measure(&fixture, &session, enabled).await; + if iteration >= 2 { + measurement.print(case, enabled, &(iteration - 2).to_string()); + samples[usize::from(enabled)].push(measurement); + } + } + } + for (mode, samples) in samples.iter_mut().enumerate() { + // Reader work must be deterministic even when elapsed times vary. + assert!(samples.iter().all(|sample| { + sample.data_bytes == samples[0].data_bytes + && sample.row_groups_read == samples[0].row_groups_read + })); + samples.sort_by_key(|sample| sample.elapsed); + samples[samples.len() / 2].print(case, mode != 0, "median"); + } + } +} diff --git a/native/core/src/execution/operators/dynamic_filter/join/tests/read_adaptations.rs b/native/core/src/execution/operators/dynamic_filter/join/tests/read_adaptations.rs new file mode 100644 index 00000000000..0beee518d5d --- /dev/null +++ b/native/core/src/execution/operators/dynamic_filter/join/tests/read_adaptations.rs @@ -0,0 +1,345 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +mod errors; + +use super::*; +use arrow::array::{Array, StructArray}; +use arrow::datatypes::SchemaRef; +use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder; +use parquet::basic::Compression; +use std::path::{Path, PathBuf}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum ParquetJoinCase { + Direct, + Promotion, + StructSubset, + WideSelective, + WideNonPruning, +} + +impl ParquetJoinCase { + pub(super) fn name(self) -> &'static str { + match self { + Self::Direct => "direct", + Self::Promotion => "promotion", + Self::StructSubset => "struct_subset", + Self::WideSelective => "wide_selective", + Self::WideNonPruning => "wide_non_pruning", + } + } + + fn batch(self, first_key: usize, rows: usize) -> RecordBatch { + let values = Arc::new(Int32Array::from_iter_values( + (first_key..first_key + rows).map(|value| value as i32), + )) as ArrayRef; + let key = if self == Self::WideNonPruning { + Arc::new(Int32Array::from_iter_values( + (0..rows).map(|value| (value % 2) as i32), + )) as ArrayRef + } else { + Arc::clone(&values) + }; + let mut fields = vec![Field::new("key", DataType::Int32, false)]; + let mut columns = vec![key]; + match self { + Self::Direct | Self::Promotion => { + fields.push(Field::new("payload", DataType::Int32, false)); + columns.push(values); + } + Self::StructSubset => { + let children = (0..8) + .map(|index| Field::new(format!("field_{index}"), DataType::Int32, false)) + .collect::>(); + let payload = Arc::new(StructArray::new(children.into(), vec![values; 8], None)); + fields.push(Field::new("payload", payload.data_type().clone(), false)); + columns.push(payload); + } + Self::WideSelective | Self::WideNonPruning => { + for index in 1..64 { + fields.push(Field::new( + format!("payload_{index}"), + DataType::Int32, + false, + )); + columns.push(Arc::clone(&values)); + } + } + } + RecordBatch::try_new(Arc::new(Schema::new(fields)), columns).unwrap() + } + + fn logical_schema(self, physical: &Schema) -> SchemaRef { + let mut fields = physical.fields().iter().cloned().collect::>(); + match self { + Self::Promotion => fields[1] = Arc::new(Field::new("payload", DataType::Int64, false)), + Self::StructSubset => { + // Keep the first two children, matching the fixture used in PR #6067. + let DataType::Struct(children) = physical.field(1).data_type() else { + unreachable!() + }; + fields[1] = Arc::new(Field::new( + "payload", + DataType::Struct(children.iter().take(2).cloned().collect()), + false, + )); + } + _ => {} + } + Arc::new(Schema::new(fields)) + } +} + +/// File paths and scan schema for a join. The caller owns the directory lifetime. +/// Existing files are validated and reused so separate benchmark binaries read +/// identical fixtures; generation and validation happen before timed execution. +pub(super) struct ParquetJoinFixture { + case: ParquetJoinCase, + files: Vec, + logical_schema: SchemaRef, + pub(super) row_groups: usize, + pub(super) output_rows: usize, +} + +impl ParquetJoinFixture { + pub(super) fn new( + directory: &Path, + case: ParquetJoinCase, + files: usize, + rows_per_file: usize, + rows_per_group: usize, + ) -> Self { + let directory = directory.join(case.name()); + std::fs::create_dir_all(&directory).unwrap(); + let properties = WriterProperties::builder() + .set_compression(Compression::SNAPPY) + .set_dictionary_enabled(false) + .set_statistics_enabled(EnabledStatistics::Chunk) + .set_max_row_group_row_count(Some(rows_per_group)) + .build(); + let physical_schema = case.batch(0, 0).schema(); + let mut paths = Vec::with_capacity(files); + let mut row_groups = 0; + for index in 0..files { + let path = directory.join(format!("part-{index:04}.parquet")); + if !path.exists() { + let batch = case.batch(index * rows_per_file, rows_per_file); + let mut writer = ArrowWriter::try_new( + std::fs::File::create(&path).unwrap(), + Arc::clone(&physical_schema), + Some(properties.clone()), + ) + .unwrap(); + writer.write(&batch).unwrap(); + writer.close().unwrap(); + } + let reader = + ParquetRecordBatchReaderBuilder::try_new(std::fs::File::open(&path).unwrap()) + .unwrap(); + assert_eq!(reader.schema(), &physical_schema, "{}", path.display()); + assert_eq!( + reader.metadata().file_metadata().num_rows(), + rows_per_file as i64 + ); + assert_eq!( + reader.metadata().num_row_groups(), + rows_per_file.div_ceil(rows_per_group) + ); + row_groups += reader.metadata().num_row_groups(); + paths.push(path); + } + Self { + case, + files: paths, + logical_schema: case.logical_schema(&physical_schema), + row_groups, + output_rows: if case == ParquetJoinCase::WideNonPruning { + files * rows_per_file.div_ceil(2) + } else { + 1 + }, + } + } + + pub(super) fn session() -> Arc { + let mut config = SessionConfig::new() + .with_target_partitions(1) + .with_batch_size(8192) + .with_parquet_page_index_pruning(false); + // Isolate row-group pruning, retaining the decoded-batch join filter. + config.options_mut().execution.parquet.pushdown_filters = false; + Arc::new(SessionContext::new_with_config(config)) + } + + fn scan( + &self, + session: &Arc, + projection: Vec, + filters: Option>>, + allow_type_promotion: bool, + ) -> Arc { + init_datasource_exec( + Arc::new(self.logical_schema.project(&projection).unwrap()), + Some(Arc::clone(&self.logical_schema)), + None, + ObjectStoreUrl::local_filesystem(), + ObjectStoreBackend::Local, + vec![self + .files + .iter() + .map(|path| { + PartitionedFile::from_path(path.to_string_lossy().into_owned()).unwrap() + }) + .collect()], + Some(projection), + filters, + None, + "UTC", + false, + false, + allow_type_promotion, + false, + session, + false, + false, + false, + ) + .unwrap() + } + + pub(super) fn plan( + &self, + session: &Arc, + enabled: bool, + ) -> (Arc, Arc) { + let scan = self.scan( + session, + (0..self.logical_schema.fields().len()).collect(), + None, + self.case == ParquetJoinCase::Promotion, + ); + let plan = join_scan(Arc::clone(&scan), session, enabled, 0); + (scan, plan) + } + + pub(super) fn data_bytes(scan: &Arc) -> usize { + scan.metrics() + .unwrap() + .sum_by_name("scan_io_data_bytes") + .unwrap() + .as_usize() + } +} + +fn join_scan( + scan: Arc, + session: &Arc, + enabled: bool, + build_key: i32, +) -> Arc { + let build = memory_exec(vec![RecordBatch::try_new( + Arc::new(Schema::new(vec![Field::new("key", DataType::Int32, false)])), + vec![Arc::new(Int32Array::from(vec![build_key]))], + ) + .unwrap()]); + let join = single_key_join_plans(build, scan, PartitionMode::Partitioned); + PhysicalPlanner::apply_join_dynamic_filter( + Arc::new(join), + enabled, + session.copied_config().options(), + ) + .unwrap() +} + +async fn assert_adaptation_keeps_pruning(case: ParquetJoinCase) { + let directory = tempfile::tempdir().unwrap(); + let fixture = ParquetJoinFixture::new(directory.path(), case, 2, 400, 100); + let session = ParquetJoinFixture::session(); + let mut outputs = Vec::new(); + let mut bytes = Vec::new(); + for enabled in [false, true] { + let (scan, plan) = fixture.plan(&session, enabled); + let output = collect(Arc::clone(&plan), session.task_ctx()) + .await + .unwrap(); + assert_eq!(row_count(&output), fixture.output_rows); + assert_eq!( + pruning_metric(&scan, "row_groups_pruned_statistics"), + if enabled { fixture.row_groups - 1 } else { 0 }, + "case={case:?}, enabled={enabled}" + ); + if enabled { + assert_eq!(metric(&plan, "dynamic_filter_join_filters_attached"), 1); + } + bytes.push(ParquetJoinFixture::data_bytes(&scan)); + outputs.push(batches_to_sort_string(&output)); + } + assert_eq!(outputs[0], outputs[1]); + assert!( + bytes[1] < bytes[0], + "case={case:?}, data bytes OFF/ON={bytes:?}" + ); +} + +#[tokio::test] +async fn allowed_int32_to_bigint_keeps_reader_pruning() { + assert_adaptation_keeps_pruning(ParquetJoinCase::Promotion).await; +} + +#[tokio::test] +async fn struct_subset_keeps_reader_pruning() { + assert_adaptation_keeps_pruning(ParquetJoinCase::StructSubset).await; +} + +#[tokio::test] +async fn static_predicate_only_column_preserves_conversion_error() { + let directory = tempfile::tempdir().unwrap(); + let fixture = + ParquetJoinFixture::new(directory.path(), ParquetJoinCase::Promotion, 1, 400, 100); + for enabled in [false, true] { + let mut config = SessionConfig::new() + .with_target_partitions(1) + .with_parquet_page_index_pruning(false); + // This dependency is absent from the projection. Enable row filtering + // so the static predicate actually evaluates the payload conversion. + config.options_mut().execution.parquet.pushdown_filters = true; + let session = Arc::new(SessionContext::new_with_config(config)); + let scan = fixture.scan( + &session, + vec![0], + Some(vec![Arc::new(BinaryExpr::new( + Arc::new(Column::new("payload", 1)), + Operator::Gt, + lit(-1_i64), + ))]), + false, + ); + // No probe keys match. The payload is needed only by the static reader + // predicate, whose unsupported conversion still has to be evaluated. + let plan = join_scan(scan, &session, enabled, -1); + let error = collect(plan, session.task_ctx()).await.expect_err( + "runtime filtering must retain conversion errors in static-predicate-only columns", + ); + // The Parquet row filter wraps the structured conversion error using Debug. + let message = error.to_string(); + assert!( + message.contains("ParquetSchemaConvert") && message.contains("[payload]"), + "enabled={enabled}: {error}" + ); + } +} diff --git a/native/core/src/execution/operators/dynamic_filter/join/tests/read_adaptations/errors.rs b/native/core/src/execution/operators/dynamic_filter/join/tests/read_adaptations/errors.rs new file mode 100644 index 00000000000..108e44ea5bb --- /dev/null +++ b/native/core/src/execution/operators/dynamic_filter/join/tests/read_adaptations/errors.rs @@ -0,0 +1,150 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use super::*; +use arrow::array::TimestampMillisecondArray; +use arrow::datatypes::TimeUnit; + +async fn assert_conversion_error( + batch: RecordBatch, + logical_schema: SchemaRef, + build_key: i32, + allow_type_promotion: bool, + expected_error: &str, +) { + let file = tempfile::NamedTempFile::new().unwrap(); + let properties = WriterProperties::builder() + .set_max_row_group_row_count(Some(100)) + .set_statistics_enabled(EnabledStatistics::Chunk) + .set_dictionary_enabled(false) + .build(); + let mut writer = + ArrowWriter::try_new(file.reopen().unwrap(), batch.schema(), Some(properties)).unwrap(); + writer.write(&batch).unwrap(); + assert_eq!(writer.close().unwrap().num_row_groups(), 2); + let session = ParquetJoinFixture::session(); + for enabled in [false, true] { + let scan = init_datasource_exec( + Arc::clone(&logical_schema), + Some(Arc::clone(&logical_schema)), + None, + ObjectStoreUrl::local_filesystem(), + ObjectStoreBackend::Local, + vec![vec![PartitionedFile::from_path( + file.path().to_string_lossy().into_owned(), + ) + .unwrap()]], + Some((0..logical_schema.fields().len()).collect()), + None, + None, + "UTC", + false, + false, + allow_type_promotion, + false, + &session, + false, + false, + false, + ) + .unwrap(); + let plan = join_scan(scan, &session, enabled, build_key); + let error = collect(plan, session.task_ctx()).await.expect_err( + "a safe adaptation must not let runtime pruning hide another conversion error", + ); + assert!( + error.to_string().contains(expected_error), + "enabled={enabled}: {error}" + ); + } +} + +#[tokio::test] +async fn struct_subset_preserves_retained_timestamp_overflow() { + // The nonmatching second row group overflows. Dropping an unrelated struct + // field does not make conversion of the retained timestamp infallible. + let timestamp = Arc::new(TimestampMillisecondArray::from_iter_values( + (0..200).map(|key| if key < 100 { 0 } else { i64::MAX / 1_000 + 1 }), + )) as ArrayRef; + let payload = Arc::new(StructArray::new( + vec![ + Field::new("ts", timestamp.data_type().clone(), false), + Field::new("omitted", DataType::Int32, false), + ] + .into(), + vec![timestamp, Arc::new(Int32Array::from_iter_values(0..200))], + None, + )); + let key_field = Field::new("key", DataType::Int32, false); + let batch = RecordBatch::try_new( + Arc::new(Schema::new(vec![ + key_field.clone(), + Field::new("payload", payload.data_type().clone(), false), + ])), + vec![Arc::new(Int32Array::from_iter_values(0..200)), payload], + ) + .unwrap(); + let logical_schema = Arc::new(Schema::new(vec![ + key_field, + Field::new( + "payload", + DataType::Struct( + vec![Field::new( + "ts", + DataType::Timestamp(TimeUnit::Microsecond, None), + false, + )] + .into(), + ), + false, + ), + ])); + assert_conversion_error(batch, logical_schema, 0, false, "overflow").await; +} + +#[tokio::test] +async fn allowed_promotion_does_not_hide_an_unrelated_conversion_error() { + let key_field = Field::new("key", DataType::Int32, false); + let batch = RecordBatch::try_new( + Arc::new(Schema::new(vec![ + key_field.clone(), + Field::new("safe_payload", DataType::Int32, false), + Field::new("unsafe_payload", DataType::Int64, false), + ])), + vec![ + Arc::new(Int32Array::from_iter_values(0..200)), + Arc::new(Int32Array::from_iter_values(0..200)), + Arc::new(Int64Array::from(vec![i64::MAX; 200])), + ], + ) + .unwrap(); + let logical_schema = Arc::new(Schema::new(vec![ + key_field, + Field::new("safe_payload", DataType::Int64, false), + Field::new("unsafe_payload", DataType::Int32, false), + ])); + // No keys match, so an eligibility check that stops at the permitted + // promotion would prune both groups and incorrectly suppress the error. + assert_conversion_error( + batch, + logical_schema, + -1, + true, + "Parquet column cannot be converted", + ) + .await; +} diff --git a/native/core/src/execution/operators/dynamic_filter/parquet_reader.rs b/native/core/src/execution/operators/dynamic_filter/parquet_reader.rs index 2fcd54d8bd4..b0ec84275f9 100644 --- a/native/core/src/execution/operators/dynamic_filter/parquet_reader.rs +++ b/native/core/src/execution/operators/dynamic_filter/parquet_reader.rs @@ -17,6 +17,7 @@ //! Attach a live predicate to a native Parquet reader while preserving input boundaries. +use crate::parquet::schema_adapter::SparkPhysicalExprAdapterFactory; use std::sync::Arc; use datafusion::common::config::ConfigOptions; @@ -156,10 +157,10 @@ pub(super) fn try_attach_parquet_reader_filter( .then_with(|| a.name().cmp(b.name())) }); read_columns.dedup(); - let adapter_factory = Arc::new(RuntimeFilterSchemaAdapterFactory::new( - Arc::clone(adapter_factory), - read_columns, - )); + let adapter_factory = Arc::new( + RuntimeFilterSchemaAdapterFactory::new(Arc::clone(adapter_factory), read_columns) + .with_spark_factory(SparkPhysicalExprAdapterFactory::from_file_scan(file_config)), + ); let predicate: Arc = predicate; let propagation = match scan diff --git a/native/core/src/execution/operators/dynamic_filter/parquet_reader/schema_adapter.rs b/native/core/src/execution/operators/dynamic_filter/parquet_reader/schema_adapter.rs index 69163c0d0e0..c4ee9af2d8b 100644 --- a/native/core/src/execution/operators/dynamic_filter/parquet_reader/schema_adapter.rs +++ b/native/core/src/execution/operators/dynamic_filter/parquet_reader/schema_adapter.rs @@ -17,12 +17,15 @@ //! Preserve per-file conversion errors when runtime reader filters are attached. +use crate::parquet::schema_adapter::{ + is_infallible_read_adaptation, SparkPhysicalExprAdapterFactory, +}; use std::sync::Arc; use arrow::datatypes::SchemaRef; use datafusion::common::tree_node::{Transformed, TransformedResult, TreeNode}; use datafusion::common::Result; -use datafusion::physical_expr::expressions::{lit, Column, DynamicFilterPhysicalExpr, Literal}; +use datafusion::physical_expr::expressions::{lit, Column, DynamicFilterPhysicalExpr}; use datafusion::physical_expr::PhysicalExpr; use datafusion::physical_expr_adapter::{PhysicalExprAdapter, PhysicalExprAdapterFactory}; @@ -31,6 +34,7 @@ use datafusion::physical_expr_adapter::{PhysicalExprAdapter, PhysicalExprAdapter pub(super) struct RuntimeFilterSchemaAdapterFactory { inner: Arc, read_columns: Vec, + spark_factory: Option>, } impl RuntimeFilterSchemaAdapterFactory { @@ -41,8 +45,17 @@ impl RuntimeFilterSchemaAdapterFactory { Self { inner, read_columns, + spark_factory: None, } } + + pub(super) fn with_spark_factory( + mut self, + factory: Option>, + ) -> Self { + self.spark_factory = factory; + self + } } impl PhysicalExprAdapterFactory for RuntimeFilterSchemaAdapterFactory { @@ -51,27 +64,21 @@ impl PhysicalExprAdapterFactory for RuntimeFilterSchemaAdapterFactory { logical_schema: SchemaRef, physical_schema: SchemaRef, ) -> Result> { - let inner = self - .inner - .create(logical_schema, Arc::clone(&physical_schema))?; - // DataFusion adapts predicates before projections and row-group pruning. - // Direct remapping and missing/default literals are safe. Other adaptations - // can reject nonempty batches or overflow, so let normal decoding run first. - // Spark's adapter can leave unresolved columns unchanged. Require rewritten - // column names to resolve in the original physical schema; case and field-ID - // remapping restore those names before returning the expression. - // Safe-adaptation and matching-schema optimizations are tracked in - // https://github.com/apache/datafusion-comet/issues/6123. - let allow_runtime_filter = - self.read_columns - .iter() - .all(|column| match inner.rewrite(Arc::new(column.clone())) { - Ok(expr) if expr.is::() => true, - Ok(expr) => expr - .downcast_ref::() - .is_some_and(|column| physical_schema.index_of(column.name()).is_ok()), - Err(_) => false, - }); + let (inner, allow_runtime_filter) = if let Some(factory) = &self.spark_factory { + factory.create_with_read_safety(logical_schema, physical_schema, &self.read_columns)? + } else { + let inner = self + .inner + .create(logical_schema, Arc::clone(&physical_schema))?; + // Unknown factories must establish safety through their actual rewrites. + // Errors only disable pruning; normal decoding retains its error timing. + let allow_runtime_filter = self.read_columns.iter().all(|column| { + inner + .rewrite(Arc::new(column.clone())) + .is_ok_and(|expr| is_infallible_read_adaptation(&expr, &physical_schema)) + }); + (inner, allow_runtime_filter) + }; Ok(Arc::new(RuntimeFilterSchemaAdapter { inner, allow_runtime_filter, diff --git a/native/core/src/execution/operators/dynamic_filter/parquet_reader/schema_adapter/tests/resolution.rs b/native/core/src/execution/operators/dynamic_filter/parquet_reader/schema_adapter/tests/resolution.rs index 21b7bcc6315..761bda2c725 100644 --- a/native/core/src/execution/operators/dynamic_filter/parquet_reader/schema_adapter/tests/resolution.rs +++ b/native/core/src/execution/operators/dynamic_filter/parquet_reader/schema_adapter/tests/resolution.rs @@ -19,6 +19,7 @@ use super::*; use crate::parquet::parquet_support::SparkParquetOptions; use crate::parquet::schema_adapter::SparkPhysicalExprAdapterFactory; use datafusion::common::ScalarValue; +use datafusion::physical_expr::expressions::Literal; use datafusion_comet_spark_expr::EvalMode; use parquet::arrow::PARQUET_FIELD_ID_META_KEY; use std::collections::HashMap; diff --git a/native/core/src/parquet/parquet_exec.rs b/native/core/src/parquet/parquet_exec.rs index 5c5d26d3922..86586f6dc08 100644 --- a/native/core/src/parquet/parquet_exec.rs +++ b/native/core/src/parquet/parquet_exec.rs @@ -231,13 +231,29 @@ pub(crate) fn init_datasource_exec( _ => Arc::new(parquet_source), }; - let expr_adapter_factory: Arc = Arc::new( - SparkPhysicalExprAdapterFactory::new(spark_parquet_options, default_values), - ); + let spark_adapter_factory = Arc::new(SparkPhysicalExprAdapterFactory::new( + spark_parquet_options, + default_values, + )); + let expr_adapter_factory: Arc = + Arc::::clone(&spark_adapter_factory); let file_groups = file_groups - .iter() - .map(|files| FileGroup::new(files.clone())) + .into_iter() + .map(|files| { + FileGroup::new( + files + .into_iter() + .map(|mut file| { + // Retain the concrete adapter for runtime-filter read-safety checks. + // Extensions are keyed by type; existing access plans stay intact. + file.extensions + .insert_arc(Arc::clone(&spark_adapter_factory)); + file + }) + .collect(), + ) + }) .collect(); let mut file_scan_config_builder = FileScanConfigBuilder::new(object_store_url, file_source) diff --git a/native/core/src/parquet/schema_adapter.rs b/native/core/src/parquet/schema_adapter.rs index c7ad0418578..bd80e2fb386 100644 --- a/native/core/src/parquet/schema_adapter.rs +++ b/native/core/src/parquet/schema_adapter.rs @@ -15,6 +15,11 @@ // specific language governing permissions and limitations // under the License. +mod read_adaptation; +mod runtime_filter; + +pub(crate) use read_adaptation::is_infallible_read_adaptation; + use crate::parquet::cast_column::CometCastColumnExpr; use crate::parquet::name_fold::{fold_name, fold_names, fold_schema_names}; use crate::parquet::parquet_support::{ @@ -862,6 +867,19 @@ impl PhysicalExprAdapterFactory for SparkPhysicalExprAdapterFactory { logical_file_schema: SchemaRef, physical_file_schema: SchemaRef, ) -> DataFusionResult> { + Ok(Arc::new(self.create_adapter( + logical_file_schema, + physical_file_schema, + )?)) + } +} + +impl SparkPhysicalExprAdapterFactory { + fn create_adapter( + &self, + logical_file_schema: SchemaRef, + physical_file_schema: SchemaRef, + ) -> DataFusionResult { // Remap physical schema field names to match logical names by Parquet field id // (when the logical schema carries IDs and `use_field_id` is set) and/or by // case-insensitive name match. The DefaultPhysicalExprAdapter uses exact name @@ -979,7 +997,7 @@ impl PhysicalExprAdapterFactory for SparkPhysicalExprAdapterFactory { Arc::clone(&adapted_physical_schema), )?; - Ok(Arc::new(SparkPhysicalExprAdapter { + Ok(SparkPhysicalExprAdapter { logical_file_schema, physical_file_schema: adapted_physical_schema, parquet_options: self.parquet_options.clone(), @@ -991,7 +1009,7 @@ impl PhysicalExprAdapterFactory for SparkPhysicalExprAdapterFactory { id_duplicate_roots, logical_folded, physical_folded, - })) + }) } } diff --git a/native/core/src/parquet/schema_adapter/read_adaptation.rs b/native/core/src/parquet/schema_adapter/read_adaptation.rs new file mode 100644 index 00000000000..73d723c0920 --- /dev/null +++ b/native/core/src/parquet/schema_adapter/read_adaptation.rs @@ -0,0 +1,121 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Prove when skipping a file's read adaptation cannot hide a conversion error. + +use std::collections::HashSet; +use std::sync::Arc; + +use arrow::datatypes::{DataType, Field, Schema}; +use datafusion::common::nested_struct::validate_data_type_compatibility; +use datafusion::physical_expr::expressions::{CastExpr, Column, Literal}; +use datafusion::physical_expr::PhysicalExpr; +use datafusion_comet_spark_expr::Cast; + +/// Whether an already rewritten read-column expression cannot fail on a valid decoded batch. +/// +/// Call this only after the file's ordinary schema-adapter rewrite. That rewrite owns Spark's +/// name/field-ID resolution, missing defaults, and conversion rules. In particular, a disallowed +/// INT32-to-BIGINT promotion becomes `RejectOnNonEmpty`, not a Spark `Cast`. +/// The reader must still bind column indices to its projected batch schema before evaluation. +/// +/// This proof covers errors introduced by adaptation, not arbitrary Parquet decoding failures. +/// Unknown expressions remain unsafe. A supported conversion is not necessarily infallible: +/// timestamp scaling can overflow, and even equal Variant storage types require normalization. +pub(crate) fn is_infallible_read_adaptation( + expr: &Arc, + physical_schema: &Schema, +) -> bool { + if expr.is::() || resolved_column_field(expr, physical_schema).is_some() { + return true; + } + + if let Some(cast) = expr.downcast_ref::() { + return cast.cast_options.is_adapting_schema + && cast.data_type == DataType::Int64 + && resolved_column_field(&cast.child, physical_schema).is_some_and(|field| { + field.extension_type_name().is_none() && field.data_type() == &DataType::Int32 + }); + } + + let Some(cast) = expr.downcast_ref::() else { + return false; + }; + let Some(source) = resolved_column_field(cast.expr(), physical_schema) else { + return false; + }; + let target = cast.target_field(); + // Only retained DataFusion structural casts have a proof here. Keep their expression shape + // unchanged so Parquet can still clip unrequested leaves. Do not generalize this to opaque + // Comet casts, which can decode additional fields and perform fallible value conversions. + matches!(source.data_type(), DataType::Struct(_) | DataType::List(_) | DataType::LargeList(_)) + && source.extension_type_name().is_none() + && target.extension_type_name().is_none() + && source.metadata() == target.metadata() + && is_structural_subset(source.data_type(), target.data_type()) + // DataFusion validates nested nullability and field overlap at evaluation time. A + // failed check is an unsafe verdict, never an eager error from this optimization. + && validate_data_type_compatibility("", source.data_type(), target.data_type()).is_ok() +} + +fn resolved_column_field<'a>( + expr: &Arc, + physical_schema: &'a Schema, +) -> Option<&'a Field> { + let column = expr.downcast_ref::()?; + // Input indices may belong to an earlier projection. The adapter restores physical names, + // and DataFusion reassigns indices when it builds the file's projected batch schema. + physical_schema.field_with_name(column.name()).ok() +} + +fn is_structural_subset(source: &DataType, target: &DataType) -> bool { + match (source, target) { + (DataType::Struct(source_fields), DataType::Struct(target_fields)) => { + if target_fields.is_empty() { + return false; + } + let source_names: HashSet<_> = source_fields.iter().map(|f| f.name()).collect(); + let target_names: HashSet<_> = target_fields.iter().map(|f| f.name()).collect(); + source_names.len() == source_fields.len() + && target_names.len() == target_fields.len() + && target_fields.iter().all(|target| { + source_fields + .iter() + .find(|source| source.name() == target.name()) + .is_some_and(|source| is_structural_subset_field(source, target)) + }) + } + (DataType::List(source), DataType::List(target)) + | (DataType::LargeList(source), DataType::LargeList(target)) => { + is_structural_subset_field(source, target) + } + // Containers outside this proof must not slip through an equality shortcut. + (DataType::Dictionary(_, _), _) | (_, DataType::Dictionary(_, _)) => false, + _ => !source.is_nested() && source == target, + } +} + +fn is_structural_subset_field(source: &Field, target: &Field) -> bool { + (!source.is_nullable() || target.is_nullable()) + && source.extension_type_name().is_none() + && target.extension_type_name().is_none() + && source.metadata() == target.metadata() + && is_structural_subset(source.data_type(), target.data_type()) +} + +#[cfg(test)] +mod tests; diff --git a/native/core/src/parquet/schema_adapter/read_adaptation/tests.rs b/native/core/src/parquet/schema_adapter/read_adaptation/tests.rs new file mode 100644 index 00000000000..c693d1bc070 --- /dev/null +++ b/native/core/src/parquet/schema_adapter/read_adaptation/tests.rs @@ -0,0 +1,62 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +mod scalar; +mod structural; + +use super::*; +use crate::parquet::parquet_support::SparkParquetOptions; +use crate::parquet::schema_adapter::SparkPhysicalExprAdapterFactory; +use arrow::datatypes::SchemaRef; +use datafusion::physical_expr_adapter::PhysicalExprAdapterFactory; +use datafusion_comet_spark_expr::EvalMode; + +fn options() -> SparkParquetOptions { + let mut options = SparkParquetOptions::new(EvalMode::Legacy, "UTC", false); + options.case_sensitive = true; + options +} + +fn adapt( + source: Field, + target: Field, + options: SparkParquetOptions, +) -> (SchemaRef, Arc) { + let physical = Arc::new(Schema::new(vec![source])); + let logical = Arc::new(Schema::new(vec![target])); + let adapter = SparkPhysicalExprAdapterFactory::new(options, None) + .create(logical, Arc::clone(&physical)) + .unwrap(); + let expr = adapter + .rewrite(Arc::new(Column::new("s", usize::MAX))) + .unwrap(); + (physical, expr) +} + +fn structural_cast(source: DataType, target: DataType) -> (Schema, Arc) { + let schema = Schema::new(vec![Field::new("s", source, true)]); + let expr = Arc::new(CastExpr::new_with_target_field( + Arc::new(Column::new("s", 0)), + Arc::new(Field::new("s", target, true)), + None, + )); + (schema, expr) +} + +fn struct_type(fields: Vec) -> DataType { + DataType::Struct(fields.into()) +} diff --git a/native/core/src/parquet/schema_adapter/read_adaptation/tests/scalar.rs b/native/core/src/parquet/schema_adapter/read_adaptation/tests/scalar.rs new file mode 100644 index 00000000000..94c6d5d6b6d --- /dev/null +++ b/native/core/src/parquet/schema_adapter/read_adaptation/tests/scalar.rs @@ -0,0 +1,108 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use super::*; +use arrow::array::{Int32Array, Int64Array, RecordBatch}; +use arrow::datatypes::TimeUnit; +use datafusion::physical_expr::expressions::lit; +use datafusion_comet_spark_expr::SparkCastOptions; + +#[test] +fn literals_and_resolved_columns_need_no_conversion() { + let physical = Schema::new(vec![Field::new("s", DataType::Int32, true)]); + assert!(is_infallible_read_adaptation(&lit(7_i32), &physical)); + let resolved: Arc = Arc::new(Column::new("s", usize::MAX)); + assert!(is_infallible_read_adaptation(&resolved, &physical)); + let missing: Arc = Arc::new(Column::new("missing", 0)); + assert!(!is_infallible_read_adaptation(&missing, &physical)); +} + +#[test] +fn accepted_int32_promotion_handles_boundaries_and_nulls() { + for eval_mode in [EvalMode::Legacy, EvalMode::Ansi] { + let mut options = options(); + options.eval_mode = eval_mode; + options.allow_type_promotion = true; + let (schema, expr) = adapt( + Field::new("s", DataType::Int32, true), + Field::new("s", DataType::Int64, true), + options, + ); + assert!(is_infallible_read_adaptation(&expr, &schema)); + let batch = RecordBatch::try_new( + schema, + vec![Arc::new(Int32Array::from(vec![ + Some(i32::MIN), + None, + Some(i32::MAX), + ]))], + ) + .unwrap(); + let output = expr.evaluate(&batch).unwrap().into_array(3).unwrap(); + assert_eq!( + output.as_any().downcast_ref::().unwrap(), + &Int64Array::from(vec![ + Some(i64::from(i32::MIN)), + None, + Some(i64::from(i32::MAX)) + ]), + ); + } +} + +#[test] +fn promotion_rejected_by_spark_remains_unsafe() { + let mut options = options(); + options.allow_type_promotion = false; + let (schema, expr) = adapt( + Field::new("s", DataType::Int32, true), + Field::new("s", DataType::Int64, true), + options, + ); + assert!(!is_infallible_read_adaptation(&expr, &schema)); + assert!(expr + .evaluate(&RecordBatch::new_empty(Arc::clone(&schema))) + .is_ok()); + let batch = RecordBatch::try_new(schema, vec![Arc::new(Int32Array::from(vec![1]))]).unwrap(); + assert!(expr.evaluate(&batch).is_err()); +} + +#[test] +fn scalar_casts_without_a_read_adaptation_proof_remain_unsafe() { + let schema = Schema::new(vec![Field::new("s", DataType::Int32, true)]); + let ordinary_cast: Arc = Arc::new(Cast::new( + Arc::new(Column::new("s", 0)), + DataType::Int64, + SparkCastOptions::new(EvalMode::Ansi, "UTC", false), + None, + None, + )); + assert!(!is_infallible_read_adaptation(&ordinary_cast, &schema)); + let df_cast: Arc = Arc::new(CastExpr::new( + Arc::new(Column::new("s", 0)), + DataType::Int64, + None, + )); + assert!(!is_infallible_read_adaptation(&df_cast, &schema)); + + let (schema, timestamp) = adapt( + Field::new("s", DataType::Timestamp(TimeUnit::Millisecond, None), true), + Field::new("s", DataType::Timestamp(TimeUnit::Microsecond, None), true), + options(), + ); + assert!(!is_infallible_read_adaptation(×tamp, &schema)); +} diff --git a/native/core/src/parquet/schema_adapter/read_adaptation/tests/structural.rs b/native/core/src/parquet/schema_adapter/read_adaptation/tests/structural.rs new file mode 100644 index 00000000000..bf89c6efc8b --- /dev/null +++ b/native/core/src/parquet/schema_adapter/read_adaptation/tests/structural.rs @@ -0,0 +1,237 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use super::*; +use crate::parquet::cast_column::CometCastColumnExpr; +use arrow::array::{ + Array, ArrayRef, GenericListArray, Int32Array, OffsetSizeTrait, RecordBatch, StructArray, +}; +use arrow::buffer::{NullBuffer, OffsetBuffer}; +use arrow::datatypes::Fields; +use parquet::variant::VariantType; +use std::collections::HashMap; + +#[test] +fn nested_subset_preserves_values_and_parent_nulls() { + let kept = Arc::new(Field::new("kept", DataType::Int32, true)); + let removed = Arc::new(Field::new("removed", DataType::Int32, true)); + let nested_source: Fields = vec![Arc::clone(&kept), Arc::clone(&removed)].into(); + let nested_target: Fields = vec![Arc::clone(&kept)].into(); + let source_fields: Fields = vec![ + Arc::clone(&kept), + Arc::new(Field::new( + "nested", + DataType::Struct(nested_source.clone()), + true, + )), + removed, + ] + .into(); + let target_fields: Fields = vec![ + Arc::new(Field::new( + "nested", + DataType::Struct(nested_target.clone()), + true, + )), + kept, + ] + .into(); + let (schema, expr) = adapt( + Field::new("s", DataType::Struct(source_fields.clone()), true), + Field::new("s", DataType::Struct(target_fields.clone()), true), + options(), + ); + assert!(expr.is::(), "retain DataFusion leaf clipping"); + assert!(is_infallible_read_adaptation(&expr, &schema)); + let values: ArrayRef = Arc::new(Int32Array::from(vec![Some(7), None, Some(-2)])); + let nested_nulls = Some(NullBuffer::from(vec![true, true, false])); + let parent_nulls = Some(NullBuffer::from(vec![true, false, true])); + let nested: ArrayRef = Arc::new(StructArray::new( + nested_source, + vec![Arc::clone(&values), Arc::clone(&values)], + nested_nulls.clone(), + )); + let source: ArrayRef = Arc::new(StructArray::new( + source_fields, + vec![Arc::clone(&values), nested, Arc::clone(&values)], + parent_nulls.clone(), + )); + let batch = RecordBatch::try_new(schema, vec![source]).unwrap(); + let output = expr.evaluate(&batch).unwrap().into_array(3).unwrap(); + let expected_nested: ArrayRef = Arc::new(StructArray::new( + nested_target, + vec![Arc::clone(&values)], + nested_nulls, + )); + let expected = StructArray::new(target_fields, vec![expected_nested, values], parent_nulls); + assert_eq!( + output.as_any().downcast_ref::().unwrap(), + &expected + ); +} + +#[test] +fn subset_proof_rejects_missing_duplicate_and_changing_fields() { + let field = Field::new("kept", DataType::Int32, true); + let source = struct_type(vec![ + field.clone(), + Field::new("removed", DataType::Int32, true), + ]); + for target in [ + struct_type(vec![]), + struct_type(vec![Field::new("missing", DataType::Int32, true)]), + struct_type(vec![field.clone(), field.clone()]), + struct_type(vec![Field::new("kept", DataType::Int64, true)]), + struct_type(vec![Field::new("kept", DataType::Int32, false)]), + struct_type(vec![field.clone().with_metadata(HashMap::from([( + "custom".into(), + "changed".into(), + )]))]), + ] { + let (schema, expr) = structural_cast(source.clone(), target); + assert!(!is_infallible_read_adaptation(&expr, &schema), "{expr}"); + } + let (schema, expr) = structural_cast( + struct_type(vec![field.clone(), field.clone()]), + struct_type(vec![field]), + ); + assert!(!is_infallible_read_adaptation(&expr, &schema)); +} + +#[test] +fn list_subsets_require_unchanged_representation_and_compatible_nullability() { + let source_inner = struct_type(vec![ + Field::new("kept", DataType::Int32, true), + Field::new("removed", DataType::Int32, true), + ]); + let target_inner = struct_type(vec![Field::new("kept", DataType::Int32, true)]); + for large in [false, true] { + let list = |field| { + if large { + DataType::LargeList(Arc::new(field)) + } else { + DataType::List(Arc::new(field)) + } + }; + let source = list(Field::new("item", source_inner.clone(), true)); + for (nullable, expected) in [(true, true), (false, false)] { + let target = list(Field::new("element", target_inner.clone(), nullable)); + let (schema, expr) = structural_cast(source.clone(), target); + assert_eq!(is_infallible_read_adaptation(&expr, &schema), expected); + } + } + let (schema, expr) = structural_cast( + DataType::List(Arc::new(Field::new("item", source_inner, true))), + DataType::LargeList(Arc::new(Field::new("item", target_inner, true))), + ); + assert!(!is_infallible_read_adaptation(&expr, &schema)); +} + +fn assert_list_subset_preserves_nulls() { + let kept = Arc::new(Field::new("kept", DataType::Int32, true)); + let source_fields: Fields = vec![ + Arc::clone(&kept), + Arc::new(Field::new("removed", DataType::Int32, false)), + ] + .into(); + let target_fields: Fields = vec![kept].into(); + let values: ArrayRef = Arc::new(Int32Array::from(vec![ + Some(7), + Some(99), + Some(13), + None, + Some(-2), + ])); + let element_nulls = Some(NullBuffer::from(vec![true, false, true, true, true])); + let source_values = Arc::new(StructArray::new( + source_fields.clone(), + vec![ + Arc::clone(&values), + Arc::new(Int32Array::from(vec![107, 199, 113, 117, 98])), + ], + element_nulls.clone(), + )); + let expected_values = Arc::new(StructArray::new( + target_fields.clone(), + vec![values], + element_nulls, + )); + // Include a null element, a null list with a nonempty backing range, a null + // retained leaf, and an empty list. Offsets must remain unchanged. + let offsets = OffsetBuffer::::from_lengths([2, 1, 2, 0]); + let list_nulls = Some(NullBuffer::from(vec![true, false, true, true])); + let source = GenericListArray::::new( + Arc::new(Field::new("item", DataType::Struct(source_fields), true)), + offsets.clone(), + source_values, + list_nulls.clone(), + ); + let expected = GenericListArray::::new( + Arc::new(Field::new("element", DataType::Struct(target_fields), true)), + offsets, + expected_values, + list_nulls, + ); + let (schema, expr) = adapt( + Field::new("s", source.data_type().clone(), true), + Field::new("s", expected.data_type().clone(), true), + options(), + ); + assert!(expr.is::(), "retain DataFusion leaf clipping"); + assert!(is_infallible_read_adaptation(&expr, &schema)); + let batch = RecordBatch::try_new(schema, vec![Arc::new(source)]).unwrap(); + let output = expr.evaluate(&batch).unwrap().into_array(4).unwrap(); + assert_eq!( + output + .as_any() + .downcast_ref::>() + .unwrap(), + &expected, + ); +} + +#[test] +fn list_subsets_preserve_offsets_values_and_nulls() { + assert_list_subset_preserves_nulls::(); + assert_list_subset_preserves_nulls::(); +} + +#[test] +fn variant_and_opaque_comet_casts_are_not_identity_proofs() { + let storage = struct_type(vec![ + Field::new("value", DataType::Binary, false), + Field::new("metadata", DataType::Binary, false), + ]); + let field = Field::new("s", storage, true).with_extension_type(VariantType); + let (schema, expr) = adapt(field.clone(), field, options()); + assert!(expr.is::()); + assert!(!is_infallible_read_adaptation(&expr, &schema)); + + let source = Arc::new(Field::new("s", DataType::Int32, true)); + let target = Arc::new(Field::new("s", DataType::Int64, true)); + let expr: Arc = Arc::new( + CometCastColumnExpr::try_new( + Arc::new(Column::new("s", 0)), + Arc::clone(&source), + target, + None, + ) + .unwrap(), + ); + let schema = Schema::new(vec![source]); + assert!(!is_infallible_read_adaptation(&expr, &schema)); +} diff --git a/native/core/src/parquet/schema_adapter/runtime_filter.rs b/native/core/src/parquet/schema_adapter/runtime_filter.rs new file mode 100644 index 00000000000..11bf80fea2e --- /dev/null +++ b/native/core/src/parquet/schema_adapter/runtime_filter.rs @@ -0,0 +1,117 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Per-file read safety using the Spark adapter's resolved fields. + +use super::{ + is_infallible_read_adaptation, SparkPhysicalExprAdapter, SparkPhysicalExprAdapterFactory, +}; +use arrow::datatypes::{Schema, SchemaRef}; +use datafusion::common::Result; +use datafusion::datasource::physical_plan::FileScanConfig; +use datafusion::physical_expr::expressions::Column; +use datafusion::physical_expr_adapter::{PhysicalExprAdapter, PhysicalExprAdapterFactory}; +use parquet::variant::VariantType; +use std::collections::HashMap; +use std::sync::Arc; + +impl SparkPhysicalExprAdapterFactory { + /// Recover a typed factory only when every file carries this scan's active factory. + /// File extensions retain the type erased by DataFusion's adapter interface. Checking + /// pointer identity keeps a replaced or custom adapter on the ordinary rewrite path. + pub(crate) fn from_file_scan(scan: &FileScanConfig) -> Option> { + let active = scan.expr_adapter_factory.as_ref()?; + let mut files = scan.file_groups.iter().flat_map(|group| group.files()); + let factory = files.next()?.extensions.get_arc::()?; + let erased: Arc = Arc::::clone(&factory); + if !Arc::ptr_eq(active, &erased) { + return None; + } + files + .all(|file| { + file.extensions + .get_arc::() + .is_some_and(|other| Arc::ptr_eq(&factory, &other)) + }) + .then_some(factory) + } + + /// Create the normal per-file adapter and determine whether pruning can skip its reads. + pub(crate) fn create_with_read_safety( + &self, + logical_schema: SchemaRef, + physical_schema: SchemaRef, + read_columns: &[Column], + ) -> Result<(Arc, bool)> { + let adapter = self.create_adapter(logical_schema, Arc::clone(&physical_schema))?; + let allow_runtime_filter = + adapter.read_columns_are_infallible(read_columns, &physical_schema); + Ok((Arc::new(adapter), allow_runtime_filter)) + } +} + +impl SparkPhysicalExprAdapter { + fn read_columns_are_infallible(&self, columns: &[Column], physical_schema: &Schema) -> bool { + self.read_columns_are_direct(columns) + || columns.iter().all(|column| { + self.rewrite(Arc::new(column.clone())) + .is_ok_and(|expr| is_infallible_read_adaptation(&expr, physical_schema)) + }) + } + + fn read_columns_are_direct(&self, columns: &[Column]) -> bool { + // These paths have additional validation or value replacement in rewrite(). + // Preserve those checks, including defaults for columns outside this projection. + if self.original_physical_dup_check.is_some() + || self + .default_values + .as_ref() + .is_some_and(|values| !values.is_empty()) + { + return false; + } + + // Resolve names once per file, including stale predicate indices. A per-column + // Schema::index_of lookup would still make wide-file eligibility quadratic. + // Keep the first entry to match Arrow's field_with_name behavior. + let mut logical_fields = HashMap::new(); + for field in self.logical_file_schema.fields() { + logical_fields.entry(field.name().as_str()).or_insert(field); + } + let mut physical_fields = HashMap::new(); + for field in self.physical_file_schema.fields() { + physical_fields + .entry(field.name().as_str()) + .or_insert(field); + } + columns.iter().all(|column| { + let Some(logical) = logical_fields.get(column.name()) else { + return false; + }; + let Some(physical) = physical_fields.get(column.name()) else { + return false; + }; + // Equality includes nullability and metadata, so the default adapter returns + // a bare column. Spark still normalizes Variant even for equal fields. + // Case/field-ID remapping was already resolved when this adapter was created. + logical == physical && !logical.has_valid_extension_type::() + }) + } +} + +#[cfg(test)] +mod tests; diff --git a/native/core/src/parquet/schema_adapter/runtime_filter/tests.rs b/native/core/src/parquet/schema_adapter/runtime_filter/tests.rs new file mode 100644 index 00000000000..b7c286aa1ad --- /dev/null +++ b/native/core/src/parquet/schema_adapter/runtime_filter/tests.rs @@ -0,0 +1,129 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +mod factory; +mod resolution; + +use super::*; +use crate::parquet::parquet_support::SparkParquetOptions; +use arrow::datatypes::{DataType, Field, Schema}; +use datafusion::physical_expr::PhysicalExpr; +use datafusion_comet_spark_expr::EvalMode; +use std::sync::atomic::{AtomicUsize, Ordering}; + +#[derive(Debug)] +struct CountRewrites { + inner: Arc, + count: Arc, +} + +impl PhysicalExprAdapter for CountRewrites { + fn rewrite(&self, expr: Arc) -> Result> { + self.count.fetch_add(1, Ordering::Relaxed); + self.inner.rewrite(expr) + } +} + +fn factory() -> SparkPhysicalExprAdapterFactory { + let mut options = SparkParquetOptions::new(EvalMode::Legacy, "UTC", false); + options.case_sensitive = true; + SparkPhysicalExprAdapterFactory::new(options, None) +} + +fn counted_adapter( + factory: &SparkPhysicalExprAdapterFactory, + logical: SchemaRef, + physical: SchemaRef, +) -> (SparkPhysicalExprAdapter, Arc) { + let mut adapter = factory.create_adapter(logical, physical).unwrap(); + let count = Arc::new(AtomicUsize::new(0)); + adapter.default_adapter = Arc::new(CountRewrites { + inner: Arc::clone(&adapter.default_adapter), + count: Arc::clone(&count), + }); + (adapter, count) +} + +#[test] +fn matching_wide_schema_needs_no_eligibility_rewrites() { + let schema = Arc::new(Schema::new( + (0..256) + .map(|index| Field::new(format!("col_{index}"), DataType::Int64, true)) + .collect::>(), + )); + let columns = schema + .fields() + .iter() + // Include stale predicate indices: name resolution must establish safety. + .map(|field| Column::new(field.name(), usize::MAX)) + .collect::>(); + let (adapter, count) = counted_adapter(&factory(), Arc::clone(&schema), Arc::clone(&schema)); + assert!(adapter.read_columns_are_infallible(&columns, &schema)); + assert_eq!(count.load(Ordering::Relaxed), 0); +} + +#[test] +fn matching_schemas_do_not_prove_an_unresolved_column_safe() { + let schema = Arc::new(Schema::new(vec![Field::new("key", DataType::Int32, true)])); + let (adapter, count) = counted_adapter(&factory(), Arc::clone(&schema), Arc::clone(&schema)); + let columns = [Column::new("missing", 0)]; + assert!(!adapter.read_columns_are_infallible(&columns, &schema)); + assert_eq!(count.load(Ordering::Relaxed), 1); +} + +#[test] +fn matching_variant_fields_still_require_normalization() { + let storage = DataType::Struct( + vec![ + Field::new("value", DataType::Binary, false), + Field::new("metadata", DataType::Binary, false), + ] + .into(), + ); + let schema = Arc::new(Schema::new(vec![ + Field::new("payload", storage, true).with_extension_type(VariantType) + ])); + let (adapter, count) = counted_adapter(&factory(), Arc::clone(&schema), Arc::clone(&schema)); + assert!(!adapter.read_columns_are_infallible(&[Column::new("payload", 0)], &schema)); + assert_eq!(count.load(Ordering::Relaxed), 1); +} + +#[test] +fn conversion_in_a_required_payload_uses_the_rewrite_verdict() { + let physical = Arc::new(Schema::new(vec![ + Field::new("key", DataType::Int32, true), + Field::new("payload", DataType::Int32, true), + ])); + let logical = Arc::new(Schema::new(vec![ + Field::new("key", DataType::Int32, true), + Field::new("payload", DataType::Int64, true), + ])); + for allowed in [false, true] { + let mut factory = factory(); + factory.parquet_options.allow_type_promotion = allowed; + let (adapter, count) = + counted_adapter(&factory, Arc::clone(&logical), Arc::clone(&physical)); + assert_eq!( + adapter.read_columns_are_infallible( + &[Column::new("key", 0), Column::new("payload", usize::MAX)], + &physical, + ), + allowed, + ); + assert_eq!(count.load(Ordering::Relaxed), 2); + } +} diff --git a/native/core/src/parquet/schema_adapter/runtime_filter/tests/factory.rs b/native/core/src/parquet/schema_adapter/runtime_filter/tests/factory.rs new file mode 100644 index 00000000000..99a58595298 --- /dev/null +++ b/native/core/src/parquet/schema_adapter/runtime_filter/tests/factory.rs @@ -0,0 +1,80 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use super::*; +use datafusion::datasource::listing::PartitionedFile; +use datafusion::datasource::physical_plan::{FileGroup, FileScanConfigBuilder, ParquetSource}; +use datafusion::execution::object_store::ObjectStoreUrl; +use datafusion::physical_expr_adapter::DefaultPhysicalExprAdapterFactory; + +fn scan_with_factories( + active: Arc, + file_factories: &[Option>], +) -> FileScanConfig { + let schema = Arc::new(Schema::new(vec![Field::new("key", DataType::Int32, true)])); + let files = file_factories + .iter() + .enumerate() + .map(|(index, factory)| { + let mut file = PartitionedFile::new(format!("part-{index}.parquet"), 1); + // Unrelated extensions coexist with the concrete adapter factory. + file.extensions.insert(19_u64); + if let Some(factory) = factory { + file.extensions.insert_arc(Arc::clone(factory)); + } + file + }) + .collect(); + FileScanConfigBuilder::new( + ObjectStoreUrl::local_filesystem(), + Arc::new(ParquetSource::new(schema)), + ) + .with_file_group(FileGroup::new(files)) + .with_expr_adapter(Some(active)) + .build() +} + +#[test] +fn typed_factory_requires_every_file_to_identify_the_active_adapter() { + let factory = Arc::new(factory()); + let active: Arc = + Arc::::clone(&factory); + let mut scan = scan_with_factories( + Arc::clone(&active), + &[Some(Arc::clone(&factory)), Some(Arc::clone(&factory))], + ); + let recovered = SparkPhysicalExprAdapterFactory::from_file_scan(&scan).unwrap(); + assert!(Arc::ptr_eq(&factory, &recovered)); + assert_eq!(scan.file_groups[0].files()[0].extension::(), Some(&19)); + + scan.expr_adapter_factory = Some(Arc::new(DefaultPhysicalExprAdapterFactory)); + assert!(SparkPhysicalExprAdapterFactory::from_file_scan(&scan).is_none()); + + let unrelated = Arc::new(SparkPhysicalExprAdapterFactory::new( + factory.parquet_options.clone(), + None, + )); + for factories in [ + vec![Some(Arc::clone(&factory)), None], + vec![Some(Arc::clone(&factory)), Some(Arc::clone(&unrelated))], + vec![Some(unrelated)], + vec![], + ] { + let scan = scan_with_factories(Arc::clone(&active), &factories); + assert!(SparkPhysicalExprAdapterFactory::from_file_scan(&scan).is_none()); + } +} diff --git a/native/core/src/parquet/schema_adapter/runtime_filter/tests/resolution.rs b/native/core/src/parquet/schema_adapter/runtime_filter/tests/resolution.rs new file mode 100644 index 00000000000..fd5ea9e0040 --- /dev/null +++ b/native/core/src/parquet/schema_adapter/runtime_filter/tests/resolution.rs @@ -0,0 +1,126 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use super::*; +use arrow::array::{Int32Array, RecordBatch}; +use datafusion::common::ScalarValue; +use parquet::arrow::PARQUET_FIELD_ID_META_KEY; + +#[test] +fn matching_case_and_field_id_remaps_need_no_eligibility_rewrites() { + for use_field_id in [false, true] { + let mut factory = factory(); + factory.parquet_options.case_sensitive = false; + factory.parquet_options.use_field_id = use_field_id; + let physical_name = if use_field_id { "stored_key" } else { "KEY" }; + let mut logical_field = Field::new("key", DataType::Int32, true); + let mut physical_field = Field::new(physical_name, DataType::Int32, true); + if use_field_id { + let metadata = + HashMap::from([(PARQUET_FIELD_ID_META_KEY.to_string(), "7".to_string())]); + logical_field = logical_field.with_metadata(metadata.clone()); + physical_field = physical_field.with_metadata(metadata); + } + let logical = Arc::new(Schema::new(vec![logical_field])); + let physical = Arc::new(Schema::new(vec![physical_field])); + let (adapter, count) = counted_adapter(&factory, logical, Arc::clone(&physical)); + let column = Column::new("key", usize::MAX); + assert!(adapter.read_columns_are_infallible(std::slice::from_ref(&column), &physical)); + assert_eq!(count.load(Ordering::Relaxed), 0); + + // The eligibility shortcut must still leave normal expression adaptation responsible + // for restoring the physical name and repairing the stale projection index. + let rewritten = adapter.rewrite(Arc::new(column)).unwrap(); + assert_eq!(count.load(Ordering::Relaxed), 1); + assert_eq!( + rewritten.downcast_ref::(), + Some(&Column::new(physical_name, 0)) + ); + let values = Int32Array::from(vec![Some(7), None, Some(-2)]); + let batch = RecordBatch::try_new(physical, vec![Arc::new(values.clone())]).unwrap(); + let output = rewritten + .evaluate(&batch) + .unwrap() + .into_array(batch.num_rows()) + .unwrap(); + assert_eq!( + output.as_any().downcast_ref::().unwrap(), + &values + ); + } +} + +#[test] +fn case_and_field_id_remapping_keep_the_rewrite_fallback() { + for use_field_id in [false, true] { + let mut factory = factory(); + factory.parquet_options.case_sensitive = false; + factory.parquet_options.use_field_id = use_field_id; + let mut logical_field = Field::new("key", DataType::Int32, true); + let mut physical_field = Field::new( + if use_field_id { "stored_key" } else { "KEY" }, + DataType::Int32, + true, + ); + if use_field_id { + let metadata = + HashMap::from([(PARQUET_FIELD_ID_META_KEY.to_string(), "7".to_string())]); + logical_field = logical_field.with_metadata(metadata.clone()); + physical_field = physical_field.with_metadata(metadata); + } + // Different field metadata deliberately falls back to the normal adapter even + // though its identity cast ultimately leaves only a resolved physical column. + physical_field + .metadata_mut() + .insert("extra".to_string(), "file".to_string()); + let logical = Arc::new(Schema::new(vec![logical_field])); + let physical = Arc::new(Schema::new(vec![physical_field])); + let (adapter, count) = counted_adapter(&factory, logical, Arc::clone(&physical)); + assert!(adapter.read_columns_are_infallible(&[Column::new("key", usize::MAX)], &physical)); + assert_eq!(count.load(Ordering::Relaxed), 1); + } +} + +#[test] +fn missing_defaults_keep_the_rewrite_fallback() { + let mut factory = factory(); + factory.default_values = Some(HashMap::from([( + Column::new("payload", 1), + ScalarValue::Int32(Some(9)), + )])); + let logical = Arc::new(Schema::new(vec![ + Field::new("key", DataType::Int32, true), + Field::new("payload", DataType::Int32, true), + ])); + let physical = Arc::new(Schema::new(vec![Field::new("key", DataType::Int32, true)])); + let (adapter, count) = counted_adapter(&factory, logical, Arc::clone(&physical)); + assert!(adapter.read_columns_are_infallible(&[Column::new("payload", 1)], &physical)); + assert_eq!(count.load(Ordering::Relaxed), 1); +} + +#[test] +fn root_name_ambiguity_cannot_use_matching_fields() { + let mut factory = factory(); + factory.parquet_options.case_sensitive = false; + let logical = Arc::new(Schema::new(vec![Field::new("key", DataType::Int32, true)])); + let physical = Arc::new(Schema::new(vec![ + Field::new("key", DataType::Int32, true), + Field::new("KEY", DataType::Int32, true), + ])); + let (adapter, _) = counted_adapter(&factory, logical, Arc::clone(&physical)); + assert!(!adapter.read_columns_are_infallible(&[Column::new("key", 0)], &physical)); +}