From eea51fe6408b0b376ea955f7aeb8961271c0bbf1 Mon Sep 17 00:00:00 2001 From: Sergey Zhukov Date: Fri, 21 Nov 2025 09:25:01 +0300 Subject: [PATCH 1/2] Consolidate dataframe examples (#18142) --- datafusion-examples/README.md | 8 +- .../examples/{ => dataframe}/dataframe.rs | 13 +-- .../{ => dataframe}/default_column_values.rs | 45 ++++---- .../{ => dataframe}/deserialize_to_struct.rs | 5 +- .../examples/dataframe/main.rs | 107 ++++++++++++++++++ 5 files changed, 141 insertions(+), 37 deletions(-) rename datafusion-examples/examples/{ => dataframe}/dataframe.rs (97%) rename datafusion-examples/examples/{ => dataframe}/default_column_values.rs (92%) rename datafusion-examples/examples/{ => dataframe}/deserialize_to_struct.rs (98%) create mode 100644 datafusion-examples/examples/dataframe/main.rs diff --git a/datafusion-examples/README.md b/datafusion-examples/README.md index 54534801659db..e097c3fa5ddf5 100644 --- a/datafusion-examples/README.md +++ b/datafusion-examples/README.md @@ -41,7 +41,7 @@ cd datafusion-examples/examples # Run the `dataframe` example: # ... use the equivalent for other examples -cargo run --example dataframe +cargo run --example dataframe -- dataframe ``` ## Single Process @@ -61,10 +61,10 @@ cargo run --example dataframe - [`examples/custom_data_source/custom_file_casts.rs`](examples/custom_data_source/custom_file_casts.rs): Implement custom casting rules to adapt file schemas - [`examples/custom_data_source/custom_file_format.rs`](examples/custom_data_source/custom_file_format.rs): Write data to a custom file format - [`dataframe-to-s3.rs`](examples/external_dependency/dataframe-to-s3.rs): Run a query using a DataFrame against a parquet file from s3 and writing back to s3 -- [`dataframe.rs`](examples/dataframe.rs): Run a query using a DataFrame API against parquet files, csv files, and in-memory data, including multiple subqueries. Also demonstrates the various methods to write out a DataFrame to a table, parquet file, csv file, and json file. +- [`examples/dataframe/dataframe.rs`](examples/dataframe/dataframe.rs): Run a query using a DataFrame API against parquet files, csv files, and in-memory data, including multiple subqueries. Also demonstrates the various methods to write out a DataFrame to a table, parquet file, csv file, and json file. - [`examples/builtin_functions/date_time`](examples/builtin_functions/date_time.rs): Examples of date-time related functions and queries -- [`default_column_values.rs`](examples/default_column_values.rs): Implement custom default value handling for missing columns using field metadata and PhysicalExprAdapter -- [`deserialize_to_struct.rs`](examples/deserialize_to_struct.rs): Convert query results (Arrow ArrayRefs) into Rust structs +- [`examples/dataframe/default_column_values.rs`](examples/dataframe/default_column_values.rs): Implement custom default value handling for missing columns using field metadata and PhysicalExprAdapter +- [`examples/dataframe/deserialize_to_struct.rs`](examples/dataframe/deserialize_to_struct.rs): Convert query results (Arrow ArrayRefs) into Rust structs - [`examples/query_planning/expr_api.rs`](examples/query_planning/expr_api.rs): Create, execute, simplify, analyze and coerce `Expr`s - [`examples/custom_data_source/file_stream_provider.rs`](examples/custom_data_source/file_stream_provider.rs): Run a query on `FileStreamProvider` which implements `StreamProvider` for reading and writing to arbitrary stream sources / sinks. - [`flight/sql_server.rs`](examples/flight/sql_server.rs): Run DataFusion as a standalone process and execute SQL queries from Flight and and FlightSQL (e.g. JDBC) clients diff --git a/datafusion-examples/examples/dataframe.rs b/datafusion-examples/examples/dataframe/dataframe.rs similarity index 97% rename from datafusion-examples/examples/dataframe.rs rename to datafusion-examples/examples/dataframe/dataframe.rs index a5ee571a14764..7ba4464b53d9f 100644 --- a/datafusion-examples/examples/dataframe.rs +++ b/datafusion-examples/examples/dataframe/dataframe.rs @@ -15,12 +15,13 @@ // specific language governing permissions and limitations // under the License. +//! See `main.rs` for how to run it. + use arrow::array::{ArrayRef, Int32Array, RecordBatch, StringArray, StringViewArray}; use datafusion::arrow::datatypes::{DataType, Field, Schema}; use datafusion::catalog::MemTable; use datafusion::common::config::CsvOptions; use datafusion::common::parsers::CompressionTypeVariant; -use datafusion::common::DataFusionError; use datafusion::common::ScalarValue; use datafusion::dataframe::DataFrameWriteOptions; use datafusion::error::Result; @@ -39,6 +40,7 @@ use tempfile::tempdir; /// * [read_parquet]: execute queries against parquet files /// * [read_csv]: execute queries against csv files /// * [read_memory]: execute queries against in-memory arrow data +/// * [read_memory_macro]: execute queries against in-memory arrow data using macro /// /// # Writing out to local storage /// @@ -53,12 +55,7 @@ use tempfile::tempdir; /// * [where_scalar_subquery]: execute a scalar subquery /// * [where_in_subquery]: execute a subquery with an IN clause /// * [where_exist_subquery]: execute a subquery with an EXISTS clause -/// -/// # Querying data -/// -/// * [query_to_date]: execute queries against parquet files -#[tokio::main] -async fn main() -> Result<()> { +pub async fn dataframe_example() -> Result<()> { env_logger::init(); // The SessionContext is the main high level API for interacting with DataFusion let ctx = SessionContext::new(); @@ -199,7 +196,7 @@ async fn read_memory_macro() -> Result<()> { /// 2. Write out a DataFrame to a parquet file /// 3. Write out a DataFrame to a csv file /// 4. Write out a DataFrame to a json file -async fn write_out(ctx: &SessionContext) -> std::result::Result<(), DataFusionError> { +async fn write_out(ctx: &SessionContext) -> Result<()> { let array = StringViewArray::from(vec!["a", "b", "c"]); let schema = Arc::new(Schema::new(vec![Field::new( "tablecol1", diff --git a/datafusion-examples/examples/default_column_values.rs b/datafusion-examples/examples/dataframe/default_column_values.rs similarity index 92% rename from datafusion-examples/examples/default_column_values.rs rename to datafusion-examples/examples/dataframe/default_column_values.rs index bfc60519f26e4..19d00e0a0d6f2 100644 --- a/datafusion-examples/examples/default_column_values.rs +++ b/datafusion-examples/examples/dataframe/default_column_values.rs @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +//! See `main.rs` for how to run it. + use std::any::Any; use std::collections::HashMap; use std::sync::Arc; @@ -52,25 +54,23 @@ use object_store::{ObjectStore, PutPayload}; // Metadata key for storing default values in field metadata const DEFAULT_VALUE_METADATA_KEY: &str = "example.default_value"; -// Example showing how to implement custom default value handling for missing columns -// using field metadata and PhysicalExprAdapter. -// -// This example demonstrates how to: -// 1. Store default values in field metadata using a constant key -// 2. Create a custom PhysicalExprAdapter that reads these defaults -// 3. Inject default values for missing columns in filter predicates -// 4. Use the DefaultPhysicalExprAdapter as a fallback for standard schema adaptation -// 5. Wrap string default values in cast expressions for proper type conversion -// -// Important: PhysicalExprAdapter is specifically designed for rewriting filter predicates -// that get pushed down to file scans. For handling missing columns in projections, -// other mechanisms in DataFusion are used (like SchemaAdapter). -// -// The metadata-based approach provides a flexible way to store default values as strings -// and cast them to the appropriate types at query time. - -#[tokio::main] -async fn main() -> Result<()> { +/// Example showing how to implement custom default value handling for missing columns +/// using field metadata and PhysicalExprAdapter. +/// +/// This example demonstrates how to: +/// 1. Store default values in field metadata using a constant key +/// 2. Create a custom PhysicalExprAdapter that reads these defaults +/// 3. Inject default values for missing columns in filter predicates +/// 4. Use the DefaultPhysicalExprAdapter as a fallback for standard schema adaptation +/// 5. Wrap string default values in cast expressions for proper type conversion +/// +/// Important: PhysicalExprAdapter is specifically designed for rewriting filter predicates +/// that get pushed down to file scans. For handling missing columns in projections, +/// other mechanisms in DataFusion are used (like SchemaAdapter). +/// +/// The metadata-based approach provides a flexible way to store default values as strings +/// and cast them to the appropriate types at query time. +pub async fn default_column_values() -> Result<()> { println!("=== Creating example data with missing columns and default values ==="); // Create sample data where the logical schema has more columns than the physical schema @@ -85,11 +85,10 @@ async fn main() -> Result<()> { .build(); let mut writer = - ArrowWriter::try_new(&mut buf, physical_schema.clone(), Some(props)) - .expect("creating writer"); + ArrowWriter::try_new(&mut buf, physical_schema.clone(), Some(props))?; - writer.write(&batch).expect("Writing batch"); - writer.close().unwrap(); + writer.write(&batch)?; + writer.close()?; buf }; let path = Path::from("example.parquet"); diff --git a/datafusion-examples/examples/deserialize_to_struct.rs b/datafusion-examples/examples/dataframe/deserialize_to_struct.rs similarity index 98% rename from datafusion-examples/examples/deserialize_to_struct.rs rename to datafusion-examples/examples/dataframe/deserialize_to_struct.rs index d6655b3b654f9..e19d45554131a 100644 --- a/datafusion-examples/examples/deserialize_to_struct.rs +++ b/datafusion-examples/examples/dataframe/deserialize_to_struct.rs @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +//! See `main.rs` for how to run it. + use arrow::array::{AsArray, PrimitiveArray}; use arrow::datatypes::{Float64Type, Int32Type}; use datafusion::common::assert_batches_eq; @@ -29,8 +31,7 @@ use futures::StreamExt; /// as [ArrayRef] /// /// [ArrayRef]: arrow::array::ArrayRef -#[tokio::main] -async fn main() -> Result<()> { +pub async fn deserialize_to_struct() -> Result<()> { // Run a query that returns two columns of data let ctx = SessionContext::new(); let testdata = datafusion::test_util::parquet_test_data(); diff --git a/datafusion-examples/examples/dataframe/main.rs b/datafusion-examples/examples/dataframe/main.rs new file mode 100644 index 0000000000000..4cace0590567d --- /dev/null +++ b/datafusion-examples/examples/dataframe/main.rs @@ -0,0 +1,107 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! # These are core DataFrame API usage +//! +//! These examples demonstrate core DataFrame API usage. +//! +//! ## Usage +//! ```bash +//! cargo run --example dataframe -- [dataframe|default_column_values|deserialize_to_struct] +//! ``` +//! +//! Each subcommand runs a corresponding example: +//! - `dataframe` — run a query using a DataFrame API against parquet files, csv files, and in-memory data, including multiple subqueries +//! - `default_column_values` — implement custom default value handling for missing columns using field metadata and PhysicalExprAdapter +//! - `deserialize_to_struct` — convert query results (Arrow ArrayRefs) into Rust structs + +mod dataframe; +mod default_column_values; +mod deserialize_to_struct; + +use std::str::FromStr; + +use datafusion::error::{DataFusionError, Result}; + +enum ExampleKind { + Dataframe, + DefaultColumnValues, + DeserializeToStruct, +} + +impl AsRef for ExampleKind { + fn as_ref(&self) -> &str { + match self { + Self::Dataframe => "dataframe", + Self::DefaultColumnValues => "default_column_values", + Self::DeserializeToStruct => "deserialize_to_struct", + } + } +} + +impl FromStr for ExampleKind { + type Err = DataFusionError; + + fn from_str(s: &str) -> Result { + match s { + "dataframe" => Ok(Self::Dataframe), + "default_column_values" => Ok(Self::DefaultColumnValues), + "deserialize_to_struct" => Ok(Self::DeserializeToStruct), + _ => Err(DataFusionError::Execution(format!("Unknown example: {s}"))), + } + } +} + +impl ExampleKind { + const ALL: [Self; 3] = [ + Self::Dataframe, + Self::DefaultColumnValues, + Self::DeserializeToStruct, + ]; + + const EXAMPLE_NAME: &str = "dataframe"; + + fn variants() -> Vec<&'static str> { + Self::ALL.iter().map(|x| x.as_ref()).collect() + } +} + +#[tokio::main] +async fn main() -> Result<()> { + let usage = format!( + "Usage: cargo run --example {} -- [{}]", + ExampleKind::EXAMPLE_NAME, + ExampleKind::variants().join("|") + ); + + let arg = std::env::args().nth(1).ok_or_else(|| { + eprintln!("{usage}"); + DataFusionError::Execution("Missing argument".to_string()) + })?; + + match arg.parse::()? { + ExampleKind::Dataframe => dataframe::dataframe_example().await?, + ExampleKind::DefaultColumnValues => { + default_column_values::default_column_values().await? + } + ExampleKind::DeserializeToStruct => { + deserialize_to_struct::deserialize_to_struct().await? + } + } + + Ok(()) +} From 1559334a036b9c083f9704120a85ce0aa19bc990 Mon Sep 17 00:00:00 2001 From: Sergey Zhukov Date: Sat, 22 Nov 2025 16:00:14 +0300 Subject: [PATCH 2/2] Move default_column_values from dataframe to custom_data_source --- datafusion-examples/README.md | 2 +- .../default_column_values.rs | 0 .../examples/custom_data_source/main.rs | 13 +++++++++++-- datafusion-examples/examples/dataframe/main.rs | 16 ++-------------- 4 files changed, 14 insertions(+), 17 deletions(-) rename datafusion-examples/examples/{dataframe => custom_data_source}/default_column_values.rs (100%) diff --git a/datafusion-examples/README.md b/datafusion-examples/README.md index e097c3fa5ddf5..2d21a2731a096 100644 --- a/datafusion-examples/README.md +++ b/datafusion-examples/README.md @@ -63,7 +63,7 @@ cargo run --example dataframe -- dataframe - [`dataframe-to-s3.rs`](examples/external_dependency/dataframe-to-s3.rs): Run a query using a DataFrame against a parquet file from s3 and writing back to s3 - [`examples/dataframe/dataframe.rs`](examples/dataframe/dataframe.rs): Run a query using a DataFrame API against parquet files, csv files, and in-memory data, including multiple subqueries. Also demonstrates the various methods to write out a DataFrame to a table, parquet file, csv file, and json file. - [`examples/builtin_functions/date_time`](examples/builtin_functions/date_time.rs): Examples of date-time related functions and queries -- [`examples/dataframe/default_column_values.rs`](examples/dataframe/default_column_values.rs): Implement custom default value handling for missing columns using field metadata and PhysicalExprAdapter +- [`examples/custom_data_source/default_column_values.rs`](examples/custom_data_source/default_column_values.rs): Implement custom default value handling for missing columns using field metadata and PhysicalExprAdapter - [`examples/dataframe/deserialize_to_struct.rs`](examples/dataframe/deserialize_to_struct.rs): Convert query results (Arrow ArrayRefs) into Rust structs - [`examples/query_planning/expr_api.rs`](examples/query_planning/expr_api.rs): Create, execute, simplify, analyze and coerce `Expr`s - [`examples/custom_data_source/file_stream_provider.rs`](examples/custom_data_source/file_stream_provider.rs): Run a query on `FileStreamProvider` which implements `StreamProvider` for reading and writing to arbitrary stream sources / sinks. diff --git a/datafusion-examples/examples/dataframe/default_column_values.rs b/datafusion-examples/examples/custom_data_source/default_column_values.rs similarity index 100% rename from datafusion-examples/examples/dataframe/default_column_values.rs rename to datafusion-examples/examples/custom_data_source/default_column_values.rs diff --git a/datafusion-examples/examples/custom_data_source/main.rs b/datafusion-examples/examples/custom_data_source/main.rs index ce0585f8c3f7b..66d8f083422e2 100644 --- a/datafusion-examples/examples/custom_data_source/main.rs +++ b/datafusion-examples/examples/custom_data_source/main.rs @@ -21,7 +21,7 @@ //! //! ## Usage //! ```bash -//! cargo run --example custom_data_source -- [csv_json_opener|csv_sql_streaming|custom_datasource|custom_file_casts|custom_file_format|file_stream_provider] +//! cargo run --example custom_data_source -- [csv_json_opener|csv_sql_streaming|custom_datasource|custom_file_casts|custom_file_format|default_column_values|file_stream_provider] //! ``` //! //! Each subcommand runs a corresponding example: @@ -30,6 +30,7 @@ //! - `custom_datasource` — run queries against a custom datasource (TableProvider) //! - `custom_file_casts` — implement custom casting rules to adapt file schemas //! - `custom_file_format` — write data to a custom file format +//! - `default_column_values` — implement custom default value handling for missing columns using field metadata and PhysicalExprAdapter //! - `file_stream_provider` — run a query on FileStreamProvider which implements StreamProvider for reading and writing to arbitrary stream sources/sinks mod csv_json_opener; @@ -37,6 +38,7 @@ mod csv_sql_streaming; mod custom_datasource; mod custom_file_casts; mod custom_file_format; +mod default_column_values; mod file_stream_provider; use std::str::FromStr; @@ -49,6 +51,7 @@ enum ExampleKind { CustomDatasource, CustomFileCasts, CustomFileFormat, + DefaultColumnValues, FileFtreamProvider, } @@ -60,6 +63,7 @@ impl AsRef for ExampleKind { Self::CustomDatasource => "custom_datasource", Self::CustomFileCasts => "custom_file_casts", Self::CustomFileFormat => "custom_file_format", + Self::DefaultColumnValues => "default_column_values", Self::FileFtreamProvider => "file_stream_provider", } } @@ -75,6 +79,7 @@ impl FromStr for ExampleKind { "custom_datasource" => Ok(Self::CustomDatasource), "custom_file_casts" => Ok(Self::CustomFileCasts), "custom_file_format" => Ok(Self::CustomFileFormat), + "default_column_values" => Ok(Self::DefaultColumnValues), "file_stream_provider" => Ok(Self::FileFtreamProvider), _ => Err(DataFusionError::Execution(format!("Unknown example: {s}"))), } @@ -82,12 +87,13 @@ impl FromStr for ExampleKind { } impl ExampleKind { - const ALL: [Self; 6] = [ + const ALL: [Self; 7] = [ Self::CsvJsonOpener, Self::CsvSqlStreaming, Self::CustomDatasource, Self::CustomFileCasts, Self::CustomFileFormat, + Self::DefaultColumnValues, Self::FileFtreamProvider, ]; @@ -117,6 +123,9 @@ async fn main() -> Result<()> { ExampleKind::CustomDatasource => custom_datasource::custom_datasource().await?, ExampleKind::CustomFileCasts => custom_file_casts::custom_file_casts().await?, ExampleKind::CustomFileFormat => custom_file_format::custom_file_format().await?, + ExampleKind::DefaultColumnValues => { + default_column_values::default_column_values().await? + } ExampleKind::FileFtreamProvider => { file_stream_provider::file_stream_provider().await? } diff --git a/datafusion-examples/examples/dataframe/main.rs b/datafusion-examples/examples/dataframe/main.rs index 4cace0590567d..0846bce2e811d 100644 --- a/datafusion-examples/examples/dataframe/main.rs +++ b/datafusion-examples/examples/dataframe/main.rs @@ -21,16 +21,14 @@ //! //! ## Usage //! ```bash -//! cargo run --example dataframe -- [dataframe|default_column_values|deserialize_to_struct] +//! cargo run --example dataframe -- [dataframe|deserialize_to_struct] //! ``` //! //! Each subcommand runs a corresponding example: //! - `dataframe` — run a query using a DataFrame API against parquet files, csv files, and in-memory data, including multiple subqueries -//! - `default_column_values` — implement custom default value handling for missing columns using field metadata and PhysicalExprAdapter //! - `deserialize_to_struct` — convert query results (Arrow ArrayRefs) into Rust structs mod dataframe; -mod default_column_values; mod deserialize_to_struct; use std::str::FromStr; @@ -39,7 +37,6 @@ use datafusion::error::{DataFusionError, Result}; enum ExampleKind { Dataframe, - DefaultColumnValues, DeserializeToStruct, } @@ -47,7 +44,6 @@ impl AsRef for ExampleKind { fn as_ref(&self) -> &str { match self { Self::Dataframe => "dataframe", - Self::DefaultColumnValues => "default_column_values", Self::DeserializeToStruct => "deserialize_to_struct", } } @@ -59,7 +55,6 @@ impl FromStr for ExampleKind { fn from_str(s: &str) -> Result { match s { "dataframe" => Ok(Self::Dataframe), - "default_column_values" => Ok(Self::DefaultColumnValues), "deserialize_to_struct" => Ok(Self::DeserializeToStruct), _ => Err(DataFusionError::Execution(format!("Unknown example: {s}"))), } @@ -67,11 +62,7 @@ impl FromStr for ExampleKind { } impl ExampleKind { - const ALL: [Self; 3] = [ - Self::Dataframe, - Self::DefaultColumnValues, - Self::DeserializeToStruct, - ]; + const ALL: [Self; 2] = [Self::Dataframe, Self::DeserializeToStruct]; const EXAMPLE_NAME: &str = "dataframe"; @@ -95,9 +86,6 @@ async fn main() -> Result<()> { match arg.parse::()? { ExampleKind::Dataframe => dataframe::dataframe_example().await?, - ExampleKind::DefaultColumnValues => { - default_column_values::default_column_values().await? - } ExampleKind::DeserializeToStruct => { deserialize_to_struct::deserialize_to_struct().await? }