Skip to content
3 changes: 3 additions & 0 deletions docs/source/user-guide/latest/iceberg.md
Original file line number Diff line number Diff line change
Expand Up @@ -196,6 +196,9 @@ The following scenarios will fall back to the JVM Iceberg reader:
or renamed. The native reader cannot yet match such a field to data files written before the
change. The check uses the table's schema history, so the fallback stays after those files are
rewritten
- Scans where no schema in the table's history gives a partition source or equality-delete key a
name that the scan's other columns don't use (every task of a native scan must give such a
column the same name)

Writes are not covered by this list. By default Iceberg writes use Spark's own writer; see
[Iceberg Writes](iceberg-writes.md) for the experimental native writer and when it applies.
Expand Down
10 changes: 10 additions & 0 deletions spark/src/main/scala/org/apache/comet/CometConf.scala
Original file line number Diff line number Diff line change
Expand Up @@ -152,6 +152,16 @@ object CometConf extends ShimCometConf {
.checkValue(v => v > 0, "Data file concurrency limit must be positive")
.createWithDefault(1)

val COMET_ICEBERG_NESTED_SCHEMA_PRUNING_ENABLED: ConfigEntry[Boolean] =
conf("spark.comet.scan.icebergNative.nestedSchemaPruning.enabled")
.category(CATEGORY_SCAN)
.doc(
"Whether the native Iceberg scan reads only the nested fields of a struct, list, or map " +
"column that Spark's nested schema pruning keeps. When false, it reads every nested " +
"field of a projected column and drops the unused ones after decoding.")
.booleanConf
.createWithDefault(true)

val COMET_ICEBERG_IO_TIMEOUT: ConfigEntry[Long] =
conf("spark.comet.iceberg.ioTimeout")
.category(CATEGORY_SCAN)
Expand Down
244 changes: 176 additions & 68 deletions spark/src/main/scala/org/apache/comet/iceberg/IcebergReflection.scala
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import org.apache.hadoop.conf.Configuration
import org.apache.spark.internal.Logging
import org.apache.spark.sql.SparkSession

import org.apache.comet.CometConf
import org.apache.comet.util.ClassLoaders

/**
Expand Down Expand Up @@ -707,89 +708,163 @@ object IcebergReflection extends Logging {
}
}

/** Returns the `Types.NestedField` for `fieldId` in `schema`, or None. */
def findFieldObject(schema: Any, fieldId: Int): Option[Any] = {
try {
val findFieldMethod = getMethod(schema.getClass, "findField", classOf[Int])
Option(findFieldMethod.invoke(schema, fieldId.asInstanceOf[AnyRef]))
} catch {
case _: Exception => None
/**
* Returns the `Types.NestedField` for `fieldId` at any depth of `schema`, or None when `schema`
* has no such field. A reflection failure propagates rather than reading as a missing field.
*/
def findFieldObject(schema: Any, fieldId: Int): Option[Any] =
Option(
getMethod(schema.getClass, "findField", classOf[Int])
.invoke(schema, fieldId.asInstanceOf[AnyRef]))

/**
* The task schemas of a native Iceberg scan, built once for all of its tasks during planning:
* the pruned scan schema, and the full schema read when pruning is off or cannot apply. Both
* hold the partition sources and equality-delete keys (`requiredFieldIds`) the tasks need
* beyond the projection: iceberg-rust resolves a task's partition spec against its schema, and
* reads the keys to apply equality deletes. The pruned one is None when a required column is
* nested inside a struct the query pruned away, as appending it at the top level would put it
* outside its struct. The full one is None, so that the scan falls back, when no name fits.
*
* Every task reads with the same schema because iceberg-rust builds an equality-delete
* predicate once per delete file, with the column names of the task that loads it first, and
* binds it by name in every other task of the reader. The full schema is the newest schema,
* current first, that has every field the scan reads, at any depth, with the same name and
* type, as the batch adaptation after iceberg-rust cannot narrow a promoted type back. That is
* the current schema unless a VERSION AS OF snapshot named or typed a field differently. A key
* that schema lacks, dropped before it, takes the newest name that no other column has.
*/
def nativeTaskSchemas(
table: Any,
tableSchema: Any,
scanSchema: Any,
requiredFieldIds: Set[Int]): (Option[Any], Option[Any]) = {
val prunedScanSchema = withoutMetadataColumns(scanSchema)
val scanFields = fieldsAtAnyDepth(prunedScanSchema)
val schemas = tableSchema +: getAllSchemas(table).reverse
val full = schemas.find { schema =>
scanFields.forall { case (id, name, leafType) =>
findFieldObject(schema, id).exists { field =>
fieldName(field) == name && leafType.forall(_ == fieldTypeOf(field))
}
}
}
full match {
case None => (None, None)
case Some(fullSchema) =>
val fullColumns = columnNamesById(fullSchema)
val fullNames = fullColumns.values.toSet
val ids = requiredFieldIds.toSeq.sorted
val inFull = ids.flatMap(id => findFieldObject(fullSchema, id).map(id -> _)).toMap
// Keys dropped before `fullSchema`, each under a name that no other column has.
val dropped =
ids.filterNot(inFull.contains).foldLeft(Option(Map.empty[Int, Any])) { (resolved, id) =>
resolved.flatMap { fields =>
val taken = fullNames ++ fields.values.map(fieldName)
schemas.iterator
.flatMap(findFieldObject(_, id))
.find(field => !taken.contains(fieldName(field)))
.map(field => fields + (id -> field))
}
}
dropped match {
case None => (None, None)
case Some(droppedFields) =>
val missing = ids.filter(findFieldObject(prunedScanSchema, _).isEmpty)
val prunedTaskSchema =
if (missing.forall(id => fullColumns.contains(id) || !inFull.contains(id))) {
val fields = missing.map(id => inFull.getOrElse(id, droppedFields(id)))
Some(withColumns(prunedScanSchema, fields))
} else {
None
}
val fullFields = droppedFields.toSeq.sortBy(_._1).map(_._2)
(prunedTaskSchema, Some(withColumns(fullSchema, fullFields)))

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[P2] [P2] Validate the historical full task schema for nested evolution. Create t(id INT, c INT, s STRUCT<k: STRING, a: INT>) USING iceberg PARTITIONED BY (s.k), write a file, add s.b INT, write another file, and record snapshot S. Then drop c and s.b. SELECT id, c, s.a FROM t VERSION AS OF S selects a historical full schema containing s.b, because c is historical and the unprojected nested partition source s.k prevents pruning. However, CometScanRule checks only metadata.tableSchema and metadata.scanSchema, neither of which contains s.b. Native execution consequently fails reading the older file with Incorrect number of arrays for StructArray fields, expected 3 got 2, while Spark and the base schema succeed. Both pruning settings select this unsafe full schema. Could the planning guard validate metadata.fullTaskSchema as well and fall back for this case?

Evidence: Fresh reproduction in /tmp/pr6725-6213-review-oct12-c2z7stms/: NestedHistoryProbe.scala writes (1,100,{k:east,a:10}), adds s.b, writes (2,200,{k:east,a:20,b:99}), records S, then drops c and s.b. Spark 4.1.3/Iceberg 1.11.0 returns [1,100,10] and [2,200,20]. The exact-head helpers produce prunedTaskSchema=None; the current guard reports no changes, whereas checking the selected full schema reports s.b (added). Reading those same files with pinned iceberg-rust af1da4c5bc86178c38c1c6db0578bdd9fb7021e3 reports BASE: SUCCESS rows=2 and HEAD: ERROR ... expected 3 got 2. The base augmentation helper also matches the current branch-1.1 implementation. Sources, generated schemas, and logs are preserved in that directory.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could the planning guard validate metadata.fullTaskSchema as well and fall back for this case?

I ran this table through Comet. At 6213d05, SELECT id, c, s.a FROM t VERSION AS OF S failed natively with pruning on and off. An unpartitioned variant also failed with pruning off: t(id INT, c INT, s STRUCT<a: INT>), add s.b, write a second file, take snapshot S, then drop c and s.b. At 8566834 both now fall back and match Spark. I left a related question on the new check in CometScanRule.scala.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Confirmed from the code, thanks. Since 6213d05 the full task schema can be older than the table's, and the nested-evolution check walked only the table and scan schemas. 8566834 adds the full task schema to that check, so your read falls back to Spark with s.b (added) instead of failing in the native scan. The struct fallback test now has your table. Keeping this case native would need the nested partition source merged back into its pruned struct, which I'd leave for a follow-up.

}
}
}

/** The top-level columns of `schema`, as field id to name. */
private def columnNamesById(schema: Any): Map[Int, String] = {
import scala.jdk.CollectionConverters._
getMethod(schema.getClass, "columns")
.invoke(schema)
.asInstanceOf[java.util.List[_]]
.asScala
.map(column => fieldIdOf(column) -> fieldName(column))
.toMap
}

/**
* Returns a schema equal to `baseSchema` but guaranteed to contain `requiredFieldIds`. Any id
* not already present is resolved from the table's schema history (`table.schemas()`) and
* appended.
*
* Two callers need this: an equality delete may be keyed on a column since dropped from the
* current schema (mirroring Iceberg-Java's `DeleteFilter.fileProjection`), and a partition
* spec's source columns must be present for iceberg-rust to resolve `partition_type` when the
* query projects them out. Called at serialization time, so it throws on failure (a required id
* that cannot be resolved, or any reflection error) rather than silently degrading.
* CometScanRule is responsible for falling back before we get here.
*/
def schemaWithRequiredFields(baseSchema: Any, table: Any, requiredFieldIds: Seq[Int]): Any = {
// `findFieldObject` searches recursively, so a source column already present as a nested field
// (e.g. `s.region`) is not re-appended at the top level, which would create a duplicate field
// id. Empty `requiredFieldIds` (the common non-partitioned, no-delete task) does no lookups.
val missingIds =
requiredFieldIds.distinct.filter(id => findFieldObject(baseSchema, id).isEmpty)
if (missingIds.isEmpty) {
baseSchema
* The fields of `schema` at any depth, as id, name, and the type of a field with no children.
*/
private def fieldsAtAnyDepth(schema: Any): Seq[(Int, String, Option[Any])] = {
import scala.jdk.CollectionConverters._
def walk(field: Any): Seq[(Int, String, Option[Any])] = {
val children = childFields(field)
val leafType = if (children.isEmpty) Some(fieldTypeOf(field)) else None
(fieldIdOf(field), fieldName(field), leafType) +: children.flatMap(walk)
}
getMethod(schema.getClass, "columns")
.invoke(schema)
.asInstanceOf[java.util.List[_]]
.asScala
.toSeq
.flatMap(walk)
}

/** `schema` with `fields` appended at its top level, or `schema` itself when there are none. */
private def withColumns(schema: Any, fields: Seq[Any]): Any =
if (fields.isEmpty) {
schema
} else {
logDebug(
s"Native Iceberg scan schema is missing field id(s) ${missingIds.mkString(",")}; " +
"resolving them from table schema history")
val history = getAllSchemas(table)
val resolvedFields = missingIds.map { id =>
history.iterator
.flatMap(s => findFieldObject(s, id))
.toSeq
.headOption
.getOrElse(throw new IllegalStateException(
s"Cannot resolve field id $id in table schema history"))
}
val existing =
getMethod(baseSchema.getClass, "columns")
.invoke(baseSchema)
.asInstanceOf[java.util.List[_]]
val newColumns = new java.util.ArrayList[Any](existing)
resolvedFields.foreach(newColumns.add)
baseSchema.getClass
val columns =
getMethod(schema.getClass, "columns").invoke(schema).asInstanceOf[java.util.List[_]]
val newColumns = new java.util.ArrayList[Any](columns.size + fields.length)
newColumns.addAll(columns)
fields.foreach(newColumns.add)
schema.getClass
.getConstructor(classOf[java.util.List[_]])
.newInstance(newColumns)
.asInstanceOf[AnyRef]
}
}

/** The spec reserves field ids above `Integer.MAX_VALUE - 200` for metadata columns. */
val MaxDataFieldId: Int = Int.MaxValue - 200

/**
* The source column field ids referenced by a task's partition spec.
*
* iceberg-rust validates a `FileScanTask` by resolving its partition spec against the task
* schema (`partition_type(schema)`), so a task carrying a partition spec needs those source
* columns present in the schema even when the query projects them out (for example selecting
* only `_spec_id` or `_partition`). These ids are unioned into the native task schema via
* [[schemaWithRequiredFields]]. project_field_ids still drives the read, so the columns are not
* materialized into the scan output. All ids resolve from the table's schema history, including
* a source column later dropped by partition evolution.
*
* Returns an empty sequence when the task has no partition spec.
* Returns `schema` without its top-level metadata columns (`_file`, `_pos`, `_partition` and
* the other reserved ids), or `schema` itself when it has none. A scan's expected schema
* includes the metadata columns the query selects, which a table schema never has.
*/
def partitionSourceFieldIds(task: Any, fileScanTaskClass: Class[_]): Seq[Int] = {
val spec =
try {
getMethod(fileScanTaskClass, "spec").invoke(task)
} catch {
case _: Exception => null
}
if (spec == null) Seq.empty else partitionFieldSourceIds(spec)
def withoutMetadataColumns(schema: Any): Any = {
import scala.jdk.CollectionConverters._
val columns =
getMethod(schema.getClass, "columns").invoke(schema).asInstanceOf[java.util.List[_]]
val dataColumns = new java.util.ArrayList[Any](columns.size)
columns.asScala.iterator
.filter(fieldIdOf(_) <= MaxDataFieldId)
.foreach(column => dataColumns.add(column))
if (dataColumns.size == columns.size) {
schema
} else {
schema.getClass
.getConstructor(classOf[java.util.List[_]])
.newInstance(dataColumns)
.asInstanceOf[AnyRef]
}
}

/**
* The `sourceId`s of a partition spec's fields, in spec order. Empty for an unpartitioned spec.
*
* iceberg-rust validates a `FileScanTask` by resolving its partition spec against the task
* schema (`partition_type(schema)`), so the source columns must be in the task schema even when
* the query projects them out (for example selecting only `_spec_id` or `_partition`). See
* [[nativeTaskSchemas]]. project_field_ids still drives the read, so the columns are not
* materialized into the scan output.
*/
private def partitionFieldSourceIds(spec: Any): Seq[Int] = {
def partitionFieldSourceIds(spec: Any): Seq[Int] = {
import scala.jdk.CollectionConverters._
// Exclude fields whose source column was dropped (unknown result type): their source is absent
// from the current schema and the serialized spec/values omit them too (see
Expand Down Expand Up @@ -1336,6 +1411,8 @@ object IcebergReflection extends Logging {
private def fieldIdOf(field: Any): Int =
getMethod(field.getClass, "fieldId").invoke(field).asInstanceOf[Int]

private def fieldTypeOf(field: Any): Any = getMethod(field.getClass, "type").invoke(field)

/**
* Converts an Iceberg `Schema` to the Spark `StructType` it reads as, via
* `SparkSchemaUtil.convert`. Comet serializes the whole table/scan schema to native (not just
Expand Down Expand Up @@ -2344,6 +2421,14 @@ object IcebergReflection extends Logging {
* List of FileScanTask objects from Iceberg planning
* @param scanSchema
* The expectedSchema from the SparkScan (for schema evolution / VERSION AS OF)
* @param prunedTaskSchema
* The schema every task reads with when nested schema pruning is on: scanSchema without its
* metadata columns, which iceberg-rust resolves by reserved field id, plus the partition
* sources and equality-delete keys the tasks need. None when one of those is nested inside a
* struct the query pruned away. See [[IcebergReflection.nativeTaskSchemas]].
* @param fullTaskSchema
* The schema every task reads with otherwise: the scan's full table schema plus the
* equality-delete keys it lacks. None when no name fits every task, which falls back to Spark.
* @param globalFieldIdMapping
* Mapping from column names to Iceberg field IDs (built from scanSchema)
* @param catalogProperties
Expand All @@ -2358,11 +2443,26 @@ case class CometIcebergNativeScanMetadata(
nameMapping: Option[String],
@transient tasks: java.util.List[_],
scanSchema: Any,
prunedTaskSchema: Option[Any],
fullTaskSchema: Option[Any],
tableSchema: Any,
globalFieldIdMapping: Map[String, Int],
catalogProperties: Map[String, String],
catalogName: Option[String],
fileFormat: String)
fileFormat: String) {

/**
* The schema every task of the scan reads with: the pruned task schema while nested schema
* pruning is on and can apply, the full one otherwise. It is fixed when `extract` builds the
* metadata during planning, so the schema CometScanRule checks is the one
* CometIcebergNativeScan serializes at execution, even if the config changes in between. None
* falls back to Spark.
*/
val taskSchema: Option[Any] =
prunedTaskSchema
.filter(_ => CometConf.COMET_ICEBERG_NESTED_SCHEMA_PRUNING_ENABLED.get())
.orElse(fullTaskSchema)
}

object CometIcebergNativeScanMetadata extends Logging {

Expand All @@ -2382,14 +2482,18 @@ object CometIcebergNativeScanMetadata extends Logging {
* The scan's FileScanTasks (already extracted). Passed in rather than re-read via
* [[IcebergReflection.getTasks]], which for a staged scan rebuilds a flattened list of every
* task on each call.
* @param requiredFieldIds
* The partition sources and equality-delete keys of those tasks, which every task schema
* holds
* @return
* Some(metadata) if all reflection succeeds, None to trigger fallback
*/
def extract(
scan: Any,
metadataLocation: String,
catalogProperties: Map[String, String],
tasks: java.util.List[_]): Option[CometIcebergNativeScanMetadata] = {
tasks: java.util.List[_],
requiredFieldIds: Set[Int]): Option[CometIcebergNativeScanMetadata] = {
import org.apache.comet.iceberg.IcebergReflection._

for {
Expand All @@ -2408,6 +2512,8 @@ object CometIcebergNativeScanMetadata extends Logging {
}

val globalFieldIdMapping = buildFieldIdMapping(scanSchema)
val (prunedTaskSchema, fullTaskSchema) =
nativeTaskSchemas(table, tableSchema, scanSchema, requiredFieldIds)

// File format is always PARQUET,
// validated in CometScanRule.validateIcebergFileScanTasks()
Expand All @@ -2418,6 +2524,8 @@ object CometIcebergNativeScanMetadata extends Logging {
nameMapping = nameMapping,
tasks = tasks,
scanSchema = scanSchema,
prunedTaskSchema = prunedTaskSchema,
fullTaskSchema = fullTaskSchema,
tableSchema = tableSchema,
globalFieldIdMapping = globalFieldIdMapping,
catalogProperties = catalogProperties,
Expand Down
Loading
Loading