sunchao commented on code in PR #5365: URL: https://github.com/apache/datafusion-comet/pull/5365#discussion_r3865631945
########## contrib/delta-spark/src/main/scala/org/apache/comet/contrib/delta/DeltaScanSupport.scala: ########## @@ -0,0 +1,1000 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.comet.contrib.delta + +import java.io.IOException +import java.net.URI +import java.util.Locale + +import scala.collection.mutable.ListBuffer +import scala.jdk.CollectionConverters._ + +import org.apache.hadoop.conf.Configuration +import org.apache.hadoop.fs.Path +import org.apache.spark.sql.catalyst.expressions.{Alias, GenericInternalRow, InputFileBlockLength, InputFileBlockStart, InputFileName} +import org.apache.spark.sql.catalyst.util.{ArrayBasedMapData, GenericArrayData} +import org.apache.spark.sql.catalyst.util.ResolveDefaultColumns.getExistenceDefaultValues +import org.apache.spark.sql.comet.CometScanExec +import org.apache.spark.sql.delta.DeltaParquetFileFormat +import org.apache.spark.sql.delta.actions.DeletionVectorDescriptor +import org.apache.spark.sql.execution.{FileSourceScanExec, ProjectExec, SparkPlan} +import org.apache.spark.sql.internal.SQLConf +import org.apache.spark.sql.types.{ArrayType, DataType, MapType, StructType} + +import org.apache.comet.CometConf +import org.apache.comet.CometConf.COMET_LIBHDFS_SCHEMES +import org.apache.comet.parquet.CometParquetUtils +import org.apache.comet.rules.{CometScanRule, CometScanTypeChecker} +import org.apache.comet.serde.operator.CometNativeScan +import org.apache.comet.shims.ShimFileFormat + +/** + * Claim/decline gates for the native Delta scan. Correctness rule: when in doubt, decline, + * Spark's Delta reader handles the scan and results stay correct, just unaccelerated. + */ +object DeltaScanSupport { + + /** + * Reader features the native path understands; anything else on the protocol declines the + * table. `deletionVectors`/`columnMapping` are declined separately below for specific reasons. + */ + private val understoodReaderFeatures: Set[String] = + Set("columnMapping", "deletionVectors", "timestampNtz", "v2Checkpoint", "vacuumProtocolCheck") + + /** + * Is this exactly Delta's DSv1 parquet format? Compared by class name, not `classOf`: a + * `classOf` reference would raise `NoClassDefFoundError` and break every parquet scan when + * delta-spark is absent from the classpath. + */ + def isDeltaScan(scanExec: FileSourceScanExec): Boolean = + scanExec.relation.fileFormat.getClass.getName == + "org.apache.spark.sql.delta.DeltaParquetFileFormat" + + /** + * First reason this Delta scan cannot go native, or None when claimable. Only called when + * [[isDeltaScan]] is true. `scanHelper` is the [[CometScanExec]] built to drive + * [[CometDeltaNativeScan.convert]] on a claim, reused for the multi-store gate below. + */ + def declineReason( + plan: SparkPlan, + scanExec: FileSourceScanExec, + scanHelper: CometScanExec): Option[String] = { + val format = scanExec.relation.fileFormat.asInstanceOf[DeltaParquetFileFormat] + val protocol = format.protocol + val metadata = format.metadata + // Descriptor deserialization is expensive, so hoist it into a `lazy val`: it runs at most + // once per claim attempt, and not at all for the common non-DV-shape case. + val tableRoot = scanExec.relation.location.rootPaths.head.toString + lazy val dvDescriptors: Seq[DeletionVectorDescriptor] = + selectedDvDescriptors(scanHelper, tableRoot) + + // Mirrors core's CometScanRule.isSchemaSupported so scan-time type gates (unsigned-small-int + // fallback, collation, shredded-variant-struct) apply identically here. Pure in-memory check, + // so it runs first, ahead of every I/O-bearing gate below. + val schemaFallbackReasons = new ListBuffer[String]() + val typeChecker = CometScanTypeChecker() + val requiredSchemaSupported = + typeChecker.isSchemaSupported(scanExec.requiredSchema, schemaFallbackReasons) + val partitionSchemaSupported = + typeChecker.isSchemaSupported(scanExec.relation.partitionSchema, schemaFallbackReasons) + if (!requiredSchemaSupported || !partitionSchemaSupported) { + return Some( + "Native Delta scan does not support the schema: " + schemaFallbackReasons.mkString(", ")) + } + + if (format.isCDCRead) { + return Some("Native Delta scan does not support Change Data Feed reads") + } + + // Delta's DML machinery (findTouchedFiles) disables reader optimizations and needs real + // row indexes from Spark's reader; claiming here would feed NULL indexes into DV construction. + if (!format.optimizationsEnabled) { + return Some("Native Delta scan does not support reads with reader optimizations disabled") + } + if (scanExec.requiredSchema.exists(_.name == DeltaParquetFileFormat.ROW_INDEX_COLUMN_NAME) || + scanExec.relation.dataSchema.exists( + _.name == DeltaParquetFileFormat.ROW_INDEX_COLUMN_NAME)) { + return Some("Native Delta scan does not support Delta's generated row-index column") + } + + // Name mode is supported via physical-name schemas; id mode needs the field-id path and + // stays declined until validated. + val cmMode = metadata.columnMappingMode.name + if (cmMode != "none" && cmMode != "name") { + return Some(s"Native Delta scan does not support column mapping mode $cmMode") + } + // createPhysicalSchema wholesale-replaces field metadata, silently dropping EXISTS_DEFAULT. + if (cmMode == "name" && + getExistenceDefaultValues(scanExec.requiredSchema).exists(_ != null)) { + return Some( + "Native Delta scan does not support column defaults together with column mapping") + } + // createPhysicalSchema rewrites nested StructField names too, and the native builder emits the + // required schema verbatim as output, so name-sensitive expressions (e.g. to_json) would leak + // physical names. Decline until a rename adapter exists. + if (cmMode == "name" && + scanExec.requiredSchema.exists(f => containsNestedStruct(f.dataType))) { + return Some("Native Delta scan does not support column mapping with nested struct fields") + } + + val readerFeatures = protocol.readerFeatureNames + val unknownFeatures = readerFeatures -- understoodReaderFeatures + if (unknownFeatures.nonEmpty) { + return Some( + s"Native Delta scan does not support reader feature(s) ${unknownFeatures.mkString(", ")}") + } + + // Non-constant metadata columns are generated per-row by Spark's reader and unsupported, + // except Delta's DV bookkeeping columns, which the native path emits as constants. + val knownColNames = + scanExec.relation.dataSchema.map(_.name).toSet ++ + scanExec.relation.partitionSchema.map(_.name).toSet ++ + scanExec.fileConstantMetadataColumns.map(_.name).toSet ++ + CometDeltaNativeScan.internalColumnNames + val unknownOutput = scanExec.output.map(_.name).filterNot(knownColNames.contains) + if (unknownOutput.nonEmpty) { + return Some( + s"Native Delta scan does not support generated column(s) ${unknownOutput.mkString(", ")}") + } + + // Deletion-vector shape invariants (see CometDeltaNativeScan.buildDvScanCommon). + if (CometDeltaNativeScan.isDvShape(scanExec)) { + // A row-index column WITHOUT is_row_deleted is Delta DML bookkeeping (real row indexes), + // not a DV read; claiming it with a constant would corrupt the DVs being written. + val hasIsRowDeleted = + scanExec.requiredSchema.exists(_.name == CometDeltaNativeScan.IsRowDeletedColumn) + val hasRowIndex = + scanExec.requiredSchema.exists(_.name == CometDeltaNativeScan.RowIndexColumn) + if (hasRowIndex && !hasIsRowDeleted) { + return Some( + "Native Delta scan does not support row-index reads outside a deletion-vector scan") + } + // Internal columns must form a suffix of the read schema so data-column positions agree + // between Spark's output and the stripped native schema. + val names = scanExec.requiredSchema.fields.map(_.name) + val firstInternal = names.indexWhere(CometDeltaNativeScan.internalColumnNames.contains) + if (!names.drop(firstInternal).forall(CometDeltaNativeScan.internalColumnNames.contains)) { + return Some("Native Delta scan requires DV bookkeeping columns to trail the read schema") + } + // Native applies the DV itself and emits a dead constant for row-index, so the real value + // must be provably unused above the scan. + if (!rowIndexUnusedAbove(plan, scanExec)) { + return Some( + "Native Delta scan cannot supply _metadata.row_index values consumed by the query") + } + // The DV common builder does not serialize existence defaults yet. + if (getExistenceDefaultValues(scanExec.requiredSchema).exists(_ != null)) { + return Some( + "Native Delta scan does not support column defaults together with deletion vectors") + } + // Bounds native's memory for expanded DV row selectors (delta_dv.rs), pessimistically + // bounded by 2*cardinality + #row-groups; the conf below makes an over-pessimistic decline + // recoverable. + val maxDeletedRowsPerFile = DeltaScanConf.COMET_DELTA_MAX_DELETED_ROWS_PER_FILE.get() + val oversizedCardinalities = dvDescriptors + .map(_.cardinality) + .filter(_ > maxDeletedRowsPerFile) + if (oversizedCardinalities.nonEmpty) { + return Some( + "Native Delta scan does not support a deletion vector deleting " + + s"${oversizedCardinalities.max} rows in a single file, exceeding " + + s"${DeltaScanConf.COMET_DELTA_MAX_DELETED_ROWS_PER_FILE.key}=$maxDeletedRowsPerFile") + } + } + + // input_file_name & friends read from a thread-local Spark's FileScanRDD sets; the native scan + // does not populate it, and Delta's DML find-touched-files scans use it (mirrors core's check + // in CometScanRule.nativeScan). + if (plan.exists(node => + node.expressions.exists(_.exists { + case _: InputFileName | _: InputFileBlockStart | _: InputFileBlockLength => true + case _ => false + }))) { + return Some( + "Native Delta scan is not compatible with input_file_name, " + + "input_file_block_start, or input_file_block_length") + } + + // Row-index metadata columns are generated per-row by Spark's reader (mirrors core); the DV + // shape's trailing row-index column is exempt since the gates above already proved it dead. + if (!CometDeltaNativeScan.isDvShape(scanExec) && + ShimFileFormat.findRowIndexColumnIndexInSchema(scanExec.requiredSchema) >= 0) { + return Some("Native Delta scan does not support row index generation") + } + + // Mirror core's vectorized-reader compatibility gate. + if (!SQLConf.get.getConf(SQLConf.PARQUET_VECTORIZED_READER_ENABLED) && + !CometConf.COMET_SCAN_ALLOW_DISABLED_PARQUET_VECTORIZED_READER.get()) { + return Some( + "Native Delta scan is incompatible with " + + s"${SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key}=false") + } + + // Decline ALL encrypted-parquet configurations (stricter than core): the exec node does not + // yet wire the decryption-key broadcast to executors. + val hadoopConf = scanExec.relation.sparkSession.sessionState + .newHadoopConfWithOptions(scanExec.relation.options) + if (CometParquetUtils.encryptionEnabled(hadoopConf)) { + return Some("Native Delta scan does not support encrypted parquet") + } + + // Nested-type column defaults cannot be serialized; a dropped default would misalign the + // value/index lists consumed positionally on the native side. Mirrors core's + // transformV1Scan gate. + val possibleDefaultValues = getExistenceDefaultValues(scanExec.requiredSchema) + if (possibleDefaultValues.exists(d => + d != null && (d.isInstanceOf[ArrayBasedMapData] || d + .isInstanceOf[GenericInternalRow] || d.isInstanceOf[GenericArrayData]))) { + return Some("Native Delta scan does not support default values for nested types") + } + + // Only claim scans whose root paths object_store (or the configured libhdfs schemes) can + // actually read (mirrors core's unsupportedFsSchemes gate). + val libhdfs = libhdfsSchemes + val unsupportedRootSchemes = + unsupportedSchemes(scanExec.relation.location.rootPaths.map(_.toUri), libhdfs) + if (unsupportedRootSchemes.nonEmpty) { + return Some( + "Native Delta scan does not support filesystem scheme(s) " + + s"${unsupportedRootSchemes.mkString(", ")}") + } + + // A shallow clone can span multiple object-store authorities, but the native builder resolves + // ObjectStoreUrl from only the FIRST selected file; force file listing and decline rather than + // risk reading a later file through the wrong handle. + val dataFileUris = + scanHelper.selectedPartitions.iterator.flatMap(_.files).map(_.getPath.toUri).toSeq + + // Both gates below need the DV absolute-path URIs; dvDescriptors is already memoized. + val dvUris = dvDescriptors + .filter(_.storageType != DeletionVectorDescriptor.INLINE_DV_MARKER) + .map(_.absolutePath(new Path(tableRoot)).toUri) + + // The root-path gate above only inspects the table root(s); selected files can resolve + // through a different scheme (e.g. `viewfs:`). Checked before the authority gates below, + // which presume every URI is natively resolvable. + val unsupportedSelected = unsupportedSelectedSchemeReason(dataFileUris ++ dvUris, libhdfs) + if (unsupportedSelected.isDefined) { + return unsupportedSelected + } + + // Checked before multiStoreReason, which presumes every URI resolves to a single store + // identity -- a userinfo-bearing authority provably does not (store keying drops userinfo). + val userInfoReason = userInfoBearingAuthorityReason(dataFileUris ++ dvUris) + if (userInfoReason.isDefined) { + return userInfoReason + } + + val multiStore = multiStoreReason(dataFileUris) + if (multiStore.isDefined) { + return multiStore + } + + // GCS's zero-I/O, conf-only credential-forwarding gate; ordered alongside the S3 credential + // gates below since all presume a single, well-formed store identity per URI. + val gcsAuthReason = gcsHadoopOnlyAuthReason(hadoopConf, dataFileUris ++ dvUris) + if (gcsAuthReason.isDefined) { + return gcsAuthReason + } + + // Credentials that resolve ONLY through a Hadoop credential provider (JCEKS et al.) are + // invisible to the plain-conf extraction forwarded to the native S3 client; reuses hadoopConf + // from the encryption gate above. + val credentialReason = credentialAliasReason(hadoopConf, dataFileUris ++ dvUris) + if (credentialReason.isDefined) { + return credentialReason + } + + // A credential-provider class native's build_aws_credential_provider_metadata (s3.rs) does + // not recognize errors at scan EXECUTION time, after the scan was already claimed; decline + // eagerly instead. + val providerReason = providerClassGateReason(hadoopConf, dataFileUris ++ dvUris) + if (providerReason.isDefined) { + return providerReason + } + + // Reuse core's generic native-scan gates (ignoreCorruptFiles/ignoreMissingFiles, AQE DPP on + // Spark 3.4, exec enabled); tags its own fallback reasons. + if (!CometNativeScan.isSupported(scanExec)) { + return Some("Core native scan gates rejected the scan (see reasons above)") + } + + None + } + + /** + * Deletion-vector descriptors for every file this DV-shape scan selected, normalized to + * absolute on-disk paths. Returns `Seq.empty` for the plain shape. Shared by the DV cardinality + * gate and [[CometDeltaNativeScan.convert]]'s object-store option merge. + */ + private[delta] def selectedDvDescriptors( + scanHelper: CometScanExec, + tableRoot: String): Seq[DeletionVectorDescriptor] = { + if (!CometDeltaNativeScan.isDvShape(scanHelper.wrapped)) { + return Seq.empty + } + val tableRootPath = new Path(tableRoot) + scanHelper.selectedPartitions.iterator + .flatMap(_.files) + .flatMap { file => + file.metadata + .get(DeltaParquetFileFormat.FILE_ROW_INDEX_FILTER_ID_ENCODED) + .map(enc => DeletionVectorDescriptor.deserializeFromBase64(enc.asInstanceOf[String])) + } + .map(_.copyWithAbsolutePath(tableRootPath)) + .toSeq + } + + /** + * The libhdfs scheme exemption set from [[org.apache.comet.CometConf.COMET_LIBHDFS_SCHEMES]], + * lowercased and defaulting to `Set("hdfs")` when unset. + */ + private[delta] def libhdfsSchemes: Set[String] = COMET_LIBHDFS_SCHEMES.get() match { + case Some(s) => + s.split(",").map(_.trim.toLowerCase(Locale.ROOT)).filter(_.nonEmpty).toSet + case None => Set("hdfs") + } + + /** + * The lowercased, deduplicated schemes among `uris` that neither `libhdfs` nor Comet's native + * object_store layer ([[CometScanRule.isNativelyReadableScheme]]) can read. A `null` scheme is + * tolerated, not flagged, since such a URI cannot come from a Hadoop-backed source. + */ + private[delta] def unsupportedSchemes(uris: Seq[URI], libhdfs: Set[String]): Set[String] = { + uris + .filter { uri => + val sch = uri.getScheme + sch != null && { + val sl = sch.toLowerCase(Locale.ROOT) + !libhdfs.contains(sl) && !CometScanRule.isNativelyReadableScheme(uri) + } + } + .map(_.getScheme.toLowerCase(Locale.ROOT)) + .toSet + } + + /** + * Decline reason when any of `uris` -- the scan's selected data-file and deletion-vector URIs + * -- use a scheme [[unsupportedSchemes]] flags, or `None` when every URI is natively readable + * (or libhdfs-exempt). + */ + private[delta] def unsupportedSelectedSchemeReason( + uris: Seq[URI], + libhdfs: Set[String]): Option[String] = { + val schemes = unsupportedSchemes(uris, libhdfs) + if (schemes.isEmpty) { + None + } else { + Some( + "Native Delta scan does not support selected data file or deletion vector filesystem " + + s"scheme(s) ${schemes.mkString(", ")}") + } + } + + /** + * Decline reason when `uris` span more than one object-store authority (scheme + lowercased raw + * authority, so e.g. `S3A://Bucket` and `s3a://bucket` collapse), or `None` when they share + * one. `file://` paths carry no authority, so local scans across many directories are + * unaffected. + */ + private[delta] def multiStoreReason(uris: Seq[URI]): Option[String] = { + val authorities = uris.map(uriAuthority).distinct + if (authorities.size > 1) { + Some( + "Native Delta scan does not support data files spanning multiple object stores " + + s"(found: ${authorities.sorted.mkString(", ")})") + } else { + None + } + } + + /** + * Normalizes `uri` to a lowercased `scheme://authority` string, keyed on the raw `getAuthority` + * rather than the parsed host/port/userinfo fields: `getHost` (and `getUserInfo`/`getPort`) + * return `null` for the whole authority when it fails RFC 3986 `reg-name` syntax (e.g. an + * underscore in a GCS bucket name, `gs://my_bucket`), which would silently collapse distinct + * buckets into one empty-host key. A `null` authority normalizes to the empty string. + */ + private[delta] def uriAuthority(uri: URI): String = { + val scheme = Option(uri.getScheme).map(_.toLowerCase(Locale.ROOT)).getOrElse("") + val authority = Option(uri.getAuthority).map(_.toLowerCase(Locale.ROOT)).getOrElse("") + s"$scheme://$authority" + } + + /** + * The raw userinfo component of `uri`'s authority, or empty when none. Splits at the LAST `@` + * rather than using `URI#getUserInfo`, which (like [[uriAuthority]]'s getters) returns `null` + * for the whole authority on an RFC 3986 `reg-name` violation. Never lowercased: userinfo is + * case-sensitive. + */ + private[delta] def uriUserInfo(uri: URI): String = { + val authority = Option(uri.getAuthority).getOrElse("") + val at = authority.lastIndexOf('@') + if (at >= 0) authority.substring(0, at) else "" + } + + /** + * Redacts `uri`'s authority to `scheme`, then `://`, then a literal `***` masking userinfo, + * then `@host[:port]`, for embedding in a decline reason. NEVER interpolate `uri.getAuthority` + * or [[uriUserInfo]] directly into a reason string: doing so would leak credentials embedded as + * URI userinfo into the SQL plan's explain output, fallback-reason logging, or the Spark UI. + */ + private[delta] def redactedAuthority(uri: URI): String = { + val scheme = Option(uri.getScheme).map(_.toLowerCase(Locale.ROOT)).getOrElse("") + val authority = Option(uri.getAuthority).getOrElse("") + val at = authority.lastIndexOf('@') + val hostPort = if (at >= 0) authority.substring(at + 1) else authority + s"$scheme://***@$hostPort" + } + + /** + * Decline reason when any of `uris` carries userinfo in its authority (e.g. the container in an + * abfss:// path), or `None` when none do. The native store cache, `ObjectStoreUrl`, and + * DataFusion registry all key on scheme/host/port only, dropping userinfo, so two authorities + * differing only in userinfo collide onto the same store handle. + */ + private[delta] def userInfoBearingAuthorityReason(uris: Seq[URI]): Option[String] = { + val offending = uris.filter(uri => uriUserInfo(uri).nonEmpty).map(redactedAuthority).distinct + if (offending.isEmpty) { + None + } else { + Some("Native Delta scan does not support object-store paths whose authority carries " + + "userinfo (e.g. the container in an abfss:// path): the native object-store cache, " + + "ObjectStoreUrl and DataFusion registry all key on scheme, host and port only, so two " + + "containers on one storage account share a single store handle " + + s"(found: ${offending.sorted.mkString(", ")})") + } + } + + /** + * String-literal Hadoop conf keys consulted by [[credentialAliasReason]] below. `hadoop-aws` is + * NOT on this module's runtime classpath, so `org.apache.hadoop.fs.s3a.Constants` must never be + * referenced here (would raise `NoClassDefFoundError` for sessions with no S3 dependency). + */ + private val HadoopCredentialProviderPathKey = "hadoop.security.credential.provider.path" + private val S3aCredentialProviderPathKey = "fs.s3a.security.credential.provider.path" + + private def s3aBucketProviderPathKey(bucket: String): String = + s"fs.s3a.bucket.$bucket.security.credential.provider.path" + + /** + * The LONG form of [[s3aBucketProviderPathKey]]: `S3AUtils#lookupPassword` resolves per-bucket + * overrides through both a long key (`fs.s3a.bucket.B.<full base key>`) and a short key; both + * must be covered here too. + */ + private def s3aBucketLongProviderPathKey(bucket: String): String = + s"fs.s3a.bucket.$bucket.fs.s3a.security.credential.provider.path" + + /** + * The aliases native's S3 credentials provider chain tries per bucket (short, then global), + * PLUS the LONG form `S3AUtils#lookupPassword` also consults. Hadoop resolves long before short + * before global; native reads only short+global, so a JCEKS entry set ONLY under the long alias + * is invisible to native -- a shadowed value under ANY alias here means decline. + */ + private def s3aCredentialAliases(bucket: String): Seq[String] = + Seq( + s"fs.s3a.bucket.$bucket.fs.s3a.access.key", + s"fs.s3a.bucket.$bucket.fs.s3a.secret.key", + s"fs.s3a.bucket.$bucket.fs.s3a.session.token", + s"fs.s3a.bucket.$bucket.access.key", + s"fs.s3a.bucket.$bucket.secret.key", + s"fs.s3a.bucket.$bucket.session.token", + "fs.s3a.access.key", + "fs.s3a.secret.key", + "fs.s3a.session.token") + + private def nonEmptyConf(hadoopConf: Configuration, key: String): Boolean = + Option(hadoopConf.get(key)).exists(_.nonEmpty) + + /** + * The lowercase-scheme-checked S3/S3A bucket name from `uri`'s authority, or `None` when + * `uri`'s scheme is not `s3`/`s3a`. Parses the raw authority manually rather than + * `URI#getHost`, avoiding the same RFC 3986 `reg-name` pitfall as [[uriAuthority]]. + */ + private def s3Bucket(uri: URI): Option[String] = { + val scheme = Option(uri.getScheme).map(_.toLowerCase(Locale.ROOT)) + if (scheme.contains("s3") || scheme.contains("s3a")) { + val authority = Option(uri.getAuthority).getOrElse("") + val at = authority.lastIndexOf('@') + val hostAndPort = if (at >= 0) authority.substring(at + 1) else authority + val colon = hostAndPort.lastIndexOf(':') + val host = if (colon >= 0) hostAndPort.substring(0, colon) else hostAndPort + if (host.isEmpty) None else Some(host) + } else { + None + } + } + + /** + * The three per-bucket S3A credential base keys the native S3 client's `get_config` (`s3.rs`) + * resolves: short bucket key first, then global (native never reads the long bucket key). + */ + private val PlainCredentialBaseKeys: Seq[String] = + Seq("fs.s3a.access.key", "fs.s3a.secret.key", "fs.s3a.session.token") + + private def plainValue(hadoopConf: Configuration, key: String): Option[String] = + Option(hadoopConf.get(key)).filter(_.nonEmpty) + + /** + * The short-bucket-then-global value native's `get_config` (s3.rs) resolves for `baseKey` under + * `bucket`. Used by the credential-provider-class gates below, which read S3A options the same + * way native does. + */ + private def effectiveOptionValue( + hadoopConf: Configuration, + bucket: String, + baseKey: String): Option[String] = { + val shortKey = s"fs.s3a.bucket.$bucket." + baseKey.stripPrefix("fs.s3a.") + plainValue(hadoopConf, shortKey).orElse(plainValue(hadoopConf, baseKey)) + } + + private def plainLongFormCredentialDivergenceReason(bucket: String, longKey: String): String = + "Native Delta scan cannot forward long-form bucket credentials for " + + s"$bucket ($longKey is set but the native S3 client only reads the short bucket and " + + "global keys, so its credentials would differ from Hadoop's)" + + /** + * Zero-I/O plain-value divergence check: native's `get_config` never reads the long form, so it + * can silently diverge from Hadoop's effective value (`getPassword` lets a non-empty long value + * WIN over short/global). Declines when the long form is set and the two sides disagree; never + * interpolates a resolved value, only the key name. + * + * Credentials only: every OTHER `fs.s3a` option flows through `propagateBucketOptions`, which + * strips only ONE `fs.s3a.bucket.B.` prefix layer, folding a long-form key into + * `fs.s3a.fs.s3a.<key>` (inert, unread elsewhere), so Hadoop's effective value already reduces + * to short.orElse(global) like native. No gate is needed for that set. + */ + private def plainLongFormCredentialReason( + hadoopConf: Configuration, + bucket: String): Option[String] = { + PlainCredentialBaseKeys.foldLeft(Option.empty[String]) { (declined, baseKey) => + if (declined.isDefined) { + declined + } else { + val shortKey = s"fs.s3a.bucket.$bucket." + baseKey.stripPrefix("fs.s3a.") + val longKey = s"fs.s3a.bucket.$bucket.$baseKey" + val long = plainValue(hadoopConf, longKey) + if (long.isEmpty) { + None + } else { + val short = plainValue(hadoopConf, shortKey) + val global = plainValue(hadoopConf, baseKey) + val hadoopEffective = long.orElse(short).orElse(global) + val nativeEffective = short.orElse(global) + if (hadoopEffective != nativeEffective) { + Some(plainLongFormCredentialDivergenceReason(bucket, longKey)) + } else { + None + } + } + } + } + } + + /** + * String-literal mirror of every credential-provider class name s3.rs's + * `build_aws_credential_provider_metadata` recognizes (Hadoop S3A plus AWS SDK v1/v2 names). + * `hadoop-aws` is NOT on this module's runtime classpath, so these stay string literals, never + * `classOf` references. + */ + private val SupportedCredentialProviderClasses: Set[String] = Set( + "org.apache.hadoop.fs.s3a.auth.IAMInstanceCredentialsProvider", + "org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider", + "org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider", + "org.apache.hadoop.fs.s3a.auth.AssumedRoleCredentialProvider", + "org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider", + "software.amazon.awssdk.auth.credentials.ContainerCredentialsProvider", + "com.amazonaws.auth.ContainerCredentialsProvider", + "com.amazonaws.auth.EC2ContainerCredentialsProviderWrapper", + "software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider", + "com.amazonaws.auth.InstanceProfileCredentialsProvider", + "software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider", + "com.amazonaws.auth.EnvironmentVariableCredentialsProvider", + "software.amazon.awssdk.auth.credentials.WebIdentityTokenFileCredentialsProvider", + "com.amazonaws.auth.WebIdentityTokenCredentialsProvider", + "software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider", + "com.amazonaws.auth.profile.ProfileCredentialsProvider", + "software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider", + "com.amazonaws.auth.AnonymousAWSCredentials") + + private val AnonymousCredentialProviderClasses: Set[String] = Set( + "org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider", + "software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider", + "com.amazonaws.auth.AnonymousAWSCredentials") + + private val HadoopAssumedRoleProviderClass = + "org.apache.hadoop.fs.s3a.auth.AssumedRoleCredentialProvider" + + private val AwsCredentialsProviderKey = "fs.s3a.aws.credentials.provider" + private val AssumedRoleCredentialsProviderKey = "fs.s3a.assumed.role.credentials.provider" + + /** Splits a comma-separated credential-provider-class list the same way s3.rs's parser does. */ + private def parseProviderClassNames(value: String): Seq[String] = + value.split(",").map(_.trim).filter(_.nonEmpty).toSeq + + private def unsupportedProviderReason(bucket: String, key: String, className: String): String = + s"Native Delta scan does not support the credential provider class $className " + + s"configured via $key for $bucket (the native S3 client only supports a fixed set of " + + "provider classes; an unsupported class would fail at scan execution time, after the " + + "scan was already claimed, rather than at planning time)" + + private def mixedAnonymousProviderReason(bucket: String, key: String): String = + s"Native Delta scan does not support $key for $bucket naming an anonymous credential " + + "provider together with any other provider (the native S3 client rejects this " + + "combination at scan execution time)" + + private def anonymousAssumedRoleProviderReason(bucket: String, key: String): String = + s"Native Delta scan does not support an anonymous credential provider in $key for " + + s"$bucket (the native S3 client does not allow an anonymous provider as the base " + + "credentials for an assumed-role chain)" + + private def unsupportedProviderNameReason( + bucket: String, + key: String, + names: Seq[String]): Option[String] = + names + .find(name => !SupportedCredentialProviderClasses.contains(name)) + .map(unsupportedProviderReason(bucket, key, _)) + + /** + * Decline reason when `bucket`'s effective `assumed.role.credentials.provider` names an + * unsupported class, or an anonymous one (native rejects ANY anonymous entry here, not just a + * mix). Unset defaults to native's own always-supported fallback, so `None` is safe. + */ + private def assumedRoleProviderClassReason( + hadoopConf: Configuration, + bucket: String): Option[String] = { + effectiveOptionValue(hadoopConf, bucket, AssumedRoleCredentialsProviderKey) match { + case None => None + case Some(value) => + val names = parseProviderClassNames(value) + unsupportedProviderNameReason(bucket, AssumedRoleCredentialsProviderKey, names).orElse { + if (names.exists(AnonymousCredentialProviderClasses.contains)) { + Some(anonymousAssumedRoleProviderReason(bucket, AssumedRoleCredentialsProviderKey)) + } else { + None + } + } + } + } + + /** + * Decline reason when `bucket`'s effective `aws.credentials.provider` names an unrecognized + * class, mixes an anonymous provider with any other, or a nested + * `AssumedRoleCredentialProvider` sub-chain has the same problem. Unset/empty falls back to + * native's default chain. + */ + private def providerClassReason(hadoopConf: Configuration, bucket: String): Option[String] = { + effectiveOptionValue(hadoopConf, bucket, AwsCredentialsProviderKey).flatMap { value => + val names = parseProviderClassNames(value) + unsupportedProviderNameReason(bucket, AwsCredentialsProviderKey, names) + .orElse { + if (names.length > 1 && names.exists(AnonymousCredentialProviderClasses.contains)) { + Some(mixedAnonymousProviderReason(bucket, AwsCredentialsProviderKey)) + } else { + None + } + } + .orElse { + if (names.contains(HadoopAssumedRoleProviderClass)) { + assumedRoleProviderClassReason(hadoopConf, bucket) + } else { + None + } + } + } + } + + /** First reason any bucket among `uris` names an unsupported credential-provider class. */ + private[delta] def providerClassGateReason( + hadoopConf: Configuration, + uris: Seq[URI]): Option[String] = { + val buckets = uris.flatMap(s3Bucket).distinct + buckets.foldLeft(Option.empty[String]) { (declined, bucket) => + if (declined.isDefined) declined else providerClassReason(hadoopConf, bucket) + } + } + + private def s3aScopedProviderPathReason(bucket: String, providerPathKey: String): String = + "Native Delta scan cannot forward Hadoop credential-provider aliases for " + + s"$bucket ($providerPathKey configures an S3A-scoped Hadoop credential provider that " + + "Configuration#getPassword does not consult, so the native S3 client's credentials " + + "cannot be verified)" + + private def shadowedCredentialAliasReason(bucket: String, alias: String): String = + "Native Delta scan cannot forward Hadoop credential-provider aliases for " + + s"$bucket ($alias resolves through $HadoopCredentialProviderPathKey but is not present " + + "as a plain configuration value, so the native S3 client would have no credentials)" + + private def unverifiableCredentialProviderReason(bucket: String, error: Throwable): String = + "Native Delta scan cannot verify Hadoop credential-provider aliases for " + + s"$bucket (reading $HadoopCredentialProviderPathKey raised " + + s"${error.getClass.getName}), declining rather than risk missing credentials" + + /** + * Compares each [[s3aCredentialAliases]] alias's plain `Configuration#get` against + * `Configuration#getPassword` (providers first, plain conf fallback); a difference means + * decline. Runs in try/catch: `getPassword` does real keystore I/O, and a corrupt/unreadable + * store must decline this bucket, not abort planning for the whole session. + */ + private def verifyGlobalProviderAliases( + hadoopConf: Configuration, + bucket: String): Option[String] = { + try { + s3aCredentialAliases(bucket).foldLeft(Option.empty[String]) { (declined, alias) => + if (declined.isDefined) { + declined + } else { + val resolved = + Option(hadoopConf.getPassword(alias)).map(new String(_)).filter(_.nonEmpty) + resolved match { + case Some(value) if !Option(hadoopConf.get(alias)).contains(value) => + Some(shadowedCredentialAliasReason(bucket, alias)) + case _ => None + } + } + } + } catch { + case e @ (_: IOException | _: RuntimeException) => + Some(unverifiableCredentialProviderReason(bucket, e)) + } + } + + /** + * The decline reason for `bucket` alone, or `None` when its credentials are safe to forward. + * [[plainLongFormCredentialReason]] is checked first (zero-I/O, decisive on its own); then + * `None` if no provider-path key is set. Arm A: an S3A-scoped provider-path key points at a + * provider `getPassword` does not consult, so any being set declines with no keystore read. Arm + * B: only the global path is set (which `getPassword` DOES consult); delegates to + * [[verifyGlobalProviderAliases]]. + */ + private def bucketCredentialAliasReason( + hadoopConf: Configuration, + bucket: String, + globalPathSet: Boolean, + s3aPathSet: Boolean): Option[String] = { + plainLongFormCredentialReason(hadoopConf, bucket).orElse { + val bucketPathKey = s3aBucketProviderPathKey(bucket) + val bucketLongPathKey = s3aBucketLongProviderPathKey(bucket) + val bucketPathSet = nonEmptyConf(hadoopConf, bucketPathKey) + val bucketLongPathSet = nonEmptyConf(hadoopConf, bucketLongPathKey) + if (!globalPathSet && !s3aPathSet && !bucketPathSet && !bucketLongPathSet) { + None + } else if (s3aPathSet || bucketPathSet || bucketLongPathSet) { + val offendingKey = + if (s3aPathSet) S3aCredentialProviderPathKey + else if (bucketPathSet) bucketPathKey + else bucketLongPathKey + Some(s3aScopedProviderPathReason(bucket, offendingKey)) + } else { + verifyGlobalProviderAliases(hadoopConf, bucket) + } + } + } + + /** + * String-literal Hadoop conf key prefix for GCS authentication options (`fs.gs.auth.*`). + * `gcs-connector` is NOT on this module's runtime classpath by default, so referencing an + * actual GCS auth class would risk `NoClassDefFoundError`, same rationale as the S3A literals + * above. + */ + private val GcsAuthKeyPrefix = "fs.gs.auth." + + /** + * True when `uri`'s scheme is `gs` (case-insensitive) -- the ONLY scheme object_store's + * `ObjectStoreScheme::parse` (parquet_support.rs) routes to `GoogleCloudStorage`; `gcs` is not + * recognized there and is deliberately excluded. + */ + private def isGcsScheme(uri: URI): Boolean = + Option(uri.getScheme).exists(_.equalsIgnoreCase("gs")) + + /** + * The lowercase-scheme-checked GCS bucket name from `uri`'s authority (host, minus any userinfo + * or port), or `None` when `uri`'s scheme is not `gs`. Parses the raw authority manually, + * mirroring [[s3Bucket]]'s `URI#getHost`/RFC 3986 `reg-name` reasoning. + */ + private def gcsBucket(uri: URI): Option[String] = { + if (!isGcsScheme(uri)) { + None + } else { + val authority = Option(uri.getAuthority).getOrElse("") + val at = authority.lastIndexOf('@') + val hostAndPort = if (at >= 0) authority.substring(at + 1) else authority + val colon = hostAndPort.lastIndexOf(':') + val host = if (colon >= 0) hostAndPort.substring(0, colon) else hostAndPort + if (host.isEmpty) None else Some(host) + } + } + + /** + * The non-empty `fs.gs.auth.*` Hadoop conf keys set on `hadoopConf`, full key names only -- + * NEVER their values, which are credential material and must never enter a decline reason. + * Iterates the conf map directly: no provider resolution, no I/O. + */ + private def gcsAuthKeys(hadoopConf: Configuration): Seq[String] = + hadoopConf + .iterator() + .asScala + .collect { + case entry + if entry.getKey.startsWith(GcsAuthKeyPrefix) && + entry.getValue != null && entry.getValue.nonEmpty => + entry.getKey + } + .toSeq + .distinct + .sorted + + /** + * Decline reason when any of `uris` resolves to a `gs://` authority AND `hadoopConf` sets any + * `fs.gs.auth.*` key, or `None` when claimable. Native forwards none of `fs.gs.*` to the object + * store, so a scan relying solely on Hadoop-side GCS credentials would claim here but then fail + * authentication natively. Application Default Credentials work identically in both engines and + * need no Hadoop conf key, so an ADC-only configuration still claims. Never interpolates a + * resolved value, only key names. + */ + private[delta] def gcsHadoopOnlyAuthReason( + hadoopConf: Configuration, + uris: Seq[URI]): Option[String] = { + val gcsUris = uris.filter(isGcsScheme) + if (gcsUris.isEmpty) { + return None + } + val authKeys = gcsAuthKeys(hadoopConf) + if (authKeys.isEmpty) { + return None + } + val buckets = gcsUris.flatMap(gcsBucket).distinct.sorted + Some( + "Native Delta scan does not support GCS authentication configured only via Hadoop conf " + + s"key(s) ${authKeys.mkString(", ")} for gs://${buckets.mkString(", gs://")} " + + "(the native GCS client does not forward fs.gs.* options; only Application Default " + + "Credentials -- environment or metadata-server -- are available natively)") + } + + /** + * First reason a native S3 scan cannot faithfully forward this table's Hadoop credentials, or + * `None` when claimable. Only `s3`/`s3a` authorities matter here (ABFS/WASB mooted by the + * userinfo gate, GCS handled by [[gcsHadoopOnlyAuthReason]]). `Configuration#getPassword` + * checks every provider first, falling back to plain conf only when unset, so a keystore-only + * or shadowing alias is invisible/wrong to native's plain-conf extraction; when in doubt, + * decline. Never interpolates a resolved value, only key names. + */ + private[delta] def credentialAliasReason( + hadoopConf: Configuration, + uris: Seq[URI]): Option[String] = { + val buckets = uris.flatMap(s3Bucket).distinct + if (buckets.isEmpty) { + return None + } + val globalPathSet = nonEmptyConf(hadoopConf, HadoopCredentialProviderPathKey) + val s3aPathSet = nonEmptyConf(hadoopConf, S3aCredentialProviderPathKey) + buckets.foldLeft(Option.empty[String]) { (declined, bucket) => + if (declined.isDefined) { + declined + } else { + bucketCredentialAliasReason(hadoopConf, bucket, globalPathSet, s3aPathSet) + } + } + } + + /** + * True when `dataType` is, or structurally contains (through array elements or map keys/ + * values), a [[StructType]]. Only [[StructType]] fields carry Delta's physical, column-mapped + * names; array/map labels themselves are never column-mapped. + */ + private def containsNestedStruct(dataType: DataType): Boolean = dataType match { + case _: StructType => true + case ArrayType(elementType, _) => containsNestedStruct(elementType) + case MapType(keyType, valueType, _) => + containsNestedStruct(keyType) || containsNestedStruct(valueType) + case _ => false + } + + /** + * True when `node` is a positional-output union -- `UnionExec` or `CometUnionExec`. Both + * compute output positionally from the FIRST child's attributes, so a value carried only by a + * LATER branch needs an explicit positional walk below. Compared by class name (the + * [[isDeltaScan]] idiom) to avoid a compile-time dependency; an unmatched name is still safe, + * caught by the generic multi-child safety net below. + */ + private def isPositionalUnion(node: SparkPlan): Boolean = { + val name = node.getClass.getSimpleName + name == "UnionExec" || name == "CometUnionExec" + } + + /** + * True when the scan's row-index column value is provably dead above the scan. The standard DV + * plan shape routes it only into a `named_struct(... row_index ...) AS _metadata` projection + * whose result the final projection discards; anything else (a query actually selecting + * `_metadata.row_index`) makes the value live and must decline. Conservative: any unrecognized + * consumption pattern returns false. + */ + private def rowIndexUnusedAbove(plan: SparkPlan, scanExec: FileSourceScanExec): Boolean = { + val rowIndexAttrs = scanExec.output + .filter(_.name == CometDeltaNativeScan.RowIndexColumn) + .map(_.exprId) + .toSet + if (rowIndexAttrs.isEmpty) { + return true + } + // Transitive taint analysis: everything derived from the row-index attribute within the + // visible plan, via Project aliases or positionally across a union. The plan may be an AQE + // stage fragment, so tainted values escaping to the fragment's own output must decline too. + var tainted = rowIndexAttrs + var changed = true + while (changed) { + changed = false + plan.foreach { + case p: ProjectExec => + p.projectList.foreach { + case a: Alias + if !tainted.contains(a.exprId) && + a.references.exists(r => tainted.contains(r.exprId)) => + tainted += a.exprId + changed = true + case _ => + } + case u if isPositionalUnion(u) => + // Output attributes carry the FIRST child's expression IDs, so a value tainted only in + // a LATER branch is otherwise invisible; walk it forward positionally instead. + // `children` can be re-parented by AQE after `output` is frozen, so an arity mismatch on + // ANY child (which would make a positional zip silently truncate) forces a decline. + if (u.children.exists(_.output.length != u.output.length)) { + return false + } + u.children.foreach { child => + child.output.zip(u.output).foreach { + case (from, to) if tainted.contains(from.exprId) && !tainted.contains(to.exprId) => + tainted += to.exprId + changed = true + case _ => + } + } + case _ => + } + } + val nonProjectConsumer = plan.exists { + case _: ProjectExec => false + case n if n ne scanExec => + n.expressions.exists(_.references.exists(r => tainted.contains(r.exprId))) + case _ => false + } + val escapes = plan.output.exists(a => tainted.contains(a.exprId)) + // Generic safety net for every OTHER multi-child node (joins, etc; positional unions are + // exempt, already handled precisely above). A tainted attribute a child contributes must + // either survive into the node's own output under the SAME expression ID or be consumed by + // one of the node's own expressions; otherwise decline (e.g. a LEFT SEMI/ANTI join dropping + // the side carrying the tainted attribute). + val multiChildLeak = plan.exists { + case u if isPositionalUnion(u) => false + case n if n.children.size >= 2 => Review Comment: **[P1] Treat write sinks as consumers of the reader's row index** Could we make the unused-row-index proof account for unary write sinks? This is a reader eligibility issue: Spark's existing writer simply persists the values returned by the reader. For example, with the Delta contrib/native scan enabled and the native Parquet writer disabled, this workload reads the source row position into an ordinary output column: ```scala val root = java.nio.file.Files.createTempDirectory("comet-row-index") val src = root.resolve("src").toString val dst = root.resolve("dst").toString spark.range(32).coalesce(1).write.format("delta") .option("delta.enableDeletionVectors", "true").save(src) spark.sql(s"DELETE FROM delta.`$src` WHERE id IN (1, 7, 13)").collect() spark.read.format("delta").load(src) .selectExpr("id", "_metadata.row_index AS ri") .write.parquet(dst) spark.read.parquet(dst).where("id = 31").show() // expected: (31, 31) spark.read.parquet(dst).selectExpr("sum(ri)").show() // expected: 475 ``` The real physical write plan is `DataWritingCommandExec -> WriteFilesExec -> Project -> ... -> Delta scan`. Both write nodes have one child, empty output, and no expression reference to `ri`; they consume their child's rows positionally. The two Projects propagate the row-index dependency, but neither `nonProjectConsumer`, the root-output `escapes` check, nor this multi-child-only guard sees the write consuming it. The exact current `rowIndexUnusedAbove` returns `true` for the Parquet write, while the equivalent SELECT correctly returns `false`. That permits the DV reader to supply the [synthetic Long zero](https://github.com/apache/datafusion-comet/blob/b033153da230f0239c6998532d063020751052de/contrib/delta-spark/src/main/scala/org/apache/comet/contrib/delta/CometDeltaNativeScan.scala#L474-L491). On the same actual Delta fixture, the current native reader retained all 29 correct surviving IDs but returned `ri=0` for every row: 28 incorrect row-index values, sum 0, and `(31,0)` rather than `(31,31)`. The unchanged writer would persist those incorrect input values. Validation: the exact liveness method was exercised on real cached Spark 4.0.2-based/Delta 4.0.0 write plans with AQE off/on, including an independent default-codegen check. Separately, the production native DV attachment and Parquet reader were exercised on the actual file and DV, with the source-verified scan layout. The remaining admission path was source-traced; a full current-profile Spark/Comet JNI write was **not** executed. Please treat write sinks as consumers of tainted input attributes, or conservatively decline this shape, and add a regression checking both the eligibility decision and saved `ri` values. The liveness logic already exists at `1d3557cf`; this is not introduced by the latest `b033153d` update. ########## contrib/delta-spark/src/main/scala/org/apache/comet/contrib/delta/CometDeltaNativeScan.scala: ########## @@ -0,0 +1,548 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.comet.contrib.delta + +import scala.jdk.CollectionConverters._ + +import org.apache.hadoop.fs.Path +import org.apache.spark.internal.Logging +import org.apache.spark.sql.catalyst.expressions.Literal +import org.apache.spark.sql.comet.{CometScanExec, DeltaPlanDataInjector} +import org.apache.spark.sql.delta.DeltaParquetFileFormat +import org.apache.spark.sql.delta.RowIndexFilterType +import org.apache.spark.sql.delta.actions.DeletionVectorDescriptor +import org.apache.spark.sql.execution.{FileSourceScanExec, ScalarSubquery => ExecScalarSubquery} +import org.apache.spark.sql.execution.datasources.{FilePartition, PartitionedFile} +import org.apache.spark.sql.types.{ByteType, LongType, MetadataBuilder, StructField, StructType} + +import org.apache.comet.objectstore.NativeConfig +import org.apache.comet.serde.OperatorOuterClass +import org.apache.comet.serde.OperatorOuterClass.Operator +import org.apache.comet.serde.QueryPlanSerde.{exprToProto, serializeDataType} +import org.apache.comet.serde.operator.{literalToProto, partition2Proto, schema2Proto, CometNativeScan} +import org.apache.comet.shims.ShimFileFormat + +/** + * Serde for the native Delta scan. Two shapes: + * - Plain reads reuse core's `NativeScanCommon` builder wholesale. + * - Deletion-vector reads: Delta's planner appends `__delta_internal_is_row_deleted` (tinyint) + * and Spark's row-index temp column (bigint) to the read schema and filters on is_row_deleted + * above the scan. The native reader applies the DV as a row selection, so both internal + * columns are emitted as per-file constants (0), the parquet read schema is stripped to the + * real data columns, and the DV descriptor ships per file for native to fetch and decode. + */ +object CometDeltaNativeScan + extends Logging + with org.apache.spark.sql.catalyst.expressions.PredicateHelper { + + val IsRowDeletedColumn: String = DeltaParquetFileFormat.IS_ROW_DELETED_COLUMN_NAME + val RowIndexColumn: String = ShimFileFormat.ROW_INDEX_TEMPORARY_COLUMN_NAME + + private[delta] val internalColumnNames: Set[String] = Set(IsRowDeletedColumn, RowIndexColumn) + + // Prefix for the internal columns' slots in the partition schema, mirroring core's + // _comet_metadata_ prefix rationale: DataFusion matches partition columns by name. + // [[allocateUniqueInternalFields]] additionally suffixes on collision with a real column. + private val deltaConstFieldPrefix = "_comet_delta_" + + def isDvShape(scanExec: FileSourceScanExec): Boolean = + scanExec.requiredSchema.exists(f => internalColumnNames.contains(f.name)) + + private def deltaFormat(scanExec: FileSourceScanExec): DeltaParquetFileFormat = + scanExec.relation.fileFormat.asInstanceOf[DeltaParquetFileFormat] + + private def columnMappingMode(scanExec: FileSourceScanExec): String = + deltaFormat(scanExec).metadata.columnMappingMode.name + + /** + * Under column mapping, parquet files store physical column names (stable UUIDs / ids), so the + * schemas passed to the native parquet reader must be physical. Positions and structure are + * preserved, so output binding and projection are unaffected. The scan's internal DV columns + * are not part of the table schema and must be stripped before calling this. + */ + private def toPhysical(scanExec: FileSourceScanExec, schema: StructType): StructType = { + val format = deltaFormat(scanExec) + if (format.metadata.columnMappingMode.name == "none") { + schema + } else { + // Name mode matches file columns by physical NAME. Strip the parquet.field.id metadata + // createPhysicalSchema also stamps: files written before the column-mapping upgrade have + // no field ids and would fail the reader's id expectations. + stripFieldIds(org.apache.spark.sql.delta.DeltaColumnMapping + .createPhysicalSchema(schema, format.metadata.schema, format.metadata.columnMappingMode)) + } + } + + private def stripFieldIds(schema: StructType): StructType = { + import org.apache.spark.sql.types._ + def stripType(dt: DataType): DataType = dt match { + case s: StructType => stripFieldIds(s) + case a: ArrayType => a.copy(elementType = stripType(a.elementType)) + case m: MapType => + m.copy(keyType = stripType(m.keyType), valueType = stripType(m.valueType)) + case other => other + } + StructType(schema.fields.map { f => + val metadata = new MetadataBuilder() + .withMetadata(f.metadata) + .remove("parquet.field.id") + // Sibling key Delta stamps on array/map fields under IcebergCompat/Uniform. + .remove("parquet.field.nested.ids") + .build() + f.copy(dataType = stripType(f.dataType), metadata = metadata) + }) + } + + /** + * Build the planning-time `DeltaScan` operator (common data only; file partitions are injected + * lazily at execution). Returns None when an output data type cannot be serialized or the plan + * shape is not one we can translate faithfully. + */ + def convert(scanExec: FileSourceScanExec, scanHelper: CometScanExec): Option[Operator] = { + val relation = scanExec.relation + + val firstFileUri = scanHelper.selectedPartitions + .flatMap(_.files.headOption) + .headOption + .map(_.getPath.toUri) + + val hadoopConf = relation.sparkSession.sessionState + .newHadoopConfWithOptions(relation.options) + + val tableRootPath = relation.location.rootPaths.head + val tableRoot = tableRootPath.toString + + val commonOpt = if (!isDvShape(scanExec)) { + // Under column mapping (name mode) the parquet reader must see physical names; + // positions are preserved so output binding and projection stay untouched. + CometNativeScan.buildNativeScanCommon( + source = scanExec.simpleStringWithNodeId(), + output = scanExec.output, + requiredSchema = toPhysical(scanExec, scanExec.requiredSchema), + dataSchema = toPhysical(scanExec, relation.dataSchema), Review Comment: **[P1] Guard Unicode case-insensitive Delta reads before using the shared reader** With column mapping disabled and `spark.sql.caseSensitive=false`, this call sends the logical names unchanged to the shared native Parquet reader. Its [ASCII-only matching](https://github.com/apache/datafusion-comet/blob/b033153da230f0239c6998532d063020751052de/native/core/src/parquet/schema_adapter.rs#L208-L215) does not preserve Spark's handling of non-ASCII case variants in physical column names. A normal converted Delta table can expose this: ```scala import org.apache.spark.sql.functions.{col, lit} spark.conf.set("spark.sql.caseSensitive", false) val root = java.nio.file.Files.createTempDirectory("comet-unicode") val path = root.resolve("data").toString val table = "comet_unicode_" + java.util.UUID.randomUUID().toString.replace("-", "") spark.range(1, 2).select(col("id"), lit(71).as("É")) .coalesce(1).write.parquet(path) spark.range(2, 3).select(col("id"), lit(72).as("é")) .coalesce(1).write.mode("append").parquet(path) spark.sql(s"CREATE TABLE $table (id BIGINT, `É` INT) USING PARQUET LOCATION '$path'") spark.sql(s"CONVERT TO DELTA $table NO STATISTICS") spark.read.format("delta").load(path).selectExpr("id", "`É`").show() ``` Stock Spark/Delta returns `[(1,71),(2,72)]`. The current native reader on these exact files returns `[(1,71),(2,NULL)]`. Adding `É IS NOT NULL` or `É > 70` loses the second row. A covering Spark filter cannot recover the stored value after the scan has replaced it with NULL. This is an ordinary `CONVERT` transaction with protocol `(1,2)`, mapping `none`, no defaults or DVs, and two local files. The declared required/data schemas are both `[id, É]`, projection is `[0,1]`, and field-ID matching is off. The case difference appears only in the second file's footer. Current admission has no Unicode-name exclusion, and the new Delta handler installs the same `SparkPhysicalExprAdapterFactory` exercised by the probe. Validation: the standard conversion command and reference reads ran on cached Spark 4.0.2-based/Delta 4.0.0. Current native-reader execution was compared against those results: nine mismatches across `É/é`, `Σ/σ`, and `Б/б` for plain reads and the two predicates; all 15 ASCII/case-sensitive controls matched. The complete current claim path was source-traced, not executed through a full current-profile Spark/JNI pipeline. Could we fix the shared name matching or conservatively fall back for affected Delta schemas, with an asserted native/fallback regression for this conversion history? The matcher predates the PR, but this PR newly routes these previously Spark-read Delta tables through it. The same exposure exists at `1d3557cf`, so it is not introduced by the latest update. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
