andygrove commented on code in PR #1757:
URL: https://github.com/apache/datafusion-comet/pull/1757#discussion_r2098328838
##########
spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala:
##########
@@ -41,36 +41,44 @@ import org.apache.comet.parquet.{CometParquetScan,
SupportsComet}
* Spark physical optimizer rule for replacing Spark scans with Comet scans.
*/
case class CometScanRule(session: SparkSession) extends Rule[SparkPlan] {
+ private val scanImpl: String = COMET_NATIVE_SCAN_IMPL.get()
+
override def apply(plan: SparkPlan): SparkPlan = {
- if (!isCometLoaded(conf) || !isCometScanEnabled(conf)) {
- if (!isCometLoaded(conf)) {
- withInfo(plan, "Comet is not enabled")
- } else if (!isCometScanEnabled(conf)) {
- withInfo(plan, "Comet Scan is not enabled")
- }
- plan
- } else {
-
- def hasMetadataCol(plan: SparkPlan): Boolean = {
- plan.expressions.exists(_.exists {
- case a: Attribute =>
- a.isMetadataCol
- case _ => false
- })
- }
-
- plan.transform {
- case scan if hasMetadataCol(scan) =>
- withInfo(scan, "Metadata column is not supported")
-
- // data source V1
- case scanExec: FileSourceScanExec =>
- transformV1Scan(scanExec)
-
- // data source V2
- case scanExec: BatchScanExec =>
- transformV2Scan(scanExec)
- }
+ if (!isCometLoaded(conf)) {
+ withInfo(plan, "Comet is not enabled")
+ return plan
+ }
+
+ if (!isCometScanEnabled(conf)) {
+ withInfo(plan, "Comet Scan is not enabled")
+ return plan
+ }
+
+ if (SQLConf.get.getConf(
+ SQLConf.PARQUET_FIELD_ID_READ_ENABLED) && scanImpl !=
CometConf.SCAN_NATIVE_COMET) {
+ withInfo(plan, s"Comet $scanImpl scan does not support
PARQUET_FIELD_ID_READ_ENABLED")
+ return plan
+ }
Review Comment:
This is the change. The rest is refactoring.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscr...@datafusion.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: github-unsubscr...@datafusion.apache.org
For additional commands, e-mail: github-h...@datafusion.apache.org
