gengliangwang commented on code in PR #50875:
URL: https://github.com/apache/spark/pull/50875#discussion_r2093745473
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala:
##########
@@ -535,6 +535,110 @@ case class CreateTableAsSelect(
}
}
+/**
+ * The base command representation for a statement that can be part of a
Declarative Pipeline to
+ * define a pipeline dataset (MV or ST).
+ */
+
+trait CreatePipelineDataset extends Command {
+ // The name of the dataset.
+ val name: LogicalPlan
+
+ // The user specified columns for the dataset.
+ val columns: Seq[ColumnDefinition]
+
+ // The user specified column-based partitioning for the dataset.
+ val partitioning: Seq[Transform]
+
+ // Additional table specs for the dataset.
+ val tableSpec: TableSpecBase
+
+ // Whether the dataset should only be created if it doesn't already exist.
+ val ifNotExists: Boolean
+}
+
+/**
+ * An extension of the base command representation that represents a CTAS
style CREATE statement.
+ */
+trait CreatePipelineDatasetAsSelect extends BinaryCommand
+ with CreatePipelineDataset
+ with CTEInChildren {
+
+ // The logical plan of the CTAS subquery for the pipeline dataset.
+ val query: LogicalPlan
+
+ // The text representation of the CTAS subquery for the dataset.
+ val originalText: String
+
+ override def left: LogicalPlan = name
+ override def right: LogicalPlan = query
+
+ override def withCTEDefs(cteDefs: Seq[CTERelationDef]): LogicalPlan = {
+ withNewChildren(Seq(name, WithCTE(query, cteDefs)))
+ }
+}
+
+/**
+ * Command parsed from `CREATE MATERIALIZED VIEW ... AS ...` SQL syntax. This
command serves as a
+ * logical representation of the matching SQL syntax and cannot be executed.
Instead, it is
+ * interpreted by the pipelines submodule during a pipeline execution.
+ */
+case class CreateMaterializedViewAsSelect(
+ name: LogicalPlan,
+ columns: Seq[ColumnDefinition],
+ partitioning: Seq[Transform],
+ tableSpec: TableSpecBase,
+ query: LogicalPlan,
+ originalText: String,
+ ifNotExists: Boolean)
+ extends CreatePipelineDatasetAsSelect {
+ override protected def withNewChildrenInternal(
+ newLeft: LogicalPlan, newRight: LogicalPlan): LogicalPlan =
+ copy(name = newLeft, query = newRight)
+}
+
+/**
+ * Command parsed from `CREATE STREAMING TABLE ... AS ...` SQL syntax. This
command serves as a
+ * logical representation of the matching SQL syntax and cannot be executed.
Instead, it is
+ * interpreted by the pipelines submodule during a pipeline execution.
Review Comment:
```suggestion
* interpreted by the pipeline submodule during a pipeline execution.
```
##########
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala:
##########
@@ -535,6 +535,110 @@ case class CreateTableAsSelect(
}
}
+/**
+ * The base command representation for a statement that can be part of a
Declarative Pipeline to
+ * define a pipeline dataset (MV or ST).
+ */
+
+trait CreatePipelineDataset extends Command {
+ // The name of the dataset.
+ val name: LogicalPlan
+
+ // The user specified columns for the dataset.
+ val columns: Seq[ColumnDefinition]
+
+ // The user specified column-based partitioning for the dataset.
+ val partitioning: Seq[Transform]
+
+ // Additional table specs for the dataset.
+ val tableSpec: TableSpecBase
+
+ // Whether the dataset should only be created if it doesn't already exist.
+ val ifNotExists: Boolean
+}
+
+/**
+ * An extension of the base command representation that represents a CTAS
style CREATE statement.
+ */
+trait CreatePipelineDatasetAsSelect extends BinaryCommand
+ with CreatePipelineDataset
+ with CTEInChildren {
+
+ // The logical plan of the CTAS subquery for the pipeline dataset.
+ val query: LogicalPlan
+
+ // The text representation of the CTAS subquery for the dataset.
+ val originalText: String
+
+ override def left: LogicalPlan = name
+ override def right: LogicalPlan = query
+
+ override def withCTEDefs(cteDefs: Seq[CTERelationDef]): LogicalPlan = {
+ withNewChildren(Seq(name, WithCTE(query, cteDefs)))
+ }
+}
+
+/**
+ * Command parsed from `CREATE MATERIALIZED VIEW ... AS ...` SQL syntax. This
command serves as a
+ * logical representation of the matching SQL syntax and cannot be executed.
Instead, it is
+ * interpreted by the pipelines submodule during a pipeline execution.
+ */
+case class CreateMaterializedViewAsSelect(
+ name: LogicalPlan,
+ columns: Seq[ColumnDefinition],
+ partitioning: Seq[Transform],
+ tableSpec: TableSpecBase,
+ query: LogicalPlan,
+ originalText: String,
+ ifNotExists: Boolean)
+ extends CreatePipelineDatasetAsSelect {
+ override protected def withNewChildrenInternal(
+ newLeft: LogicalPlan, newRight: LogicalPlan): LogicalPlan =
+ copy(name = newLeft, query = newRight)
+}
+
+/**
+ * Command parsed from `CREATE STREAMING TABLE ... AS ...` SQL syntax. This
command serves as a
+ * logical representation of the matching SQL syntax and cannot be executed.
Instead, it is
+ * interpreted by the pipelines submodule during a pipeline execution.
+ */
+case class CreateStreamingTableAsSelect(
+ name: LogicalPlan,
+ columns: Seq[ColumnDefinition],
+ partitioning: Seq[Transform],
+ tableSpec: TableSpecBase,
+ query: LogicalPlan,
+ originalText: String,
+ ifNotExists: Boolean)
+ extends CreatePipelineDatasetAsSelect {
+ override protected def withNewChildrenInternal(
+ newLeft: LogicalPlan, newRight: LogicalPlan): LogicalPlan =
+ copy(name = newLeft, query = newRight)
+}
+
+/**
+ * Command parsed from `CREATE STREAMING TABLE ...` SQL syntax. This command
serves as a logical
+ * representation of the matching SQL syntac and cannot be executed. It is
instead interpreted by
+ * the pipelines submodule during a pipeline execution.
Review Comment:
```suggestion
* the pipeline submodule during a pipeline execution.
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
