aakash-db commented on code in PR #50942:
URL: https://github.com/apache/spark/pull/50942#discussion_r2098763580
##########
sql/connect/common/src/main/protobuf/spark/connect/pipelines.proto:
##########
@@ -0,0 +1,142 @@
+syntax = "proto3";
+
+package spark.connect;
+
+import "spark/connect/relations.proto";
+import "spark/connect/types.proto";
+
+option java_multiple_files = true;
+option java_package = "org.apache.spark.connect.proto";
+
+// Dispatch object for pipelines commands.
+message PipelineCommand {
+ oneof command_type {
+ CreateDataflowGraph create_dataflow_graph = 1;
+ DefineDataset define_dataset = 2;
+ DefineFlow define_flow = 3;
+ DropDataflowGraph drop_dataflow_graph = 4;
+ StartRun start_run = 5;
+ StopRun stop_run = 6;
+ DefineSqlGraphElements define_sql_graph_elements = 7;
+ }
+
+ message DefineSqlGraphElements {
+ optional string dataflow_graph_id = 1;
+ optional string sql_file_name = 2;
+ optional string sql_text = 3;
+ }
+
+ // Request to create a new pipeline.
+ message CreateDataflowGraph {
+ // The default catalog.
+ optional string default_catalog = 1;
+
+ // The default database.
+ optional string default_database = 2;
+
+ // Default SQL configurations for all flows in this graph.
+ map<string, string> sql_conf = 5;
+
+ message Response {
+ // The ID of the created graph.
+ string dataflow_graph_id = 1;
+ }
+ }
+
+ // Drops the graph and stops any running attached flows.
+ message DropDataflowGraph {
+ // The graph to drop.
+ string dataflow_graph_id = 1;
+ }
+
+ // Request to define a dataset: a table, a materialized view, or a temporary
view.
+ message DefineDataset {
+ // The graph to attach this dataset to.
+ string dataflow_graph_id = 1;
+
+ // Name of the dataset.
+ string dataset_name = 2;
+
+ // Table or view.
+ DatasetType dataset_type = 3;
+
+ // Optional comment for the dataset.
+ optional string comment = 4;
+
+ // Optional table properties.
+ map<string, string> table_properties = 5;
+
+ // Optional partition columns for the dataset (if applicable).
Review Comment:
yup. added the disclaimer in a few places.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
