MaxGekk commented on code in PR #49528:
URL: https://github.com/apache/spark/pull/49528#discussion_r1919680604
##########
sql/core/src/main/scala/org/apache/spark/sql/jdbc/JdbcDialects.scala:
##########
@@ -336,6 +336,17 @@ abstract class JdbcDialect extends Serializable with
Logging {
s"TRUNCATE TABLE $table"
}
+ @Since("3.5.0")
+ def supportsUpsert(): Boolean = false
+
+ @Since("3.5.0")
+ def getUpsertStatement(
+ tableName: String,
+ columns: Array[StructField],
+ isCaseSensitive: Boolean,
+ options: JDBCOptions): String =
+ throw new UnsupportedOperationException("upserts are not supported")
Review Comment:
Could you use `SparkUnsupportedOperationException`.
##########
connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/UpsertTests.scala:
##########
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.jdbc
+
+import java.sql.Timestamp
+import java.util.Properties
+
+import org.apache.spark.sql.{Row, SaveMode}
+import org.apache.spark.sql.functions.{col, lit, rand, when}
+
+trait UpsertTests {
+ self: DockerJDBCIntegrationSuite =>
+
+ import testImplicits._
+
+ def createTableOption: String
+ def upsertTestOptions: Map[String, String] = Map("createTableOptions" ->
createTableOption)
+
+ test(s"Upsert existing table") { doTestUpsert(tableExists = true) }
+ test(s"Upsert non-existing table") { doTestUpsert(tableExists = false) }
+
+ Seq(
+ Seq("ts", "id", "v1", "v2"),
+ Seq("ts", "v1", "id", "v2"),
+ Seq("ts", "v1", "v2", "id"),
+ Seq("v2", "v1", "ts", "id")
+ ).foreach { columns =>
+ test(s"Upsert with varying column order - ${columns.mkString(",")}") {
+ doTestUpsert(tableExists = true, Some(columns))
+ }
+ }
+
+ def doTestUpsert(tableExists: Boolean, project: Option[Seq[String]] = None):
Unit = {
+ val df = Seq(
+ (1, Timestamp.valueOf("1996-01-01 01:23:46"), 1.235, 1.234568), // row
unchanged
+ (2, Timestamp.valueOf("1996-01-01 01:23:45"), 2.346, 2.345678), //
updates v1
+ (2, Timestamp.valueOf("1996-01-01 01:23:46"), 2.347, 2.345680), //
updates v1 and v2
+ (3, Timestamp.valueOf("1996-01-01 01:23:45"), 3.456, 3.456789) //
inserts new row
+ ).toDF("id", "ts", "v1", "v2").repartition(1) // .repartition(10)
+
+ val table = if (tableExists) "upsert" else "new_upsert_table"
+ val options = upsertTestOptions ++ Map(
+ "numPartitions" -> "10",
+ "upsert" -> "true",
+ "upsertKeyColumns" -> "id, ts"
+ )
+ project.map(columns => df.select(columns.map(col): _*)).getOrElse(df)
+ .write.mode(SaveMode.Append).options(options).jdbc(jdbcUrl, table, new
Properties)
+
+ val actual = spark.read.jdbc(jdbcUrl, table, new
Properties).collect().toSet
+ val existing = if (tableExists) {
+ Set((1, Timestamp.valueOf("1996-01-01 01:23:45"), 1.234, 1.234567))
+ } else {
+ Set.empty
+ }
+ val upsertedRows = Set(
+ (1, Timestamp.valueOf("1996-01-01 01:23:46"), 1.235, 1.234568),
+ (2, Timestamp.valueOf("1996-01-01 01:23:45"), 2.346, 2.345678),
+ (2, Timestamp.valueOf("1996-01-01 01:23:46"), 2.347, 2.345680),
+ (3, Timestamp.valueOf("1996-01-01 01:23:45"), 3.456, 3.456789)
+ )
+ val expected = (existing ++ upsertedRows).map { case (id, ts, v1, v2) =>
+ Row(Integer.valueOf(id), ts, v1.doubleValue(), v2.doubleValue())
+ }
+ assert(actual === expected)
+ }
+
+ test(s"Upsert concurrency") {
Review Comment:
```suggestion
test("Upsert concurrency") {
```
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JdbcUtils.scala:
##########
@@ -132,9 +129,32 @@ object JdbcUtils extends Logging with SQLConfHelper {
}
}
}
+ }
+
+ /**
+ * Returns an Insert SQL statement for inserting a row into the target table
via JDBC conn.
+ */
+ def getInsertStatement(
+ table: String,
+ rddSchema: StructType,
+ tableSchema: Option[StructType],
+ isCaseSensitive: Boolean,
+ dialect: JdbcDialect): String = {
+ val columns = getInsertColumns(table, rddSchema, tableSchema,
isCaseSensitive, dialect)
dialect.insertIntoTable(table, columns)
}
+ def getUpsertStatement(
+ table: String,
+ rddSchema: StructType,
+ tableSchema: Option[StructType],
+ isCaseSensitive: Boolean,
+ dialect: JdbcDialect,
+ options: JDBCOptions): String = {
Review Comment:
fix indentation, see
https://github.com/databricks/scala-style-guide?tab=readme-ov-file#spacing-and-indentation
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
