HeartSaVioR commented on code in PR #50119:
URL: https://github.com/apache/spark/pull/50119#discussion_r1978569858
##########
sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/RocksDBSuite.scala:
##########
@@ -1196,6 +1196,87 @@ class RocksDBSuite extends AlsoTestWithRocksDBFeatures
with SharedSparkSession
}
}
+ testWithColumnFamilies(
+ "RocksDB: test includesPrefix parameter during changelog replay",
+ TestWithChangelogCheckpointingEnabled) { colFamiliesEnabled =>
+
+ // Only test when column families are enabled, as the includesPrefix
parameter
+ // is only relevant in that case
+ if (colFamiliesEnabled) {
+ val remoteDir = Utils.createTempDir().toString
+ val conf = dbConf.copy(minDeltasForSnapshot = 3, compactOnCommit = false)
+ new File(remoteDir).delete() // to make sure that the directory gets
created
+
+ withDB(remoteDir, conf = conf, useColumnFamilies = true) { db =>
+ // Create a test column family
+ val testCfName = "test_cf"
+ db.createColFamilyIfAbsent(testCfName, isInternal = false)
+
+ // Write initial data
+ db.load(0)
+ db.put("key1", "value1", StateStore.DEFAULT_COL_FAMILY_NAME)
+ db.put("key2", "value2", testCfName)
+ db.commit()
+
+ // Get the encoded keys with column family prefixes
+ val keyWithPrefix1 = getKeyWithPrefix(db, "key1",
StateStore.DEFAULT_COL_FAMILY_NAME)
+ val keyWithPrefix2 = getKeyWithPrefix(db, "key2", testCfName)
+
+ // Pretend we're replaying changelog with already-prefixed keys
+ db.load(1)
Review Comment:
I feel like this may not trigger reloading state from the checkpoint at all?
Same with below.
The test is trying to load the version which was just committed - this case
we do not rebuild the state. We'd probably load the different version first
(like 0) to enforce triggering the load (for both 0 and the target version).
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
