ericm-db commented on code in PR #50595:
URL: https://github.com/apache/spark/pull/50595#discussion_r2065435410
##########
sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/StateStore.scala:
##########
@@ -1030,14 +1052,41 @@ object StateStore extends Logging {
}
}
- /** Unload a state store provider */
- def unload(storeProviderId: StateStoreProviderId): Unit =
loadedProviders.synchronized {
- loadedProviders.remove(storeProviderId).foreach(_.close())
+ /**
+ * Unload a state store provider.
+ * If alreadyRemovedFromLoadedProviders is None, provider will be
+ * removed from loadedProviders and closed.
+ * If alreadyRemovedFromLoadedProviders is Some, provider will be closed
+ * using passed in provider.
+ * WARNING: CAN ONLY BE CALLED FROM MAINTENANCE THREAD!
+ */
+ def removeFromLoadedProvidersAndClose(
+ storeProviderId: StateStoreProviderId,
+ alreadyRemovedProvider: Option[StateStoreProvider] = None
+ ): Unit = {
+ // Get the provider to close - either the one passed in or one we remove
from loadedProviders
+ val providerToClose = alreadyRemovedProvider.orElse {
+ loadedProviders.synchronized {
+ loadedProviders.remove(storeProviderId)
+ }
+ }
+
+ // Close the provider if we found one
+ providerToClose.foreach(_.close())
+ }
+
+ private def closeProvider(
+ storeProviderId: StateStoreProviderId,
+ removedProvider: Option[StateStoreProvider]): Unit = {
+ loadedProviders.synchronized {
+ assert(!loadedProviders.contains(storeProviderId))
Review Comment:
I added this just because I thought the existing function names made the
code a little difficult to trace through but I'll just replace this function
call with `provider.close()`
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
