akshaychitneni commented on code in PR #2568:
URL:
https://github.com/apache/datafusion-ballista/pull/2568#discussion_r4263084836
##########
ballista/scheduler/src/cluster/mod.rs:
##########
@@ -363,6 +389,82 @@ pub trait JobState: Send + Sync {
fn produce_config(&self) -> SessionConfig;
}
+/// Scheduler-side metadata store for materialized `DataFrame::cache()`
results —
+/// the mapping from a cache key to the shuffle partition locations of its
output.
+///
+/// This is the third piece of scheduler state alongside [`ClusterState`] and
+/// [`JobState`], and follows the same shape: an async trait with an in-memory
+/// implementation today ([`InMemoryCacheState`])
+/// and an [`init`](CacheState::init) hook so a future durable backend (e.g.
one
+/// backed by an object store) can rehydrate entries on startup. The methods
are
+/// `async` even though the in-memory backend does no I/O, so a durable backend
+/// slots in without changing any signatures.
+///
+/// A cache *miss* is simply [`lookup`](CacheState::lookup) returning `None`
(the
+/// key is absent, or materialization is still in flight); a *hit* is `Some`.
+#[async_trait::async_trait]
+pub trait CacheState: Send + Sync + 'static {
+ /// Initializes the backend. A no-op for the in-memory backend; durable
+ /// backends load previously persisted entries here.
+ async fn init(&self) -> Result<()> {
+ Ok(())
+ }
+
+ /// Returns the materialized data for `key`, or `None` on a miss (absent or
+ /// still materializing).
+ async fn lookup(&self, key: &CacheKey) ->
Result<Option<MaterializedCache>>;
+
+ /// Attempts to claim `key` for materialization by `job_id`, deduplicating
+ /// concurrent first-time misses to a single [`BeginOutcome::Claimed`].
+ async fn begin_materialization(
+ &self,
+ key: CacheKey,
+ job_id: String,
Review Comment:
Addressed
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
