lakshmi-manasa-g commented on a change in pull request #938: SAMZA-1531:
Support run.id in standalone for batch processing.
URL: https://github.com/apache/samza/pull/938#discussion_r265687938
##########
File path:
samza-core/src/main/java/org/apache/samza/runtime/LocalApplicationRunner.java
##########
@@ -77,16 +90,70 @@
*/
public LocalApplicationRunner(SamzaApplication app, Config config) {
this.appDesc = ApplicationDescriptorUtil.getAppDescriptor(app, config);
- this.planner = new LocalJobPlanner(appDesc);
+ Boolean isAppModeBatch = new
ApplicationConfig(appDesc.getConfig()).getAppMode() ==
ApplicationConfig.ApplicationMode.BATCH;
+ if(isAppModeBatch) {
+ initializeCoordinationUtils(config);
+ initializeRunId();
+ }
+ this.planner = new LocalJobPlanner(appDesc, coordinationUtils, uid, runId);
}
/**
* Constructor only used in unit test to allow injection of {@link
LocalJobPlanner}
*/
@VisibleForTesting
- LocalApplicationRunner(ApplicationDescriptorImpl<? extends
ApplicationDescriptor> appDesc, LocalJobPlanner planner) {
+ LocalApplicationRunner(ApplicationDescriptorImpl<? extends
ApplicationDescriptor> appDesc, LocalJobPlanner planner, CoordinationUtils
coordinationUtils) {
this.appDesc = appDesc;
this.planner = planner;
+ this.coordinationUtils = coordinationUtils;
+ Boolean isAppModeBatch = new
ApplicationConfig(appDesc.getConfig()).getAppMode() ==
ApplicationConfig.ApplicationMode.BATCH;
+ if(isAppModeBatch) {
+ initializeRunId();
+ }
+ }
+
+ private void initializeCoordinationUtils(Config config) {
+ JobCoordinatorConfig jcConfig = new JobCoordinatorConfig(config);
+ String coordinationId = new ApplicationConfig(config).getGlobalAppId() +
CoordinationConstants.APPLICATION_RUNNER_PATH_SUFFIX;
+ this.coordinationUtils =
jcConfig.getCoordinationUtilsFactory().getCoordinationUtils(coordinationId,
uid, config);
+ }
+
+ private void initializeRunId(){
+ LOG.info("Manasa: LocalApplicationRunner: getRunId() : entered ");
+ Boolean isAppModeBatch = new
ApplicationConfig(appDesc.getConfig()).getAppMode() ==
ApplicationConfig.ApplicationMode.BATCH;
+ if(coordinationUtils == null || !isAppModeBatch) {
+ return;
+ }
+
+ runIdLock =
coordinationUtils.getReadWriteLock(CoordinationConstants.RUNID_LOCK_ID);
+ runIdAccess = coordinationUtils.getDataAccess();
+
+ if(runIdAccess == null || runIdLock == null) {
+ LOG.warn("Processor {} failed to create utils for run.id generation",
uid);
+ return;
+ }
+
+ try {
+ // acquire lock to write or read run.id
+ DistributedReadWriteLock.AccessType lockAccess =
runIdLock.lock(CoordinationConstants.LOCK_TIMEOUT_MS, TimeUnit.MILLISECONDS);
+ if(lockAccess == DistributedReadWriteLock.AccessType.WRITE) {
+ LOG.info("write lock acquired for run.id generation by Processor " +
uid);
+ runId = String.valueOf(System.currentTimeMillis()) + "-" +
UUID.randomUUID().toString().substring(0, 8);
Review comment:
It would not lead to a collision cause the runid is taken to be SystemTime
in millisecs and then the UUID substring.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
