This is an automated email from the ASF dual-hosted git repository.
somandal pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 6674ee36e5d Add support to configure the maximum minion tasks that can
be run per instance (#16981)
6674ee36e5d is described below
commit 6674ee36e5d57c962bf485c9e7bc978c392f7e21
Author: Sonam Mandal <[email protected]>
AuthorDate: Thu Oct 9 10:08:30 2025 -0700
Add support to configure the maximum minion tasks that can be run per
instance (#16981)
---
.../java/org/apache/pinot/common/utils/helix/HelixHelper.java | 9 +++++++++
.../src/main/java/org/apache/pinot/minion/BaseMinionStarter.java | 2 ++
.../src/main/java/org/apache/pinot/minion/MinionConf.java | 5 +++++
.../main/java/org/apache/pinot/spi/utils/CommonConstants.java | 4 ++++
4 files changed, 20 insertions(+)
diff --git
a/pinot-common/src/main/java/org/apache/pinot/common/utils/helix/HelixHelper.java
b/pinot-common/src/main/java/org/apache/pinot/common/utils/helix/HelixHelper.java
index e8a32bbf8e7..a052e6bee75 100644
---
a/pinot-common/src/main/java/org/apache/pinot/common/utils/helix/HelixHelper.java
+++
b/pinot-common/src/main/java/org/apache/pinot/common/utils/helix/HelixHelper.java
@@ -663,4 +663,13 @@ public class HelixHelper {
}
return false;
}
+
+ public static boolean updateMaxConcurrentTasksPerInstance(InstanceConfig
instanceConfig, int maxConcurrentTasks) {
+ int currentMaxConcurrentTasks = instanceConfig.getMaxConcurrentTask();
+ if (currentMaxConcurrentTasks != maxConcurrentTasks) {
+ instanceConfig.setMaxConcurrentTask(maxConcurrentTasks);
+ return true;
+ }
+ return false;
+ }
}
diff --git
a/pinot-minion/src/main/java/org/apache/pinot/minion/BaseMinionStarter.java
b/pinot-minion/src/main/java/org/apache/pinot/minion/BaseMinionStarter.java
index a783e2dda5e..f12b9ca6a6b 100644
--- a/pinot-minion/src/main/java/org/apache/pinot/minion/BaseMinionStarter.java
+++ b/pinot-minion/src/main/java/org/apache/pinot/minion/BaseMinionStarter.java
@@ -367,6 +367,8 @@ public abstract class BaseMinionStarter implements
ServiceStartable {
() ->
Collections.singletonList(CommonConstants.Helix.UNTAGGED_MINION_INSTANCE));
updated |= HelixHelper.removeDisabledPartitions(instanceConfig);
updated |= HelixHelper.updatePinotVersion(instanceConfig);
+ updated |= HelixHelper.updateMaxConcurrentTasksPerInstance(instanceConfig,
+ _config.getMaxConcurrentTasksPerInstance());
if (updated) {
HelixHelper.updateInstanceConfig(_helixManager, instanceConfig);
}
diff --git a/pinot-minion/src/main/java/org/apache/pinot/minion/MinionConf.java
b/pinot-minion/src/main/java/org/apache/pinot/minion/MinionConf.java
index 019f20e0945..2ec514acd9f 100644
--- a/pinot-minion/src/main/java/org/apache/pinot/minion/MinionConf.java
+++ b/pinot-minion/src/main/java/org/apache/pinot/minion/MinionConf.java
@@ -92,4 +92,9 @@ public class MinionConf extends PinotConfiguration {
.orElseGet(() ->
getProperty(CommonConstants.Minion.DEPRECATED_CONFIG_OF_METRICS_PREFIX_KEY,
CommonConstants.Minion.CONFIG_OF_METRICS_PREFIX));
}
+
+ public int getMaxConcurrentTasksPerInstance() {
+ return
getProperty(CommonConstants.Minion.CONFIG_OF_MAX_CONCURRENT_TASKS_PER_INSTANCE,
+ CommonConstants.Minion.DEFAULT_MAX_CONCURRENT_TASKS_PER_INSTANCE);
+ }
}
diff --git
a/pinot-spi/src/main/java/org/apache/pinot/spi/utils/CommonConstants.java
b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/CommonConstants.java
index 2b229abe89d..820bf3a1052 100644
--- a/pinot-spi/src/main/java/org/apache/pinot/spi/utils/CommonConstants.java
+++ b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/CommonConstants.java
@@ -1494,6 +1494,8 @@ public class CommonConstants {
// Config keys
public static final String CONFIG_OF_SWAGGER_USE_HTTPS =
"pinot.minion.swagger.use.https";
public static final String CONFIG_OF_METRICS_PREFIX_KEY =
"pinot.minion.metrics.prefix";
+ public static final String CONFIG_OF_MAX_CONCURRENT_TASKS_PER_INSTANCE =
+ "pinot.minion.max.concurrent.tasks.per.instance";
@Deprecated
public static final String DEPRECATED_CONFIG_OF_METRICS_PREFIX_KEY =
"metricsPrefix";
public static final String METRICS_REGISTRY_REGISTRATION_LISTENERS_KEY =
"metricsRegistryRegistrationListeners";
@@ -1504,6 +1506,8 @@ public class CommonConstants {
public static final String DEFAULT_INSTANCE_BASE_DIR =
System.getProperty("java.io.tmpdir") + File.separator + "PinotMinion";
public static final String DEFAULT_INSTANCE_DATA_DIR =
DEFAULT_INSTANCE_BASE_DIR + File.separator + "data";
+ // Use Helix side default if configured to be -1
+ public static final int DEFAULT_MAX_CONCURRENT_TASKS_PER_INSTANCE = -1;
// Add pinot.minion prefix on those configs to be consistent with configs
of controller and server.
public static final String PREFIX_OF_CONFIG_OF_PINOT_FS_FACTORY =
"pinot.minion.storage.factory";
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
