This is an automated email from the ASF dual-hosted git repository.
xiangfu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 2b69d6afe2 Fixing the multiple files concurrent write issue when
reloading SSLFactory (#12384)
2b69d6afe2 is described below
commit 2b69d6afe2e15ad5b7e683610d044a15f4650885
Author: Haitao Zhang <[email protected]>
AuthorDate: Thu Feb 8 12:29:49 2024 -0800
Fixing the multiple files concurrent write issue when reloading SSLFactory
(#12384)
* add retry to SSLFactory reloading
* update comment
---
.../org/apache/pinot/common/utils/TlsUtils.java | 35 ++++++++++++++++++----
1 file changed, 29 insertions(+), 6 deletions(-)
diff --git
a/pinot-common/src/main/java/org/apache/pinot/common/utils/TlsUtils.java
b/pinot-common/src/main/java/org/apache/pinot/common/utils/TlsUtils.java
index e905f7aafa..c75e171384 100644
--- a/pinot-common/src/main/java/org/apache/pinot/common/utils/TlsUtils.java
+++ b/pinot-common/src/main/java/org/apache/pinot/common/utils/TlsUtils.java
@@ -57,6 +57,7 @@ import org.apache.commons.lang.StringUtils;
import org.apache.http.ssl.SSLContexts;
import org.apache.pinot.common.config.TlsConfig;
import org.apache.pinot.spi.env.PinotConfiguration;
+import org.apache.pinot.spi.utils.retry.RetryPolicies;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -439,6 +440,8 @@ public final class TlsUtils {
Map<WatchKey, Set<Path>> watchKeyPathMap = new HashMap<>();
registerFile(watchService, watchKeyPathMap, keyStorePath);
registerFile(watchService, watchKeyPathMap, trustStorePath);
+ int maxSslFactoryReloadingAttempts = 3;
+ int sslFactoryReloadingRetryDelayMs = 1000;
WatchKey key;
while ((key = watchService.take()) != null) {
for (WatchEvent<?> event : key.pollEvents()) {
@@ -447,12 +450,32 @@ public final class TlsUtils {
LOGGER.info("Detected change in file: {}, try to renew SSLFactory {}
"
+ "(built from key store {} and truststore {})",
changedFile, baseSslFactory, keyStorePath, trustStorePath);
- SSLFactory updatedSslFactory = createSSLFactory(
- keyStoreType, keyStorePath, keyStorePassword, trustStoreType,
trustStorePath, trustStorePassword,
- sslContextProtocol, secureRandom, false);
- SSLFactoryUtils.reload(baseSslFactory, updatedSslFactory);
- LOGGER.info("Successfully renewed SSLFactory {} (built from key
store {} and truststore {}) "
- + "on file {} changes", baseSslFactory, keyStorePath,
trustStorePath, changedFile);
+ try {
+ // Need to retry a few times because when one file (key store or
trust store) is updated, the other file
+ // (trust store or key store) may not have been fully written yet,
so we need to wait a bit and retry.
+
RetryPolicies.fixedDelayRetryPolicy(maxSslFactoryReloadingAttempts,
sslFactoryReloadingRetryDelayMs)
+ .attempt(() -> {
+ try {
+ SSLFactory updatedSslFactory =
+ createSSLFactory(keyStoreType, keyStorePath,
keyStorePassword, trustStoreType, trustStorePath,
+ trustStorePassword, sslContextProtocol,
secureRandom, false);
+ SSLFactoryUtils.reload(baseSslFactory, updatedSslFactory);
+ LOGGER.info("Successfully renewed SSLFactory {} (built
from key store {} and truststore {}) on file"
+ + " {} changes", baseSslFactory, keyStorePath,
trustStorePath, changedFile);
+ return true;
+ } catch (Exception e) {
+ LOGGER.info(
+ "Encountered issues when renewing SSLFactory {} (built
from key store {} and truststore {}) on "
+ + "file {} changes", baseSslFactory, keyStorePath,
trustStorePath, changedFile, e);
+ return false;
+ }
+ });
+ } catch (Exception e) {
+ LOGGER.error(
+ "Failed to renew SSLFactory {} (built from key store {} and
truststore {}) on file {} changes after {} "
+ + "retries", baseSslFactory, keyStorePath, trustStorePath,
changedFile,
+ maxSslFactoryReloadingAttempts, e);
+ }
}
}
key.reset();
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
