[
https://issues.apache.org/jira/browse/KAFKA-7370?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16618239#comment-16618239
]
ASF GitHub Bot commented on KAFKA-7370:
---------------------------------------
rayokota closed pull request #5596: KAFKA-7370: Enhance FileConfigProvider to
read a dir
URL: https://github.com/apache/kafka/pull/5596
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git
a/clients/src/main/java/org/apache/kafka/common/config/provider/FileConfigProvider.java
b/clients/src/main/java/org/apache/kafka/common/config/provider/FileConfigProvider.java
index 4e376ecdeed..7b059af9faf 100644
---
a/clients/src/main/java/org/apache/kafka/common/config/provider/FileConfigProvider.java
+++
b/clients/src/main/java/org/apache/kafka/common/config/provider/FileConfigProvider.java
@@ -22,15 +22,22 @@
import java.io.IOException;
import java.io.Reader;
import java.nio.file.Files;
+import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
+import java.util.stream.Collectors;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
/**
- * An implementation of {@link ConfigProvider} that represents a Properties
file.
+ * An implementation of {@link ConfigProvider} that can read from either a
file or a directory.
+ * If the given path is a file, it is interpreted as a Properties file
containing key-value pairs.
+ * If the given path is a directory, the keys are the file names contained in
the directory and the values are
+ * the corresponding contents of the files.
* All property keys and values are stored as cleartext.
*/
public class FileConfigProvider implements ConfigProvider {
@@ -39,16 +46,55 @@ public void configure(Map<String, ?> configs) {
}
/**
- * Retrieves the data at the given Properties file.
+ * Retrieves the data at the given path.
*
- * @param path the file where the data resides
+ * @param path the path corresponding to either a directory or a
Properties file
* @return the configuration data
*/
public ConfigData get(String path) {
- Map<String, String> data = new HashMap<>();
if (path == null || path.isEmpty()) {
+ return new ConfigData(new HashMap<>());
+ }
+ Path p = Paths.get(path);
+ return Files.isDirectory(p) ? getFromDirectory(p) :
getFromPropertiesFile(p);
+ }
+
+ /**
+ * Retrieves the data with the given keys at the given path.
+ *
+ * @param path the path corresponding to either a directory or a
Properties file
+ * @param keys the keys whose values will be retrieved. In the case of a
directory, these are the file names.
+ * @return the configuration data
+ */
+ public ConfigData get(String path, Set<String> keys) {
+ if (path == null || path.isEmpty()) {
+ return new ConfigData(new HashMap<>());
+ }
+ Path p = Paths.get(path);
+ return Files.isDirectory(p) ? getFromDirectory(p, keys) :
getFromPropertiesFile(p, keys);
+ }
+
+ private ConfigData getFromDirectory(Path path) {
+ try {
+ Map<String, String> data = Files.list(path)
+ .filter(Files::isRegularFile)
+ .collect(Collectors.toMap(file ->
file.getFileName().toString(), this::readAll));
return new ConfigData(data);
+ } catch (IOException e) {
+ throw new ConfigException("Could not read from directory " + path);
}
+ }
+
+ private ConfigData getFromDirectory(Path path, Set<String> keys) {
+ Map<String, String> data = keys.stream()
+ .map(path::resolve)
+ .filter(Files::isRegularFile)
+ .collect(Collectors.toMap(file ->
file.getFileName().toString(), this::readAll));
+ return new ConfigData(data);
+ }
+
+ private ConfigData getFromPropertiesFile(Path path) {
+ Map<String, String> data = new HashMap<>();
try (Reader reader = reader(path)) {
Properties properties = new Properties();
properties.load(reader);
@@ -66,18 +112,8 @@ public ConfigData get(String path) {
}
}
- /**
- * Retrieves the data with the given keys at the given Properties file.
- *
- * @param path the file where the data resides
- * @param keys the keys whose values will be retrieved
- * @return the configuration data
- */
- public ConfigData get(String path, Set<String> keys) {
+ private ConfigData getFromPropertiesFile(Path path, Set<String> keys) {
Map<String, String> data = new HashMap<>();
- if (path == null || path.isEmpty()) {
- return new ConfigData(data);
- }
try (Reader reader = reader(path)) {
Properties properties = new Properties();
properties.load(reader);
@@ -93,9 +129,16 @@ public ConfigData get(String path, Set<String> keys) {
}
}
- // visible for testing
- protected Reader reader(String path) throws IOException {
- return Files.newBufferedReader(Paths.get(path));
+ private String readAll(Path path) {
+ try {
+ return new String(Files.readAllBytes(path), UTF_8);
+ } catch (IOException e) {
+ throw new ConfigException("Could not read from file " + path);
+ }
+ }
+
+ private Reader reader(Path path) throws IOException {
+ return Files.newBufferedReader(path);
}
public void close() {
diff --git
a/clients/src/test/java/org/apache/kafka/common/config/provider/FileConfigProviderTest.java
b/clients/src/test/java/org/apache/kafka/common/config/provider/FileConfigProviderTest.java
index b2c791afebe..7c8f205faaa 100644
---
a/clients/src/test/java/org/apache/kafka/common/config/provider/FileConfigProviderTest.java
+++
b/clients/src/test/java/org/apache/kafka/common/config/provider/FileConfigProviderTest.java
@@ -17,12 +17,18 @@
package org.apache.kafka.common.config.provider;
import org.apache.kafka.common.config.ConfigData;
+import org.apache.kafka.common.utils.Utils;
+import org.apache.kafka.test.TestUtils;
+import org.junit.After;
import org.junit.Before;
import org.junit.Test;
+import java.io.BufferedWriter;
+import java.io.File;
import java.io.IOException;
-import java.io.Reader;
-import java.io.StringReader;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
@@ -32,16 +38,41 @@
public class FileConfigProviderTest {
+ private File testDir;
+ private Path testFile;
+ private File testDir2;
private FileConfigProvider configProvider;
@Before
- public void setup() {
- configProvider = new TestFileConfigProvider();
+ public void setup() throws IOException {
+ testDir = TestUtils.tempDirectory();
+ testFile = Paths.get(testDir.toPath().toString(),
"testFile.properties");
+ try (BufferedWriter writer = Files.newBufferedWriter(testFile)) {
+ writer.write("testKey=testResult\ntestKey2=testResult2");
+ }
+
+ testDir2 = TestUtils.tempDirectory();
+ Path testFile1 = Paths.get(testDir2.toPath().toString(),
"testKey.txt");
+ Path testFile2 = Paths.get(testDir2.toPath().toString(),
"testKey2.txt");
+ try (BufferedWriter writer1 = Files.newBufferedWriter(testFile1)) {
+ writer1.write("testResult");
+ }
+ try (BufferedWriter writer2 = Files.newBufferedWriter(testFile2)) {
+ writer2.write("testResult2");
+ }
+
+ configProvider = new FileConfigProvider();
+ }
+
+ @After
+ public void cleanup() throws IOException {
+ Utils.delete(testDir);
+ Utils.delete(testDir2);
}
@Test
- public void testGetAllKeysAtPath() throws Exception {
- ConfigData configData = configProvider.get("dummy");
+ public void testGetAllKeysInFile() throws Exception {
+ ConfigData configData = configProvider.get(testFile.toString());
Map<String, String> result = new HashMap<>();
result.put("testKey", "testResult");
result.put("testKey2", "testResult2");
@@ -50,14 +81,33 @@ public void testGetAllKeysAtPath() throws Exception {
}
@Test
- public void testGetOneKeyAtPath() throws Exception {
- ConfigData configData = configProvider.get("dummy",
Collections.singleton("testKey"));
+ public void testGetOneKeyInFile() throws Exception {
+ ConfigData configData = configProvider.get(testFile.toString(),
Collections.singleton("testKey"));
Map<String, String> result = new HashMap<>();
result.put("testKey", "testResult");
assertEquals(result, configData.data());
assertEquals(null, configData.ttl());
}
+ @Test
+ public void testGetAllKeysInDir() throws Exception {
+ ConfigData configData = configProvider.get(testDir2.getPath());
+ Map<String, String> result = new HashMap<>();
+ result.put("testKey.txt", "testResult");
+ result.put("testKey2.txt", "testResult2");
+ assertEquals(result, configData.data());
+ assertEquals(null, configData.ttl());
+ }
+
+ @Test
+ public void testGetOneKeyInDir() throws Exception {
+ ConfigData configData = configProvider.get(testDir2.getPath(),
Collections.singleton("testKey.txt"));
+ Map<String, String> result = new HashMap<>();
+ result.put("testKey.txt", "testResult");
+ assertEquals(result, configData.data());
+ assertEquals(null, configData.ttl());
+ }
+
@Test
public void testEmptyPath() throws Exception {
ConfigData configData = configProvider.get("",
Collections.singleton("testKey"));
@@ -85,12 +135,4 @@ public void testNullPathWithKey() throws Exception {
assertTrue(configData.data().isEmpty());
assertEquals(null, configData.ttl());
}
-
- public static class TestFileConfigProvider extends FileConfigProvider {
-
- @Override
- protected Reader reader(String path) throws IOException {
- return new
StringReader("testKey=testResult\ntestKey2=testResult2");
- }
- }
}
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
> Enhance FileConfigProvider to read a directory
> ----------------------------------------------
>
> Key: KAFKA-7370
> URL: https://issues.apache.org/jira/browse/KAFKA-7370
> Project: Kafka
> Issue Type: Improvement
> Components: config
> Affects Versions: 2.0.0
> Reporter: Robert Yokota
> Assignee: Robert Yokota
> Priority: Minor
>
> Currently FileConfigProvider can read a Properties file as a set of key-value
> pairs. This enhancement is to augment FileConfigProvider so that it can also
> read a directory, where the file names are the keys and the corresponding
> file contents are the values.
> This will allow for easier integration with secret management systems where
> each secret is often an individual file, such as in Docker and Kubernetes.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
