zhedoubushishi commented on a change in pull request #3416:
URL: https://github.com/apache/hudi/pull/3416#discussion_r747979603
##########
File path:
hudi-common/src/main/java/org/apache/hudi/common/config/DFSPropertiesConfiguration.java
##########
@@ -43,70 +46,92 @@
private static final Logger LOG =
LogManager.getLogger(DFSPropertiesConfiguration.class);
+ private static final String DEFAULT_PROPERTIES_FILE = "hudi-defaults.conf";
+
+ public static final String CONF_FILE_DIR_ENV_NAME = "HUDI_CONF_DIR";
+
+ // props read from hudi-defaults.conf
+ private static final TypedProperties HUDI_CONF_PROPS = loadGlobalProps();
+
private final FileSystem fs;
- private final Path rootFile;
+ private Path currentFilePath;
+ // props read from user defined configuration file or input stream
private final TypedProperties props;
// Keep track of files visited, to detect loops
- private final Set<String> visitedFiles;
+ private final Set<String> visitedFilePaths;
- public DFSPropertiesConfiguration(FileSystem fs, Path rootFile,
TypedProperties defaults) {
+ public DFSPropertiesConfiguration(FileSystem fs, Path filePath) {
this.fs = fs;
- this.rootFile = rootFile;
- this.props = defaults;
- this.visitedFiles = new HashSet<>();
- visitFile(rootFile);
- }
-
- public DFSPropertiesConfiguration(FileSystem fs, Path rootFile) {
- this(fs, rootFile, new TypedProperties());
+ this.currentFilePath = filePath;
+ this.props = new TypedProperties();
+ this.visitedFilePaths = new HashSet<>();
+ addPropsFromFile(filePath);
}
public DFSPropertiesConfiguration() {
this.fs = null;
- this.rootFile = null;
+ this.currentFilePath = null;
this.props = new TypedProperties();
- this.visitedFiles = new HashSet<>();
+ this.visitedFilePaths = new HashSet<>();
}
- private String[] splitProperty(String line) {
- int ind = line.indexOf('=');
- String k = line.substring(0, ind).trim();
- String v = line.substring(ind + 1).trim();
- return new String[] {k, v};
+ /**
+ * Load global props from hudi-defaults.conf which is under
CONF_FILE_DIR_ENV_NAME.
+ * @return Typed Properties
+ */
+ public static TypedProperties loadGlobalProps() {
+ DFSPropertiesConfiguration conf = new DFSPropertiesConfiguration();
+ Path defaultConfPath = getDefaultConfPath();
+ if (defaultConfPath != null) {
+ conf.addPropsFromFile(defaultConfPath);
+ }
+ return conf.getProps();
}
- private void visitFile(Path file) {
+ /**
+ * Add properties from external configuration files.
+ *
+ * @param filePath File path for configuration file
+ */
+ public void addPropsFromFile(Path filePath) {
+ FileSystem fileSystem;
try {
- if (visitedFiles.contains(file.getName())) {
- throw new IllegalStateException("Loop detected; file " + file + "
already referenced");
+ fileSystem = fs != null ? fs : filePath.getFileSystem(new
Configuration());
Review comment:
You can still pass a fileSystem with hadoopConfiguration: ```public
DFSPropertiesConfiguration(FileSystem fs, Path filePath)```
Using ```new Configuration()``` only happens when there's no fileSystem be
passed to the class.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
