This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/fluo.git
The following commit(s) were added to refs/heads/master by this push:
new 57b154e Updated commons config from ver 1 to 2 (#1077)
57b154e is described below
commit 57b154e13c5c0877bb565fcabf620aa0f30c9f24
Author: Keith Turner <[email protected]>
AuthorDate: Wed Aug 14 16:34:17 2019 -0400
Updated commons config from ver 1 to 2 (#1077)
---
modules/api/pom.xml | 4 +-
.../fluo/api/config/SimpleConfiguration.java | 82 +++++++++++++++-------
.../fluo/api/config/FluoConfigurationTest.java | 4 +-
modules/api/src/test/resources/fluo-app.properties | 2 +-
modules/api/src/test/resources/fluo.properties | 6 +-
.../src/main/config/fluo-app.properties | 2 +-
.../src/main/config/fluo.properties.deprecated | 8 +--
modules/distribution/src/main/lib/fetch.sh | 3 +-
pom.xml | 10 +--
9 files changed, 77 insertions(+), 44 deletions(-)
diff --git a/modules/api/pom.xml b/modules/api/pom.xml
index 7052929..2e1ad5d 100644
--- a/modules/api/pom.xml
+++ b/modules/api/pom.xml
@@ -33,8 +33,8 @@
<artifactId>guava</artifactId>
</dependency>
<dependency>
- <groupId>commons-configuration</groupId>
- <artifactId>commons-configuration</artifactId>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-configuration2</artifactId>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
diff --git
a/modules/api/src/main/java/org/apache/fluo/api/config/SimpleConfiguration.java
b/modules/api/src/main/java/org/apache/fluo/api/config/SimpleConfiguration.java
index afe8c6f..b48dd83 100644
---
a/modules/api/src/main/java/org/apache/fluo/api/config/SimpleConfiguration.java
+++
b/modules/api/src/main/java/org/apache/fluo/api/config/SimpleConfiguration.java
@@ -23,7 +23,13 @@ import java.io.InputStream;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.io.Reader;
import java.io.Serializable;
+import java.io.StringReader;
+import java.io.UncheckedIOException;
+import java.io.Writer;
+import java.nio.file.Files;
import java.util.Iterator;
import java.util.Map;
import java.util.Map.Entry;
@@ -31,13 +37,15 @@ import java.util.Objects;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableMap.Builder;
-import org.apache.commons.configuration.CompositeConfiguration;
-import org.apache.commons.configuration.Configuration;
-import org.apache.commons.configuration.ConfigurationException;
-import org.apache.commons.configuration.ConfigurationUtils;
-import org.apache.commons.configuration.PropertiesConfiguration;
+import org.apache.commons.configuration2.CompositeConfiguration;
+import org.apache.commons.configuration2.Configuration;
+import org.apache.commons.configuration2.ConfigurationUtils;
+import org.apache.commons.configuration2.PropertiesConfiguration;
+import org.apache.commons.configuration2.ex.ConfigurationException;
import org.apache.fluo.api.exceptions.FluoException;
+import static java.nio.charset.StandardCharsets.UTF_8;
+
/**
* A simple configuration wrapper for Apache Commons configuration. The
implementation supports
* reading and writing properties style config and interpolation.
@@ -57,7 +65,6 @@ public class SimpleConfiguration implements Serializable {
private void init() {
CompositeConfiguration compositeConfig = new CompositeConfiguration();
compositeConfig.setThrowExceptionOnMissing(true);
- compositeConfig.setDelimiterParsingDisabled(true);
internalConfig = compositeConfig;
}
@@ -176,9 +183,7 @@ public class SimpleConfiguration implements Serializable {
public void load(InputStream in) {
try {
PropertiesConfiguration config = new PropertiesConfiguration();
- // disabled to prevent accumulo classpath value from being shortened
- config.setDelimiterParsingDisabled(true);
- config.load(in);
+ config.getLayout().load(config, checkProps(in));
((CompositeConfiguration) internalConfig).addConfiguration(config);
} catch (ConfigurationException e) {
throw new IllegalArgumentException(e);
@@ -192,34 +197,30 @@ public class SimpleConfiguration implements Serializable {
* @since 1.2.0
*/
public void load(File file) {
- try {
+ try (InputStream in = Files.newInputStream(file.toPath())) {
PropertiesConfiguration config = new PropertiesConfiguration();
- // disabled to prevent accumulo classpath value from being shortened
- config.setDelimiterParsingDisabled(true);
- config.load(file);
+ config.getLayout().load(config, checkProps(in));
((CompositeConfiguration) internalConfig).addConfiguration(config);
- } catch (ConfigurationException e) {
+ } catch (ConfigurationException | IOException e) {
throw new IllegalArgumentException(e);
}
}
public void save(File file) {
- PropertiesConfiguration pconf = new PropertiesConfiguration();
- pconf.setDelimiterParsingDisabled(true);
- pconf.append(internalConfig);
- try {
- pconf.save(file);
- } catch (ConfigurationException e) {
+ try (Writer writer = Files.newBufferedWriter(file.toPath())) {
+ PropertiesConfiguration pconf = new PropertiesConfiguration();
+ pconf.append(internalConfig);
+ pconf.getLayout().save(pconf, writer);
+ } catch (ConfigurationException | IOException e) {
throw new FluoException(e);
}
}
public void save(OutputStream out) {
- PropertiesConfiguration pconf = new PropertiesConfiguration();
- pconf.setDelimiterParsingDisabled(true);
- pconf.append(internalConfig);
try {
- pconf.save(out);
+ PropertiesConfiguration pconf = new PropertiesConfiguration();
+ pconf.append(internalConfig);
+ pconf.getLayout().save(pconf, new OutputStreamWriter(out, UTF_8));
} catch (ConfigurationException e) {
throw new FluoException(e);
}
@@ -335,4 +336,37 @@ public class SimpleConfiguration implements Serializable {
ByteArrayInputStream bais = new ByteArrayInputStream(data);
load(bais);
}
+
+ private String stream2String(InputStream in) {
+ try {
+ ByteArrayOutputStream result = new ByteArrayOutputStream();
+ byte[] buffer = new byte[4096];
+ int length;
+ while ((length = in.read(buffer)) != -1) {
+ result.write(buffer, 0, length);
+ }
+
+ return result.toString(UTF_8.name());
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+
+ /*
+ * Commons config 1 was used previously to implement this class. Commons
config 1 required
+ * escaping interpolation. This escaping is no longer required with commmons
config 2. If
+ * interpolation is escaped, then this API behaves differently. This
function suppresses escaped
+ * interpolation in order to maintain behavior for reading.
+ */
+ private Reader checkProps(InputStream in) {
+ String propsData = stream2String(in);
+ if (propsData.contains("\\${")) {
+ throw new IllegalArgumentException(
+ "A Fluo properties value contains \\${. In the past Fluo used
Apache Commons Config 1 and this was required for "
+ + "interpolation. Fluo now uses Commons Config 2 and this is no
longer required. Please remove the slash "
+ + "preceding the interpolation.");
+ }
+
+ return new StringReader(propsData);
+ }
}
diff --git
a/modules/api/src/test/java/org/apache/fluo/api/config/FluoConfigurationTest.java
b/modules/api/src/test/java/org/apache/fluo/api/config/FluoConfigurationTest.java
index 3193f7f..c983bcc 100644
---
a/modules/api/src/test/java/org/apache/fluo/api/config/FluoConfigurationTest.java
+++
b/modules/api/src/test/java/org/apache/fluo/api/config/FluoConfigurationTest.java
@@ -201,8 +201,8 @@ public class FluoConfigurationTest {
} catch (IllegalArgumentException e) {
}
try {
- config.getAccumuloTable();
- Assert.fail();
+ String act = config.getAccumuloTable();
+ Assert.fail("Saw " + act);
} catch (IllegalArgumentException e) {
}
try {
diff --git a/modules/api/src/test/resources/fluo-app.properties
b/modules/api/src/test/resources/fluo-app.properties
index cd0b30f..7c2da3e 100644
--- a/modules/api/src/test/resources/fluo-app.properties
+++ b/modules/api/src/test/resources/fluo-app.properties
@@ -42,7 +42,7 @@ fluo.dfs.root=hdfs://myhost:10000
## Accumulo instance to connect to
fluo.accumulo.instance=myInstance
## Accumulo table to initialize
-fluo.accumulo.table=\${fluo.connection.application.name}
+fluo.accumulo.table=${fluo.connection.application.name}
## Accumulo user
fluo.accumulo.user=testUser
## Accumulo password
diff --git a/modules/api/src/test/resources/fluo.properties
b/modules/api/src/test/resources/fluo.properties
index 4f408ab..d490219 100644
--- a/modules/api/src/test/resources/fluo.properties
+++ b/modules/api/src/test/resources/fluo.properties
@@ -29,7 +29,7 @@ fluo.client.application.name=app1
# Zookeeper connection string specifying host and chroot where Fluo stores
data.
# A chroot directory suffix must be specified but doesn't need to be named
# '/fluo'. If not specified, a Fluo application cannot be initialized.
-# Interpolation (i.e \${fluo.client.accumulo.zookeepers}/fluo) can be used
+# Interpolation (i.e ${fluo.client.accumulo.zookeepers}/fluo) can be used
# when setting this to reuse Accumulo's zookeeper connection string.
fluo.client.zookeeper.connect=localhost/fluo2
# Zookeeper timeout
@@ -57,7 +57,7 @@ fluo.admin.hdfs.root=hdfs://localhost:10000
# every tablet server. For the default jars below, `fluo init` will place them
# in HDFS. If you add more jars to the classpath below, you will need to
# add them to HDFS.
-fluo.admin.accumulo.classpath=\${fluo.admin.hdfs.root}/fluo/lib/fluo-api-${project.version}.jar,\${fluo.admin.hdfs.root}/fluo/lib/fluo-accumulo-${project.version}.jar
+fluo.admin.accumulo.classpath=${fluo.admin.hdfs.root}/fluo/lib/fluo-api-${project.version}.jar,${fluo.admin.hdfs.root}/fluo/lib/fluo-accumulo-${project.version}.jar
# Observer properties
# -------------------
@@ -139,7 +139,7 @@
fluo.admin.accumulo.classpath=\${fluo.admin.hdfs.root}/fluo/lib/fluo-api-${proje
# MiniFluo properties
# -------------------
# Path to directory where MiniFluo stores its data
-#fluo.mini.data.dir=\${env:FLUO_HOME}/apps/\${fluo.client.application.name}/mini
+#fluo.mini.data.dir=${env:FLUO_HOME}/apps/${fluo.client.application.name}/mini
# Indicates if MiniFluo should start a MiniAccumulo cluster
#fluo.mini.start.accumulo=true
diff --git a/modules/distribution/src/main/config/fluo-app.properties
b/modules/distribution/src/main/config/fluo-app.properties
index 88ffa74..a59cd12 100644
--- a/modules/distribution/src/main/config/fluo-app.properties
+++ b/modules/distribution/src/main/config/fluo-app.properties
@@ -48,7 +48,7 @@ fluo.dfs.root=hdfs://localhost:8020/fluo
fluo.accumulo.instance=
## Accumulo table to initialize
-fluo.accumulo.table=\${fluo.connection.application.name}
+fluo.accumulo.table=${fluo.connection.application.name}
## Accumulo user
fluo.accumulo.user=
diff --git a/modules/distribution/src/main/config/fluo.properties.deprecated
b/modules/distribution/src/main/config/fluo.properties.deprecated
index 324f022..8f675f0 100644
--- a/modules/distribution/src/main/config/fluo.properties.deprecated
+++ b/modules/distribution/src/main/config/fluo.properties.deprecated
@@ -32,7 +32,7 @@ fluo.client.application.name=
# Zookeeper connection string specifying host and chroot where Fluo stores
data.
# A chroot directory suffix must be specified but doesn't need to be named
# '/fluo'. If not specified, a Fluo application cannot be initialized.
-# Interpolation (i.e \${fluo.client.accumulo.zookeepers}/fluo) can be used
+# Interpolation (i.e ${fluo.client.accumulo.zookeepers}/fluo) can be used
# when setting this to reuse Accumulo's zookeeper connection string.
#fluo.client.zookeeper.connect=localhost/fluo
@@ -57,7 +57,7 @@ fluo.client.accumulo.password=
# Admin properties
# ----------------
# Accumulo table to initialize
-fluo.admin.accumulo.table=\${fluo.client.application.name}
+fluo.admin.accumulo.table=${fluo.client.application.name}
# HDFS root path. Should match 'fs.defaultFS' property in Hadoop's
core-site.xml
fluo.admin.hdfs.root=hdfs://localhost:10000
@@ -68,7 +68,7 @@ fluo.admin.hdfs.root=hdfs://localhost:10000
# every tablet server. For the default jars below, `fluo init` will place them
# in HDFS. If you add more jars to the classpath below, you will need to
# add them to HDFS.
-fluo.admin.accumulo.classpath=\${fluo.admin.hdfs.root}/fluo/lib/fluo-api-${project.version}.jar,\${fluo.admin.hdfs.root}/fluo/lib/fluo-accumulo-${project.version}.jar
+fluo.admin.accumulo.classpath=${fluo.admin.hdfs.root}/fluo/lib/fluo-api-${project.version}.jar,${fluo.admin.hdfs.root}/fluo/lib/fluo-accumulo-${project.version}.jar
# Observer properties
# -------------------
@@ -156,7 +156,7 @@
fluo.admin.accumulo.classpath=\${fluo.admin.hdfs.root}/fluo/lib/fluo-api-${proje
# MiniFluo properties
# -------------------
# Path to directory where MiniFluo stores its data
-#fluo.mini.data.dir=\${env:FLUO_HOME}/apps/\${fluo.client.application.name}/mini
+#fluo.mini.data.dir=${env:FLUO_HOME}/apps/${fluo.client.application.name}/mini
# Indicates if MiniFluo should start a MiniAccumulo cluster
#fluo.mini.start.accumulo=true
diff --git a/modules/distribution/src/main/lib/fetch.sh
b/modules/distribution/src/main/lib/fetch.sh
index c23c97b..74320a9 100755
--- a/modules/distribution/src/main/lib/fetch.sh
+++ b/modules/distribution/src/main/lib/fetch.sh
@@ -60,8 +60,7 @@ extra)
download com.google.guava:guava:jar:27.0-jre
download com.google.inject:guice:jar:4.0
download org.apache.commons:commons-collections4:jar:4.2
- download commons-configuration:commons-configuration:jar:1.10
- download commons-lang:commons-lang:jar:2.6
+ download org.apache.commons:commons-configuration2:jar:2.5
download commons-io:commons-io:jar:2.6
download io.dropwizard.metrics:metrics-core:jar:3.1.1
download io.dropwizard.metrics:metrics-graphite:jar:3.1.1
diff --git a/pom.xml b/pom.xml
index dd3eed2..cadbbd5 100644
--- a/pom.xml
+++ b/pom.xml
@@ -93,11 +93,6 @@
<version>27.0-jre</version>
</dependency>
<dependency>
- <groupId>commons-configuration</groupId>
- <artifactId>commons-configuration</artifactId>
- <version>1.10</version>
- </dependency>
- <dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>2.6</version>
@@ -144,6 +139,11 @@
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
+ <artifactId>commons-configuration2</artifactId>
+ <version>2.5</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
<artifactId>commons-math3</artifactId>
<version>3.6.1</version>
</dependency>
