ctubbsii closed pull request #381: ACCUMULO-3807 Avoid Copy/Sort column on WAL
recovery exceeds 100%
URL: https://github.com/apache/accumulo/pull/381
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java
index a5c8416968..7003bb5d64 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/DfsLogger.java
@@ -449,9 +449,7 @@ public synchronized void open(String address) throws
IOException {
short replication = (short)
conf.getConfiguration().getCount(Property.TSERV_WAL_REPLICATION);
if (replication == 0)
replication = fs.getDefaultReplication(new Path(logPath));
- long blockSize =
conf.getConfiguration().getAsBytes(Property.TSERV_WAL_BLOCKSIZE);
- if (blockSize == 0)
- blockSize = (long)
(conf.getConfiguration().getAsBytes(Property.TSERV_WALOG_MAX_SIZE) * 1.1);
+ long blockSize = getWalBlockSize(conf.getConfiguration());
if (conf.getConfiguration().getBoolean(Property.TSERV_WAL_SYNC))
logFile = fs.createSyncable(new Path(logPath), 0, replication,
blockSize);
else
@@ -514,6 +512,13 @@ public synchronized void open(String address) throws
IOException {
log.debug("Got new write-ahead log: {}", this);
}
+ static long getWalBlockSize(AccumuloConfiguration conf) {
+ long blockSize = conf.getAsBytes(Property.TSERV_WAL_BLOCKSIZE);
+ if (blockSize == 0)
+ blockSize = (long) (conf.getAsBytes(Property.TSERV_WALOG_MAX_SIZE) *
1.1);
+ return blockSize;
+ }
+
@Override
public String toString() {
String fileName = getFileName();
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/LogSorter.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/LogSorter.java
index fc72b98ef0..64775adfe5 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/LogSorter.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/LogSorter.java
@@ -221,6 +221,7 @@ synchronized long getBytesCopied() throws IOException {
ThreadPoolExecutor threadPool;
private final Instance instance;
+ private double walBlockSize;
public LogSorter(Instance instance, VolumeManager fs, AccumuloConfiguration
conf) {
this.instance = instance;
@@ -228,6 +229,7 @@ public LogSorter(Instance instance, VolumeManager fs,
AccumuloConfiguration conf
this.conf = conf;
int threadPoolSize = conf.getCount(Property.TSERV_RECOVERY_MAX_CONCURRENT);
this.threadPool = new SimpleThreadPool(threadPoolSize,
this.getClass().getName());
+ this.walBlockSize = DfsLogger.getWalBlockSize(conf);
}
public void startWatchingForRecoveryLogs(ThreadPoolExecutor
distWorkQThreadPool) throws KeeperException, InterruptedException {
@@ -242,7 +244,9 @@ public void startWatchingForRecoveryLogs(ThreadPoolExecutor
distWorkQThreadPool)
RecoveryStatus status = new RecoveryStatus();
status.name = entries.getKey();
try {
- status.progress = entries.getValue().getBytesCopied() / (0.0 +
conf.getAsBytes(Property.TSERV_WALOG_MAX_SIZE));
+ double progress = entries.getValue().getBytesCopied() / walBlockSize;
+ // to be sure progress does not exceed 100%
+ status.progress = Math.min(progress, 99.9);
} catch (IOException ex) {
log.warn("Error getting bytes read");
}
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
