sankarh commented on a change in pull request #569: HIVE-21446 : Hive Server
going OOM during hive external table replications
URL: https://github.com/apache/hive/pull/569#discussion_r266296535
##########
File path:
ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ExternalTableCopyTaskBuilder.java
##########
@@ -99,53 +99,65 @@ private boolean createAndSetPathOwner(Path destPath, Path
sourcePath) throws IOE
return createdDir;
}
- private boolean setTargetPathOwner(Path targetPath, Path sourcePath,
String distCpDoAsUser)
- throws IOException {
- if (distCpDoAsUser == null) {
+ private boolean setTargetPathOwner(Path targetPath, Path sourcePath,
UserGroupInformation proxyUser)
+ throws IOException, InterruptedException {
+ if (proxyUser == null) {
return createAndSetPathOwner(targetPath, sourcePath);
}
- UserGroupInformation proxyUser = UserGroupInformation.createProxyUser(
- distCpDoAsUser, UserGroupInformation.getLoginUser());
- try {
- Path finalTargetPath = targetPath;
- Path finalSourcePath = sourcePath;
- return proxyUser.doAs((PrivilegedExceptionAction<Boolean>) () ->
- createAndSetPathOwner(finalTargetPath, finalSourcePath));
- } catch (InterruptedException e) {
- throw new IOException(e);
+ return proxyUser.doAs((PrivilegedExceptionAction<Boolean>) () ->
+ createAndSetPathOwner(targetPath, sourcePath));
+ }
+
+ private boolean checkIfPathExist(Path sourcePath, UserGroupInformation
proxyUser) throws Exception {
+ if (proxyUser == null) {
+ return sourcePath.getFileSystem(conf).exists(sourcePath);
}
+ return proxyUser.doAs((PrivilegedExceptionAction<Boolean>) () ->
+ sourcePath.getFileSystem(conf).exists(sourcePath));
}
- private int handleException(Exception e, Path sourcePath, Path targetPath,
int currentRetry) {
+ private int handleException(Exception e, Path sourcePath, Path targetPath,
+ int currentRetry, UserGroupInformation
proxyUser) {
try {
- if (!sourcePath.getFileSystem(conf).exists(sourcePath)) {
- LOG.warn("Source path missing " + sourcePath, e);
+ LOG.warn("Checking if source path " + sourcePath + " is missing for
exception ", e);
+ if (!checkIfPathExist(sourcePath, proxyUser)) {
+ LOG.warn("Source path is missing. Ignoring exception.");
Review comment:
Can be just info log as it is possible valid scenario in case of external
tables.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
