steveloughran commented on code in PR #6938:
URL: https://github.com/apache/hadoop/pull/6938#discussion_r1695514603
##########
hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java:
##########
@@ -927,28 +931,32 @@ private void uploadBlockAsync(final
S3ADataBlocks.DataBlock block,
throw e;
} finally {
// close the stream and block
- cleanupWithLogger(LOG, uploadData, block);
+ LOG.debug("closing block");
+ cleanupWithLogger(LOG, uploadData);
+ cleanupWithLogger(LOG, block);
}
});
partETagsFutures.add(partETagFuture);
}
/**
* Block awaiting all outstanding uploads to complete.
- * @return list of results
+ * @return list of results or null if interrupted.
* @throws IOException IO Problems
*/
private List<CompletedPart> waitForAllPartUploads() throws IOException {
LOG.debug("Waiting for {} uploads to complete", partETagsFutures.size());
try {
return Futures.allAsList(partETagsFutures).get();
} catch (InterruptedException ie) {
- LOG.warn("Interrupted partUpload", ie);
- Thread.currentThread().interrupt();
- return null;
+ // interruptions are raided if a task is aborted by spark.
+ LOG.warn("Interrupted while waiting for uploads to {} to complete",
key, ie);
+ // abort the upload
+ abort();
+ // then regenerate a new InterruptedIOException
+ throw (IOException) new
InterruptedIOException(ie.toString()).initCause(ie);
} catch (ExecutionException ee) {
//there is no way of recovering so abort
- //cancel all partUploads
Review Comment:
looking at this. may need some more review to be confident we are doing
abort here properly
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
