Github user StefanRRichter commented on a diff in the pull request:
https://github.com/apache/flink/pull/5601#discussion_r172152995
--- Diff:
flink-yarn/src/test/java/org/apache/flink/yarn/YarnFileStageTest.java ---
@@ -200,13 +201,23 @@ static void testCopyFromLocalRecursive(
while (targetFilesIterator.hasNext()) {
LocatedFileStatus targetFile =
targetFilesIterator.next();
- try (FSDataInputStream in =
targetFileSystem.open(targetFile.getPath())) {
- String absolutePathString =
targetFile.getPath().toString();
- String relativePath =
absolutePathString.substring(workDirPrefixLength);
- targetFiles.put(relativePath,
in.readUTF());
-
- assertEquals("extraneous data in file "
+ relativePath, -1, in.read());
- }
+ int retries = 5;
+ do {
+ try (FSDataInputStream in =
targetFileSystem.open(targetFile.getPath())) {
+ String absolutePathString =
targetFile.getPath().toString();
+ String relativePath =
absolutePathString.substring(workDirPrefixLength);
+ targetFiles.put(relativePath,
in.readUTF());
+
+ assertEquals("extraneous data
in file " + relativePath, -1, in.read());
+ break;
+ } catch (FileNotFoundException e) {
+ // For S3, read-after-write may
be eventually consistent, i.e. when trying
+ // to access the object before
writing it; see
+ //
https://docs.aws.amazon.com/AmazonS3/latest/dev/Introduction.html#ConsistencyModel
+ // -> try again a bit later
+ Thread.sleep(50);
--- End diff --
Same here.
---
