[hudi] branch master updated: [HUDI-1471] Make QuickStartUtils generate deletes according to specific ts (#2357)

[vinoyang]

This is an automated email from the ASF dual-hosted git repository.

vinoyang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git


The following commit(s) were added to refs/heads/master by this push:
     new f8ccb28  [HUDI-1471] Make QuickStartUtils generate deletes according 
to specific ts (#2357)
f8ccb28 is described below

commit f8ccb2872dc9ba10853650fac2baed1406f3b731
Author: wangxianghu <wxhj...@126.com>
AuthorDate: Tue Dec 22 21:14:18 2020 +0800

    [HUDI-1471] Make QuickStartUtils generate deletes according to specific ts 
(#2357)
---
 .../src/main/java/org/apache/hudi/QuickstartUtils.java       | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git 
a/hudi-spark-datasource/hudi-spark/src/main/java/org/apache/hudi/QuickstartUtils.java
 
b/hudi-spark-datasource/hudi-spark/src/main/java/org/apache/hudi/QuickstartUtils.java
index 025a828..2ba1e28 100644
--- 
a/hudi-spark-datasource/hudi-spark/src/main/java/org/apache/hudi/QuickstartUtils.java
+++ 
b/hudi-spark-datasource/hudi-spark/src/main/java/org/apache/hudi/QuickstartUtils.java
@@ -193,8 +193,12 @@ public class QuickstartUtils {
      * @return list of hoodie records to delete
      */
     public List<String> generateDeletes(List<Row> rows) {
-      return rows.stream().map(row ->
-          convertToString(row.getAs("uuid"), 
row.getAs("partitionpath"))).filter(os -> os.isPresent()).map(os -> os.get())
+      // if row.length() == 2, then the record contains "uuid" and 
"partitionpath" fields, otherwise,
+      // another field "ts" is available
+      return rows.stream().map(row -> row.length() == 2
+          ? convertToString(row.getAs("uuid"), row.getAs("partitionpath"), 
null) :
+          convertToString(row.getAs("uuid"), row.getAs("partitionpath"), 
row.getAs("ts"))
+      ).filter(os -> os.isPresent()).map(os -> os.get())
           .collect(Collectors.toList());
     }
 
@@ -215,10 +219,10 @@ public class QuickstartUtils {
     }
   }
 
-  private static Option<String> convertToString(String uuid, String 
partitionPath) {
+  private static Option<String> convertToString(String uuid, String 
partitionPath, Long ts) {
     StringBuffer stringBuffer = new StringBuffer();
     stringBuffer.append("{");
-    stringBuffer.append("\"ts\": 0.0,");
+    stringBuffer.append("\"ts\": \"" + (ts == null ? "0.0" : ts) + "\",");
     stringBuffer.append("\"uuid\": \"" + uuid + "\",");
     stringBuffer.append("\"partitionpath\": \"" + partitionPath + "\"");
     stringBuffer.append("}");