codope commented on a change in pull request #2768:
URL: https://github.com/apache/hudi/pull/2768#discussion_r779376837
##########
File path:
hudi-utilities/src/test/java/org/apache/hudi/utilities/TestHiveIncrementalPuller.java
##########
@@ -41,4 +67,84 @@ public void testInitHiveIncrementalPuller() {
}
+ private HiveIncrementalPuller.Config getHivePullerConfig(String
incrementalSql) throws IOException {
+ config.hiveJDBCUrl = hiveSyncConfig.jdbcUrl;
+ config.hiveUsername = hiveSyncConfig.hiveUser;
+ config.hivePassword = hiveSyncConfig.hivePass;
+ config.hoodieTmpDir =
Files.createTempDirectory("hivePullerTest").toUri().toString();
+ config.sourceDb = hiveSyncConfig.databaseName;
+ config.sourceTable = hiveSyncConfig.tableName;
+ config.targetDb = "tgtdb";
+ config.targetTable = "test2";
+ config.tmpDb = "tmp_db";
+ config.fromCommitTime = "100";
+ createIncrementalSqlFile(incrementalSql, config);
+ return config;
+ }
+
+ private void createIncrementalSqlFile(String text,
HiveIncrementalPuller.Config cfg) throws IOException {
+ java.nio.file.Path path = Paths.get(cfg.hoodieTmpDir +
"/incremental_pull.txt");
+ Files.createDirectories(path.getParent());
+ Files.createFile(path);
+ try (FileWriter fr = new FileWriter(new File(path.toUri()))) {
+ fr.write(text);
+ } catch (Exception e) {
+ // no-op
+ }
+ cfg.incrementalSQLFile = path.toString();
+ }
+
+ private void createSourceTable() throws IOException, URISyntaxException {
+ String instantTime = "101";
+ HiveTestUtil.createCOWTable(instantTime, 5, true);
+ hiveSyncConfig.syncMode = "jdbc";
+ HiveTestUtil.hiveSyncConfig.batchSyncNum = 3;
+ HiveSyncTool tool = new HiveSyncTool(hiveSyncConfig,
HiveTestUtil.getHiveConf(), fileSystem);
+ tool.syncHoodieTable();
+ }
+
+ private void createTargetTable() throws IOException, URISyntaxException {
+ String instantTime = "100";
+ targetBasePath = Files.createTempDirectory("hivesynctest1" +
Instant.now().toEpochMilli()).toUri().toString();
+ HiveTestUtil.createCOWTable(instantTime, 5, true,
+ targetBasePath, "tgtdb", "test2");
+ HiveSyncTool tool = new
HiveSyncTool(getTargetHiveSyncConfig(targetBasePath),
HiveTestUtil.getHiveConf(), fileSystem);
+ tool.syncHoodieTable();
+ }
+
+ private HiveSyncConfig getTargetHiveSyncConfig(String basePath) {
+ HiveSyncConfig config = HiveSyncConfig.copy(hiveSyncConfig);
+ config.databaseName = "tgtdb";
+ config.tableName = "test2";
+ config.basePath = basePath;
+ config.batchSyncNum = 3;
+ config.syncMode = "jdbc";
+ return config;
+ }
+
+ private void createTables() throws IOException, URISyntaxException {
+ createSourceTable();
+ createTargetTable();
+ }
+
+ @Test
+ public void testPullerWithoutIncrementalClause() throws IOException,
URISyntaxException {
Review comment:
@pratyakshsharma Is the patch ready? If not, can you please update the
happy flow test case even if it's failing. I can take over and try to fix it.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@hudi.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
