This is an automated email from the ASF dual-hosted git repository. xushiyan pushed a commit to branch revert-6110-minor-ci-fix in repository https://gitbox.apache.org/repos/asf/hudi.git
commit b5071b8cf7fee4ebfe7ce8c866f7bacd42e26444 Author: Shiyan Xu <[email protected]> AuthorDate: Fri Jul 22 12:19:04 2022 -0700 Revert "[MINOR] Fix CI issue with TestHiveSyncTool (#6110)" This reverts commit d5c904e10e04980d360129a2ed6b73432b1d2206. --- azure-pipelines.yml | 4 - .../testsuite/job/TestHoodieTestSuiteJob.java | 3 +- .../java/org/apache/hudi/hive/HiveSyncConfig.java | 3 +- .../replication/TestHiveSyncGlobalCommitTool.java | 6 +- .../hudi/hive/testutils/HiveTestService.java | 159 +++++++++++++-------- .../apache/hudi/hive/testutils/HiveTestUtil.java | 4 +- .../{HiveTestCluster.java => TestCluster.java} | 61 ++++---- .../functional/HoodieDeltaStreamerTestBase.java | 5 +- .../functional/TestHoodieDeltaStreamer.java | 2 +- .../utilities/testutils/UtilitiesTestBase.java | 7 +- 10 files changed, 152 insertions(+), 102 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 056f97edf3..dee3e326a9 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -88,7 +88,6 @@ stages: - stage: test jobs: - job: UT_FT_1 - condition: false displayName: UT FT common & flink & UT client/spark-client timeoutInMinutes: '120' steps: @@ -119,7 +118,6 @@ stages: jdkVersionOption: '1.8' mavenOptions: '-Xmx4g' - job: UT_FT_2 - condition: false displayName: FT client/spark-client timeoutInMinutes: '120' steps: @@ -171,7 +169,6 @@ stages: jdkVersionOption: '1.8' mavenOptions: '-Xmx4g' - job: UT_FT_4 - condition: false displayName: UT FT other modules timeoutInMinutes: '120' steps: @@ -202,7 +199,6 @@ stages: jdkVersionOption: '1.8' mavenOptions: '-Xmx4g' - job: IT - condition: false displayName: IT modules timeoutInMinutes: '120' steps: diff --git a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/job/TestHoodieTestSuiteJob.java b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/job/TestHoodieTestSuiteJob.java index ddf5b07247..485c43d4eb 100644 --- a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/job/TestHoodieTestSuiteJob.java +++ b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/job/TestHoodieTestSuiteJob.java @@ -55,7 +55,6 @@ import java.util.UUID; import java.util.stream.Stream; import static org.apache.hudi.hive.HiveSyncConfigHolder.HIVE_URL; -import static org.apache.hudi.hive.testutils.HiveTestService.HS2_JDBC_URL; import static org.apache.hudi.sync.common.HoodieSyncConfig.META_SYNC_DATABASE_NAME; import static org.apache.hudi.sync.common.HoodieSyncConfig.META_SYNC_PARTITION_FIELDS; import static org.apache.hudi.sync.common.HoodieSyncConfig.META_SYNC_TABLE_NAME; @@ -181,7 +180,7 @@ public class TestHoodieTestSuiteJob extends UtilitiesTestBase { // Make path selection test suite specific props.setProperty("hoodie.deltastreamer.source.input.selector", DFSTestSuitePathSelector.class.getName()); // Hive Configs - props.setProperty(HIVE_URL.key(), HS2_JDBC_URL); + props.setProperty(HIVE_URL.key(), "jdbc:hive2://127.0.0.1:9999/"); props.setProperty(META_SYNC_DATABASE_NAME.key(), "testdb1"); props.setProperty(META_SYNC_TABLE_NAME.key(), "table1"); props.setProperty(META_SYNC_PARTITION_FIELDS.key(), "datestr"); diff --git a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveSyncConfig.java b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveSyncConfig.java index 3dc0e4496c..cdb192f9fe 100644 --- a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveSyncConfig.java +++ b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveSyncConfig.java @@ -63,8 +63,7 @@ public class HiveSyncConfig extends HoodieSyncConfig { public HiveSyncConfig(Properties props, Configuration hadoopConf) { super(props, hadoopConf); - HiveConf hiveConf = hadoopConf instanceof HiveConf - ? (HiveConf) hadoopConf : new HiveConf(hadoopConf, HiveConf.class); + HiveConf hiveConf = new HiveConf(hadoopConf, HiveConf.class); // HiveConf needs to load fs conf to allow instantiation via AWSGlueClientFactory hiveConf.addResource(getHadoopFileSystem().getConf()); setHadoopConf(hiveConf); diff --git a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/replication/TestHiveSyncGlobalCommitTool.java b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/replication/TestHiveSyncGlobalCommitTool.java index 02c44f586f..9dffdd0444 100644 --- a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/replication/TestHiveSyncGlobalCommitTool.java +++ b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/replication/TestHiveSyncGlobalCommitTool.java @@ -19,7 +19,7 @@ package org.apache.hudi.hive.replication; -import org.apache.hudi.hive.testutils.HiveTestCluster; +import org.apache.hudi.hive.testutils.TestCluster; import org.apache.hadoop.fs.Path; import org.junit.jupiter.api.AfterEach; @@ -53,9 +53,9 @@ import static org.junit.jupiter.api.Assertions.assertTrue; public class TestHiveSyncGlobalCommitTool { @RegisterExtension - public static HiveTestCluster localCluster = new HiveTestCluster(); + public static TestCluster localCluster = new TestCluster(); @RegisterExtension - public static HiveTestCluster remoteCluster = new HiveTestCluster(); + public static TestCluster remoteCluster = new TestCluster(); private static final String DB_NAME = "foo"; private static final String TBL_NAME = "bar"; diff --git a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/testutils/HiveTestService.java b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/testutils/HiveTestService.java index 16f6bfe53d..66343bfd19 100644 --- a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/testutils/HiveTestService.java +++ b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/testutils/HiveTestService.java @@ -18,7 +18,7 @@ package org.apache.hudi.hive.testutils; -import org.apache.hudi.common.testutils.NetworkTestUtils; +import org.apache.hudi.common.testutils.HoodieTestUtils; import org.apache.hudi.common.util.FileIOUtils; import org.apache.hadoop.conf.Configuration; @@ -62,40 +62,71 @@ import java.util.concurrent.Executors; public class HiveTestService { private static final Logger LOG = LogManager.getLogger(HiveTestService.class); - private static final int CONNECTION_TIMEOUT_MS = 30000; - private static final String BIND_HOST = "127.0.0.1"; - private static final int HS2_THRIFT_PORT = 9999; - public static final String HS2_JDBC_URL = String.format("jdbc:hive2://%s:%s/", BIND_HOST, HS2_THRIFT_PORT); - - private final Configuration hadoopConf; - private final String workDir; - private final Map<String, String> sysProps = new HashMap<>(); + + private static final int CONNECTION_TIMEOUT = 30000; + + /** + * Configuration settings. + */ + private Configuration hadoopConf; + private String workDir; + private String bindIP = "127.0.0.1"; + private int metastorePort = 9083; + private int serverPort = 9999; + private boolean clean = true; + + private Map<String, String> sysProps = new HashMap<>(); private ExecutorService executorService; private TServer tServer; private HiveServer2 hiveServer; - private HiveConf hiveConf; + private HiveConf serverConf; public HiveTestService(Configuration hadoopConf) throws IOException { this.workDir = Files.createTempDirectory(System.currentTimeMillis() + "-").toFile().getAbsolutePath(); this.hadoopConf = hadoopConf; } + public Configuration getHadoopConf() { + return hadoopConf; + } + + public TServer getHiveMetaStore() { + return tServer; + } + + public HiveConf getServerConf() { + return serverConf; + } + public HiveServer2 start() throws IOException { Objects.requireNonNull(workDir, "The work dir must be set before starting cluster."); + if (hadoopConf == null) { + hadoopConf = HoodieTestUtils.getDefaultHadoopConf(); + } + String localHiveLocation = getHiveLocation(workDir); - LOG.info("Cleaning Hive cluster data at: " + localHiveLocation + " and starting fresh."); - File file = new File(localHiveLocation); - FileIOUtils.deleteDirectory(file); + if (clean) { + LOG.info("Cleaning Hive cluster data at: " + localHiveLocation + " and starting fresh."); + File file = new File(localHiveLocation); + FileIOUtils.deleteDirectory(file); + } - hiveConf = configureHive(hadoopConf, localHiveLocation); + serverConf = configureHive(hadoopConf, localHiveLocation); executorService = Executors.newSingleThreadExecutor(); - tServer = startMetaStore(hiveConf); + tServer = startMetaStore(bindIP, serverConf); - hiveServer = startHiveServer(hiveConf); + serverConf.set("hive.in.test", "true"); + hiveServer = startHiveServer(serverConf); - if (!waitForServerUp(hiveConf)) { + String serverHostname; + if (bindIP.equals("0.0.0.0")) { + serverHostname = "localhost"; + } else { + serverHostname = bindIP; + } + if (!waitForServerUp(serverConf, serverHostname, CONNECTION_TIMEOUT)) { throw new IOException("Waiting for startup of standalone server"); } @@ -125,69 +156,76 @@ public class HiveTestService { LOG.info("Hive Minicluster service shut down."); tServer = null; hiveServer = null; + hadoopConf = null; } public HiveServer2 getHiveServer() { return hiveServer; } - public HiveConf getHiveConf() { - return hiveConf; - } - public int getHiveServerPort() { - return hiveConf.getIntVar(ConfVars.HIVE_SERVER2_THRIFT_PORT); + return serverPort; } public String getJdbcHive2Url() { - return String.format("jdbc:hive2://%s:%s/", - hiveConf.getVar(ConfVars.HIVE_SERVER2_THRIFT_BIND_HOST), hiveConf.getIntVar(ConfVars.HIVE_SERVER2_THRIFT_PORT)); + return String.format("jdbc:hive2://%s:%s/default", bindIP, serverPort); } - public HiveConf configureHive(Configuration hadoopConf, String localHiveLocation) throws IOException { - hadoopConf.set("hive.metastore.local", "false"); - hadoopConf.set("datanucleus.schema.autoCreateTables", "true"); - hadoopConf.set("datanucleus.autoCreateSchema", "true"); - hadoopConf.set("datanucleus.fixedDatastore", "false"); - HiveConf conf = new HiveConf(hadoopConf, HiveConf.class); - conf.setBoolVar(ConfVars.HIVE_IN_TEST, true); - conf.setBoolVar(ConfVars.METASTORE_SCHEMA_VERIFICATION, false); - conf.setIntVar(ConfVars.HIVE_SERVER2_THRIFT_PORT, HS2_THRIFT_PORT); - conf.setVar(ConfVars.HIVE_SERVER2_THRIFT_BIND_HOST, BIND_HOST); - final int metastoreServerPort = NetworkTestUtils.nextFreePort(); - conf.setIntVar(ConfVars.METASTORE_SERVER_PORT, metastoreServerPort); - conf.setVar(ConfVars.METASTOREURIS, "thrift://" + BIND_HOST + ":" + metastoreServerPort); + public HiveConf configureHive(Configuration conf, String localHiveLocation) throws IOException { + conf.set("hive.metastore.local", "false"); + int port = metastorePort; + if (conf.get(HiveConf.ConfVars.METASTORE_SERVER_PORT.varname, null) == null) { + conf.setInt(ConfVars.METASTORE_SERVER_PORT.varname, metastorePort); + } else { + port = conf.getInt(ConfVars.METASTORE_SERVER_PORT.varname, metastorePort); + } + if (conf.get(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_PORT.varname, null) == null) { + conf.setInt(ConfVars.HIVE_SERVER2_THRIFT_PORT.varname, serverPort); + } + conf.set(HiveConf.ConfVars.METASTOREURIS.varname, "thrift://" + bindIP + ":" + port); + conf.set(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_BIND_HOST.varname, bindIP); + // The following line to turn of SASL has no effect since HiveAuthFactory calls + // 'new HiveConf()'. This is fixed by https://issues.apache.org/jira/browse/HIVE-6657, + // in Hive 0.14. + // As a workaround, the property is set in hive-site.xml in this module. + // conf.set(HiveConf.ConfVars.HIVE_SERVER2_AUTHENTICATION.varname, "NOSASL"); File localHiveDir = new File(localHiveLocation); localHiveDir.mkdirs(); File metastoreDbDir = new File(localHiveDir, "metastore_db"); - conf.setVar(ConfVars.METASTORECONNECTURLKEY, "jdbc:derby:" + metastoreDbDir.getPath() + ";create=true"); + conf.set(HiveConf.ConfVars.METASTORECONNECTURLKEY.varname, + "jdbc:derby:" + metastoreDbDir.getPath() + ";create=true"); File derbyLogFile = new File(localHiveDir, "derby.log"); derbyLogFile.createNewFile(); setSystemProperty("derby.stream.error.file", derbyLogFile.getPath()); setSystemProperty("derby.system.home", localHiveDir.getAbsolutePath()); - File metastoreWarehouseDir = new File(localHiveDir, "warehouse"); - metastoreWarehouseDir.mkdir(); - conf.setVar(ConfVars.METASTOREWAREHOUSE, metastoreWarehouseDir.getAbsolutePath()); + conf.set(HiveConf.ConfVars.METASTOREWAREHOUSE.varname, + Files.createTempDirectory(System.currentTimeMillis() + "-").toFile().getAbsolutePath()); + conf.set("datanucleus.schema.autoCreateTables", "true"); + conf.set("hive.metastore.schema.verification", "false"); + conf.set("datanucleus.autoCreateSchema", "true"); + conf.set("datanucleus.fixedDatastore", "false"); + setSystemProperty("derby.stream.error.file", derbyLogFile.getPath()); - return conf; + return new HiveConf(conf, this.getClass()); } - private boolean waitForServerUp(HiveConf serverConf) { - LOG.info("waiting for " + serverConf.getVar(ConfVars.METASTOREURIS)); - final long start = System.currentTimeMillis(); + private boolean waitForServerUp(HiveConf serverConf, String hostname, int timeout) { + long start = System.currentTimeMillis(); + int port = serverConf.getIntVar(HiveConf.ConfVars.METASTORE_SERVER_PORT); while (true) { try { new HiveMetaStoreClient(serverConf); return true; } catch (MetaException e) { // ignore as this is expected + LOG.info("server " + hostname + ":" + port + " not up " + e); } - if (System.currentTimeMillis() > start + CONNECTION_TIMEOUT_MS) { + if (System.currentTimeMillis() > start + timeout) { break; } try { - Thread.sleep(CONNECTION_TIMEOUT_MS / 10); + Thread.sleep(250); } catch (InterruptedException e) { // ignore } @@ -269,23 +307,28 @@ public class HiveTestService { } } - private TServer startMetaStore(HiveConf conf) throws IOException { + public TServer startMetaStore(String forceBindIP, HiveConf conf) throws IOException { try { // Server will create new threads up to max as necessary. After an idle // period, it will destory threads to keep the number of threads in the // pool to min. - String host = conf.getVar(ConfVars.HIVE_SERVER2_THRIFT_BIND_HOST); - int port = conf.getIntVar(ConfVars.METASTORE_SERVER_PORT); - int minWorkerThreads = conf.getIntVar(ConfVars.METASTORESERVERMINTHREADS); - int maxWorkerThreads = conf.getIntVar(ConfVars.METASTORESERVERMAXTHREADS); - boolean tcpKeepAlive = conf.getBoolVar(ConfVars.METASTORE_TCP_KEEP_ALIVE); - boolean useFramedTransport = conf.getBoolVar(ConfVars.METASTORE_USE_THRIFT_FRAMED_TRANSPORT); + int port = conf.getIntVar(HiveConf.ConfVars.METASTORE_SERVER_PORT); + int minWorkerThreads = conf.getIntVar(HiveConf.ConfVars.METASTORESERVERMINTHREADS); + int maxWorkerThreads = conf.getIntVar(HiveConf.ConfVars.METASTORESERVERMAXTHREADS); + boolean tcpKeepAlive = conf.getBoolVar(HiveConf.ConfVars.METASTORE_TCP_KEEP_ALIVE); + boolean useFramedTransport = conf.getBoolVar(HiveConf.ConfVars.METASTORE_USE_THRIFT_FRAMED_TRANSPORT); // don't support SASL yet - // boolean useSasl = conf.getBoolVar(ConfVars.METASTORE_USE_THRIFT_SASL); + // boolean useSasl = conf.getBoolVar(HiveConf.ConfVars.METASTORE_USE_THRIFT_SASL); + + TServerTransport serverTransport; + if (forceBindIP != null) { + InetSocketAddress address = new InetSocketAddress(forceBindIP, port); + serverTransport = tcpKeepAlive ? new TServerSocketKeepAlive(address) : new TServerSocket(address); - InetSocketAddress address = new InetSocketAddress(host, port); - TServerTransport serverTransport = tcpKeepAlive ? new TServerSocketKeepAlive(address) : new TServerSocket(address); + } else { + serverTransport = tcpKeepAlive ? new TServerSocketKeepAlive(port) : new TServerSocket(port); + } TProcessor processor; TTransportFactory transFactory; @@ -293,7 +336,7 @@ public class HiveTestService { HiveMetaStore.HMSHandler baseHandler = new HiveMetaStore.HMSHandler("new db based metaserver", conf, false); IHMSHandler handler = RetryingHMSHandler.getProxy(conf, baseHandler, true); - if (conf.getBoolVar(ConfVars.METASTORE_EXECUTE_SET_UGI)) { + if (conf.getBoolVar(HiveConf.ConfVars.METASTORE_EXECUTE_SET_UGI)) { transFactory = useFramedTransport ? new ChainedTTransportFactory(new TFramedTransport.Factory(), new TUGIContainingTransport.Factory()) : new TUGIContainingTransport.Factory(); diff --git a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/testutils/HiveTestUtil.java b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/testutils/HiveTestUtil.java index 6cae616e60..9687e55792 100644 --- a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/testutils/HiveTestUtil.java +++ b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/testutils/HiveTestUtil.java @@ -125,6 +125,7 @@ public class HiveTestUtil { hiveTestService = new HiveTestService(configuration); hiveServer = hiveTestService.start(); } + fileSystem = FileSystem.get(configuration); basePath = Files.createTempDirectory("hivesynctest" + Instant.now().toEpochMilli()).toUri().toString(); @@ -140,8 +141,7 @@ public class HiveTestUtil { hiveSyncProps.setProperty(META_SYNC_PARTITION_FIELDS.key(), "datestr"); hiveSyncProps.setProperty(HIVE_BATCH_SYNC_PARTITION_NUM.key(), "3"); - hiveSyncConfig = new HiveSyncConfig(hiveSyncProps, hiveTestService.getHiveConf()); - fileSystem = hiveSyncConfig.getHadoopFileSystem(); + hiveSyncConfig = new HiveSyncConfig(hiveSyncProps, configuration); dtfOut = DateTimeFormatter.ofPattern("yyyy/MM/dd"); ddlExecutor = new HiveQueryDDLExecutor(hiveSyncConfig); diff --git a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/testutils/HiveTestCluster.java b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/testutils/TestCluster.java similarity index 86% rename from hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/testutils/HiveTestCluster.java rename to hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/testutils/TestCluster.java index 39813394d2..c1f891fce8 100644 --- a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/testutils/HiveTestCluster.java +++ b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/testutils/TestCluster.java @@ -41,6 +41,7 @@ import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; import org.apache.hadoop.hive.metastore.IMetaStoreClient; import org.apache.hadoop.hive.metastore.RetryingMetaStoreClient; @@ -56,6 +57,7 @@ import org.junit.jupiter.api.extension.AfterEachCallback; import org.junit.jupiter.api.extension.BeforeAllCallback; import org.junit.jupiter.api.extension.BeforeEachCallback; import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.runners.model.InitializationError; import java.io.File; import java.io.FileOutputStream; @@ -63,7 +65,6 @@ import java.io.IOException; import java.io.OutputStream; import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; -import java.nio.file.Files; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.time.temporal.ChronoUnit; @@ -74,15 +75,16 @@ import java.util.UUID; import static org.junit.jupiter.api.Assertions.fail; -public class HiveTestCluster implements BeforeAllCallback, AfterAllCallback, - BeforeEachCallback, AfterEachCallback { - public MiniDFSCluster dfsCluster; +public class TestCluster implements BeforeAllCallback, AfterAllCallback, + BeforeEachCallback, AfterEachCallback { private HdfsTestService hdfsTestService; - private HiveTestService hiveTestService; - private HiveConf conf; - private HiveServer2 server2; - private DateTimeFormatter dtfOut; - private File hiveSiteXml; + public HiveTestService hiveTestService; + private Configuration conf; + public HiveServer2 server2; + private static volatile int port = 9083; + public MiniDFSCluster dfsCluster; + DateTimeFormatter dtfOut; + public File hiveSiteXml; private IMetaStoreClient client; @Override @@ -107,18 +109,24 @@ public class HiveTestCluster implements BeforeAllCallback, AfterAllCallback, hdfsTestService = new HdfsTestService(); dfsCluster = hdfsTestService.start(true); - Configuration hadoopConf = hdfsTestService.getHadoopConf(); - hiveTestService = new HiveTestService(hadoopConf); + conf = hdfsTestService.getHadoopConf(); + conf.setInt(ConfVars.METASTORE_SERVER_PORT.varname, port++); + conf.setInt(ConfVars.HIVE_SERVER2_THRIFT_PORT.varname, port++); + conf.setInt(ConfVars.HIVE_SERVER2_WEBUI_PORT.varname, port++); + hiveTestService = new HiveTestService(conf); server2 = hiveTestService.start(); dtfOut = DateTimeFormatter.ofPattern("yyyy/MM/dd"); hiveSiteXml = File.createTempFile("hive-site", ".xml"); hiveSiteXml.deleteOnExit(); - conf = hiveTestService.getHiveConf(); try (OutputStream os = new FileOutputStream(hiveSiteXml)) { - conf.writeXml(os); + hiveTestService.getServerConf().writeXml(os); } client = HiveMetaStoreClient.newSynchronizedClient( - RetryingMetaStoreClient.getProxy(conf, true)); + RetryingMetaStoreClient.getProxy(hiveTestService.getServerConf(), true)); + } + + public Configuration getConf() { + return this.conf; } public String getHiveSiteXmlLocation() { @@ -130,7 +138,7 @@ public class HiveTestCluster implements BeforeAllCallback, AfterAllCallback, } public String getHiveJdBcUrl() { - return hiveTestService.getJdbcHive2Url(); + return "jdbc:hive2://127.0.0.1:" + conf.get(ConfVars.HIVE_SERVER2_THRIFT_PORT.varname) + ""; } public String tablePath(String dbName, String tableName) throws Exception { @@ -143,12 +151,12 @@ public class HiveTestCluster implements BeforeAllCallback, AfterAllCallback, public void forceCreateDb(String dbName) throws Exception { try { - client.dropDatabase(dbName); - } catch (NoSuchObjectException ignored) { - // expected + getHMSClient().dropDatabase(dbName); + } catch (NoSuchObjectException e) { + System.out.println("db does not exist but its ok " + dbName); } Database db = new Database(dbName, "", dbPath(dbName), new HashMap<>()); - client.createDatabase(db); + getHMSClient().createDatabase(db); } public void createCOWTable(String commitTime, int numberOfPartitions, String dbName, String tableName) @@ -161,7 +169,10 @@ public class HiveTestCluster implements BeforeAllCallback, AfterAllCallback, .setTableName(tableName) .setPayloadClass(HoodieAvroPayload.class) .initTable(conf, path.toString()); - dfsCluster.getFileSystem().mkdirs(path); + boolean result = dfsCluster.getFileSystem().mkdirs(path); + if (!result) { + throw new InitializationError("cannot initialize table"); + } ZonedDateTime dateTime = ZonedDateTime.now(); HoodieCommitMetadata commitMetadata = createPartitions(numberOfPartitions, true, dateTime, commitTime, path.toString()); createCommitFile(commitMetadata, commitTime, path.toString()); @@ -228,7 +239,7 @@ public class HiveTestCluster implements BeforeAllCallback, AfterAllCallback, try { writer.write(s); } catch (IOException e) { - fail("IOException while writing test records as parquet", e); + fail("IOException while writing test records as parquet" + e.toString()); } }); writer.close(); @@ -248,15 +259,15 @@ public class HiveTestCluster implements BeforeAllCallback, AfterAllCallback, public void startHiveServer2() { if (server2 == null) { server2 = new HiveServer2(); - server2.init(hiveTestService.getHiveConf()); + server2.init(hiveTestService.getServerConf()); server2.start(); } } - public void shutDown() throws IOException { - Files.deleteIfExists(hiveSiteXml.toPath()); + public void shutDown() { + stopHiveServer2(); Hive.closeCurrent(); - hiveTestService.stop(); + hiveTestService.getHiveMetaStore().stop(); hdfsTestService.stop(); } } diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/HoodieDeltaStreamerTestBase.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/HoodieDeltaStreamerTestBase.java index ad74235ae0..b4497289fd 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/HoodieDeltaStreamerTestBase.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/HoodieDeltaStreamerTestBase.java @@ -48,7 +48,6 @@ import java.util.Map; import java.util.Random; import static org.apache.hudi.hive.HiveSyncConfigHolder.HIVE_URL; -import static org.apache.hudi.hive.testutils.HiveTestService.HS2_JDBC_URL; import static org.apache.hudi.sync.common.HoodieSyncConfig.META_SYNC_ASSUME_DATE_PARTITION; import static org.apache.hudi.sync.common.HoodieSyncConfig.META_SYNC_DATABASE_NAME; import static org.apache.hudi.sync.common.HoodieSyncConfig.META_SYNC_PARTITION_EXTRACTOR_CLASS; @@ -187,7 +186,7 @@ public class HoodieDeltaStreamerTestBase extends UtilitiesTestBase { props.setProperty("hoodie.deltastreamer.schemaprovider.target.schema.file", dfsBasePath + "/target.avsc"); // Hive Configs - props.setProperty(HIVE_URL.key(), HS2_JDBC_URL); + props.setProperty(HIVE_URL.key(), "jdbc:hive2://127.0.0.1:9999/"); props.setProperty(META_SYNC_DATABASE_NAME.key(), "testdb1"); props.setProperty(META_SYNC_TABLE_NAME.key(), "hive_trips"); props.setProperty(META_SYNC_PARTITION_FIELDS.key(), "datestr"); @@ -247,7 +246,7 @@ public class HoodieDeltaStreamerTestBase extends UtilitiesTestBase { protected static void populateCommonHiveProps(TypedProperties props) { // Hive Configs - props.setProperty(HIVE_URL.key(), HS2_JDBC_URL); + props.setProperty(HIVE_URL.key(), "jdbc:hive2://127.0.0.1:9999/"); props.setProperty(META_SYNC_DATABASE_NAME.key(), "testdb2"); props.setProperty(META_SYNC_ASSUME_DATE_PARTITION.key(), "false"); props.setProperty(META_SYNC_PARTITION_FIELDS.key(), "datestr"); diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieDeltaStreamer.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieDeltaStreamer.java index 850b0d1d60..dde0e5f73f 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieDeltaStreamer.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/functional/TestHoodieDeltaStreamer.java @@ -1359,7 +1359,7 @@ public class TestHoodieDeltaStreamer extends HoodieDeltaStreamerTestBase { // Test Hive integration HiveSyncConfig hiveSyncConfig = getHiveSyncConfig(tableBasePath, "hive_trips"); hiveSyncConfig.setValue(META_SYNC_PARTITION_FIELDS, "year,month,day"); - hiveSyncConfig.setHadoopConf(hiveTestService.getHiveConf()); + hiveSyncConfig.setHadoopConf(hiveServer.getHiveConf()); HoodieHiveSyncClient hiveClient = new HoodieHiveSyncClient(hiveSyncConfig); final String tableName = hiveSyncConfig.getString(META_SYNC_TABLE_NAME); assertTrue(hiveClient.tableExists(tableName), "Table " + tableName + " should exist"); diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/UtilitiesTestBase.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/UtilitiesTestBase.java index ff7d6cc2ed..67a002c3ba 100644 --- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/UtilitiesTestBase.java +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/testutils/UtilitiesTestBase.java @@ -56,6 +56,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.MiniDFSCluster; +import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hive.service.server.HiveServer2; import org.apache.log4j.Level; import org.apache.log4j.Logger; @@ -196,7 +197,7 @@ public class UtilitiesTestBase { */ protected static HiveSyncConfig getHiveSyncConfig(String basePath, String tableName) { Properties props = new Properties(); - props.setProperty(HIVE_URL.key(), hiveTestService.getJdbcHive2Url()); + props.setProperty(HIVE_URL.key(),"jdbc:hive2://127.0.0.1:9999/"); props.setProperty(HIVE_USER.key(), ""); props.setProperty(HIVE_PASS.key(), ""); props.setProperty(META_SYNC_DATABASE_NAME.key(), "testdb1"); @@ -214,9 +215,11 @@ public class UtilitiesTestBase { * @throws IOException */ private static void clearHiveDb() throws Exception { + HiveConf hiveConf = new HiveConf(); // Create Dummy hive sync config HiveSyncConfig hiveSyncConfig = getHiveSyncConfig("/dummy", "dummy"); - hiveSyncConfig.setHadoopConf(hiveTestService.getHiveConf()); + hiveConf.addResource(hiveServer.getHiveConf()); + hiveSyncConfig.setHadoopConf(hiveConf); HoodieTableMetaClient.withPropertyBuilder() .setTableType(HoodieTableType.COPY_ON_WRITE) .setTableName(hiveSyncConfig.getString(META_SYNC_TABLE_NAME))
