This is an automated email from the ASF dual-hosted git repository. michaelsmith pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/impala.git
commit 6e363449824b13fe368da48a15b5727ebe33ce0f Author: Sai Hemanth Gantasala <[email protected]> AuthorDate: Thu Feb 16 13:01:02 2023 -0800 IMPALA-11925: Added a new api add_write_notification_log_in_batch in the MetastoreServiceHandler class In the MetastoreServiceHandler class, the 'add_write_notification_log_ in_batch' API implementation is missing which is causing test failures. This needs to be fixed by adding implementation of the 'add_write_ notification_log_in_batch' api in the MetastoreServiceHandler class Testing: Added a test in the 'test_hms_apis()' to verify that the 'add_write_ notification_log_in_batch' api in HMS is reachable from impala. The same test can be used in the future to test newly added apis in HMS. Change-Id: Ia9a35557c2ed79ed0276c4a418b5292fc6dd6194 Reviewed-on: http://gerrit.cloudera.org:8080/19508 Reviewed-by: Impala Public Jenkins <[email protected]> Tested-by: Impala Public Jenkins <[email protected]> --- .../catalog/metastore/MetastoreServiceHandler.java | 11 +++++++++++ tests/custom_cluster/test_metastore_service.py | 17 +++++++++++++---- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/fe/src/main/java/org/apache/impala/catalog/metastore/MetastoreServiceHandler.java b/fe/src/main/java/org/apache/impala/catalog/metastore/MetastoreServiceHandler.java index c7cfb7635..fbcd66266 100644 --- a/fe/src/main/java/org/apache/impala/catalog/metastore/MetastoreServiceHandler.java +++ b/fe/src/main/java/org/apache/impala/catalog/metastore/MetastoreServiceHandler.java @@ -273,7 +273,9 @@ import org.apache.hadoop.hive.metastore.api.WMGetTriggersForResourePlanResponse; import org.apache.hadoop.hive.metastore.api.WMValidateResourcePlanRequest; import org.apache.hadoop.hive.metastore.api.WMValidateResourcePlanResponse; import org.apache.hadoop.hive.metastore.api.WriteEventInfo; +import org.apache.hadoop.hive.metastore.api.WriteNotificationLogBatchRequest; import org.apache.hadoop.hive.metastore.api.WriteNotificationLogRequest; +import org.apache.hadoop.hive.metastore.api.WriteNotificationLogBatchResponse; import org.apache.hadoop.hive.metastore.api.WriteNotificationLogResponse; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars; @@ -2408,6 +2410,15 @@ public abstract class MetastoreServiceHandler extends AbstractThriftHiveMetastor } } + @Override + public WriteNotificationLogBatchResponse add_write_notification_log_in_batch( + WriteNotificationLogBatchRequest batchRequest) throws TException { + try (MetaStoreClient client = catalog_.getMetaStoreClient()) { + return client.getHiveClient().getThriftClient() + .add_write_notification_log_in_batch(batchRequest); + } + } + @Override public void update_compactor_state(CompactionInfoStruct compactionInfoStruct, long l) throws TException { diff --git a/tests/custom_cluster/test_metastore_service.py b/tests/custom_cluster/test_metastore_service.py index 2ce83907c..d0a696c2b 100644 --- a/tests/custom_cluster/test_metastore_service.py +++ b/tests/custom_cluster/test_metastore_service.py @@ -27,6 +27,7 @@ from hive_metastore.ttypes import TruncateTableRequest from hive_metastore.ttypes import Table from hive_metastore.ttypes import StorageDescriptor from hive_metastore.ttypes import SerDeInfo +from hive_metastore.ttypes import WriteNotificationLogBatchRequest from tests.util.event_processor_utils import EventProcessorUtils from tests.common.custom_cluster_test_suite import CustomClusterTestSuite @@ -983,12 +984,12 @@ class TestMetastoreService(CustomClusterTestSuite): "--hms_port=5899 " "--fallback_to_hms_on_errors=true " ) - def test_compaction_apis(self): + def test_hms_apis(self): """ - The test verifies if the following HMS compaction apis are - reachable from impala (so that MetastoreServiceHandler in impala - can talk to HMS through these APIs): + The test verifies if the following HMS apis are reachable from impala (so that + MetastoreServiceHandler in impala can talk to HMS through these APIs): 1. find_next_compact2 + 2. add_write_notification_log_in_batch """ catalog_hms_client = None tbl_name = ImpalaTestSuite.get_random_name( @@ -997,6 +998,7 @@ class TestMetastoreService(CustomClusterTestSuite): catalog_hms_client, hive_transport = \ ImpalaTestSuite.create_hive_client(5899) assert catalog_hms_client is not None + # Test 1: verify find_next_compact2 api in HMS # create managed table self.run_stmt_in_hive("create transactional table default.{0} (c1 int)" .format(tbl_name)) @@ -1007,6 +1009,13 @@ class TestMetastoreService(CustomClusterTestSuite): compactRequest.workerVersion = "4.0.0" optionalCi = catalog_hms_client.find_next_compact2(compactRequest) # If the above call is successful then find_next_compact2 api is reachable. + # Test 2: verify add_write_notification_log_in_batch api in HMS + rqstList = list() + logBatchRequest = \ + WriteNotificationLogBatchRequest("hive", "default", tbl_name, rqstList) + catalog_hms_client.add_write_notification_log_in_batch(logBatchRequest) + # If the above call is successful then HMS api + # add_write_notification_log_in_batch is reachable. catalog_hms_client.drop_table("default", tbl_name, True) finally: if catalog_hms_client is not None:
