This is an automated email from the ASF dual-hosted git repository. stigahuang pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/impala.git
commit ded49055d3c21e4a0f92a6b827b6997dc73a09c8 Author: Michael Smith <[email protected]> AuthorDate: Thu Sep 29 16:46:41 2022 -0700 IMPALA-9448: (Addendum) Avoid generic recreate for Ozone bucket Avoids using `hadoop fs` to recreate an Ozone bucket during snapshot load, as this removes the encryption key. Change-Id: I31a3813fac6f2d68859ba7fed53b5c5a82502780 Reviewed-on: http://gerrit.cloudera.org:8080/19082 Reviewed-by: Impala Public Jenkins <[email protected]> Tested-by: Impala Public Jenkins <[email protected]> --- testdata/bin/load-test-warehouse-snapshot.sh | 11 +++++++---- tests/custom_cluster/test_exchange_delays.py | 5 +++-- tests/util/filesystem_utils.py | 1 + 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/testdata/bin/load-test-warehouse-snapshot.sh b/testdata/bin/load-test-warehouse-snapshot.sh index cef8dc6ca..11f60a237 100755 --- a/testdata/bin/load-test-warehouse-snapshot.sh +++ b/testdata/bin/load-test-warehouse-snapshot.sh @@ -70,17 +70,20 @@ if [[ "$REPLY" =~ ^[Yy]$ ]]; then else # Either isilon or hdfs, no change in procedure. if hadoop fs -test -d ${FILESYSTEM_PREFIX}${TEST_WAREHOUSE_DIR}; then - echo "Removing existing ${TEST_WAREHOUSE_DIR} directory" + echo "Removing existing ${TEST_WAREHOUSE_DIR} directory contents" # For filesystems that don't allow 'rm' without 'x', chmod to 777 for the # subsequent 'rm -r'. if [ "${TARGET_FILESYSTEM}" = "isilon" ] || \ [ "${TARGET_FILESYSTEM}" = "local" ]; then hadoop fs -chmod -R 777 ${FILESYSTEM_PREFIX}${TEST_WAREHOUSE_DIR} fi - hadoop fs -rm -r -skipTrash ${FILESYSTEM_PREFIX}${TEST_WAREHOUSE_DIR} + hadoop fs -rm -f -r -skipTrash "${FILESYSTEM_PREFIX}${TEST_WAREHOUSE_DIR}/*" + else + # Only create the directory if it doesn't exist. Some filesystems - such as Ozone - + # are created with extra properties. + echo "Creating ${TEST_WAREHOUSE_DIR} directory" + hadoop fs -mkdir -p ${FILESYSTEM_PREFIX}${TEST_WAREHOUSE_DIR} fi - echo "Creating ${TEST_WAREHOUSE_DIR} directory" - hadoop fs -mkdir -p ${FILESYSTEM_PREFIX}${TEST_WAREHOUSE_DIR} if [[ -n "${HDFS_ERASURECODE_POLICY:-}" ]]; then hdfs ec -enablePolicy -policy "${HDFS_ERASURECODE_POLICY}" hdfs ec -setPolicy -policy "${HDFS_ERASURECODE_POLICY}" \ diff --git a/tests/custom_cluster/test_exchange_delays.py b/tests/custom_cluster/test_exchange_delays.py index 1df77b31e..b6aa044c5 100644 --- a/tests/custom_cluster/test_exchange_delays.py +++ b/tests/custom_cluster/test_exchange_delays.py @@ -19,7 +19,7 @@ import pytest from tests.common.custom_cluster_test_suite import CustomClusterTestSuite from tests.common.environ import build_flavor_timeout from tests.common.skip import SkipIfBuildType -from tests.util.filesystem_utils import IS_S3, IS_ADLS, IS_ISILON +from tests.util.filesystem_utils import IS_S3, IS_ADLS, IS_ISILON, IS_OZONE, IS_ENCRYPTED # IMPALA-6100: add additional margin for error for slow build types. SLOW_BUILD_TIMEOUT=20000 @@ -67,8 +67,9 @@ class TestExchangeDelays(CustomClusterTestSuite): # can't time out). Use a longer delay for platforms that have slow scans: # IMPALA-6811: S3/ADLS have slow scans. # IMPALA-6866: Isilon has slow scans (and is counted as a slow build above). + # Scans with Ozone encryption are also slower. SLOW_SCAN_EXTRA_DELAY_MS = 10000 - if IS_S3 or IS_ADLS or IS_ISILON: + if IS_S3 or IS_ADLS or IS_ISILON or (IS_OZONE and IS_ENCRYPTED): DELAY_MS += SLOW_SCAN_EXTRA_DELAY_MS @pytest.mark.execute_serially diff --git a/tests/util/filesystem_utils.py b/tests/util/filesystem_utils.py index a9f0fff74..88f5bfd0a 100644 --- a/tests/util/filesystem_utils.py +++ b/tests/util/filesystem_utils.py @@ -35,6 +35,7 @@ IS_GCS = FILESYSTEM == "gs" IS_COS = FILESYSTEM == "cosn" IS_OZONE = FILESYSTEM == "ozone" IS_EC = os.getenv("ERASURE_CODING") == "true" +IS_ENCRYPTED = os.getenv("USE_OZONE_ENCRYPTION") == "true" # This condition satisfies both the states where one can assume a default fs # - The environment variable is set to an empty string. # - Tne environment variables is unset ( None )
