This is an automated email from the ASF dual-hosted git repository. stigahuang pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/impala.git
commit 1644cf629bd2ca47f54e77756a266e8f193777be Author: Michael Smith <[email protected]> AuthorDate: Tue Aug 30 15:26:57 2022 -0700 IMPALA-9448: Use Ozone TDE in minicluster Enables Ozone's Transparent Data Encryption when creating a bucket in the minicluster. Requires `-format` to turn on encryption in an existing minicluster. Tested by running e2e and custom cluster tests with Ozone. Change-Id: Icd82967cdab7b222e5876936f39e5198ba9773ea Reviewed-on: http://gerrit.cloudera.org:8080/18931 Reviewed-by: Impala Public Jenkins <[email protected]> Tested-by: Impala Public Jenkins <[email protected]> --- bin/impala-config.sh | 1 + testdata/bin/run-all.sh | 1 - testdata/bin/setup-dfs-keys.sh | 30 ++++++++++++++++++++++++++++++ testdata/bin/setup-hdfs-env.sh | 11 ++--------- testdata/cluster/admin | 15 ++++++++++++--- 5 files changed, 45 insertions(+), 13 deletions(-) diff --git a/bin/impala-config.sh b/bin/impala-config.sh index 736f3fc8c..4aec2d408 100755 --- a/bin/impala-config.sh +++ b/bin/impala-config.sh @@ -657,6 +657,7 @@ elif [ "${TARGET_FILESYSTEM}" = "hdfs" ]; then export HDFS_ERASURECODE_PATH="/test-warehouse" fi elif [ "${TARGET_FILESYSTEM}" = "ozone" ]; then + export USE_OZONE_ENCRYPTION=${USE_OZONE_ENCRYPTION-true} export OZONE_VOLUME="impala" export OZONE_BUCKET="base" export DEFAULT_FS="o3fs://${OZONE_BUCKET}.${OZONE_VOLUME}.${INTERNAL_LISTEN_HOST}:9862" diff --git a/testdata/bin/run-all.sh b/testdata/bin/run-all.sh index 801552144..dda171c23 100755 --- a/testdata/bin/run-all.sh +++ b/testdata/bin/run-all.sh @@ -81,7 +81,6 @@ else # - We don't yet have a good way to start YARN using a different defaultFS. Moreoever # we currently don't run hive queries against Isilon for testing. # - LLAMA is avoided because we cannot start YARN. - # - KMS is used for encryption testing, which is not available on remote storage. # - Hive needs YARN, and we don't run Hive queries. # Impala can also run on a local file system without additional services. # TODO: Figure out how to start YARN, LLAMA and Hive with a different defaultFs. diff --git a/testdata/bin/setup-dfs-keys.sh b/testdata/bin/setup-dfs-keys.sh new file mode 100755 index 000000000..aeb55f8a5 --- /dev/null +++ b/testdata/bin/setup-dfs-keys.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +set -euo pipefail +. $IMPALA_HOME/bin/report_build_error.sh +setup_report_build_error + +# Create encryption keys for HDFS encryption tests. Keys are stored by the KMS. +EXISTING_KEYS=$(hadoop key list) +for KEY in $*; do + if ! grep $KEY <<< $EXISTING_KEYS &>/dev/null; then + hadoop key create $KEY -f + fi +done diff --git a/testdata/bin/setup-hdfs-env.sh b/testdata/bin/setup-hdfs-env.sh index 51ab42ea6..2b81a49e3 100755 --- a/testdata/bin/setup-hdfs-env.sh +++ b/testdata/bin/setup-hdfs-env.sh @@ -46,15 +46,8 @@ fi # against a remote cluster, rather than the local mini-cluster (i.e., when REMOTE_LOAD # is true. See: IMPALA-4344) -if [[ $TARGET_FILESYSTEM == hdfs && -z "$REMOTE_LOAD" ]]; then # Otherwise assume KMS isn't setup. - # Create encryption keys for HDFS encryption tests. Keys are stored by the KMS. - EXISTING_KEYS=$(hadoop key list) - for KEY in testkey{1,2}; do - if grep $KEY <<< $EXISTING_KEYS &>/dev/null; then - hadoop key delete $KEY -f - fi - hadoop key create $KEY - done +if [[ -z "$REMOTE_LOAD" ]]; then # Otherwise assume KMS isn't setup. + ${IMPALA_HOME}/testdata/bin/setup-dfs-keys.sh testkey{1,2} fi if [[ -n "${REMOTE_LOAD:-}" ]]; then diff --git a/testdata/cluster/admin b/testdata/cluster/admin index 4bce41712..a8ed6ef16 100755 --- a/testdata/cluster/admin +++ b/testdata/cluster/admin @@ -57,7 +57,7 @@ if [[ "$TARGET_FILESYSTEM" == "hdfs" ]]; then # The check above indicates that the regular mini-cluster is in use. SUPPORTED_SERVICES=(hdfs kms yarn) elif [[ "$TARGET_FILESYSTEM" == "ozone" ]]; then - SUPPORTED_SERVICES=(ozone) + SUPPORTED_SERVICES=(kms ozone) else # Either a remote distributed file system or a local non-distributed file system is # in use. Currently the only service that is expected to work is Kudu, though in theory @@ -327,8 +327,17 @@ function start_cluster { fi if [[ "${TARGET_FILESYSTEM}" = "ozone" ]]; then - ozone sh volume create /${OZONE_VOLUME} || true - ozone sh bucket create /${OZONE_VOLUME}/${OZONE_BUCKET} || true + local bucketkey='' + if $USE_OZONE_ENCRYPTION; then + echo "Ozone encryption enabled for ${OZONE_VOLUME}/${OZONE_BUCKET}" + + # Encryption is done at the bucket level, so ensure the keys are available first. + ${IMPALA_HOME}/testdata/bin/setup-dfs-keys.sh testkey + bucketkey='--bucketkey testkey' + fi + + ozone sh volume create ${OZONE_VOLUME} || true + ozone sh bucket create ${bucketkey} ${OZONE_VOLUME}/${OZONE_BUCKET} || true fi return $?
