This is an automated email from the ASF dual-hosted git repository. csringhofer pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/impala.git
commit e49ed3d2430aeb032cba10e6a14fdb78619666b6 Author: Csaba Ringhofer <[email protected]> AuthorDate: Mon Mar 3 23:04:20 2025 +0100 IMPALA-13790: Fix test_wildcard_san_ssl / test_wildcard_ssl These tests failed in various ways depending on OS/openssl version. An issue identified is that the certificates contained CN=* while wildcard subject should be like *.<domain>. Recreated wildcard certs with *.impala.test common name and added some host names that match them in bootstrap_system.sh. Removed the @xfail from the tests as my expectation is that they should work on all supported OS. Tested on - Ubuntu 20.04 / OpenSSL 1.1.1f - Ubuntu 22.04 / OpenSSL 3.0.2 - RHEL 7.9 / OpenSSL 1.0.2k - RHEL 8.6 / OpenSSL 1.1.1k - Rocky 9.2 / OpenSSL 3.2.2 Change-Id: Ieedf682d06bdb6f8f68a5f77e41175e895b77ca9 Reviewed-on: http://gerrit.cloudera.org:8080/22569 Reviewed-by: Riza Suminto <[email protected]> Tested-by: Impala Public Jenkins <[email protected]> --- be/src/testutil/certificates-info.txt | 17 +++++----- be/src/testutil/wildcard-cert.pem | 38 +++++++++++----------- be/src/testutil/wildcard-san-cert.pem | 38 +++++++++++----------- bin/bootstrap_system.sh | 11 ++++++- tests/custom_cluster/test_client_ssl.py | 56 ++++++++++++++++++++------------- 5 files changed, 91 insertions(+), 69 deletions(-) diff --git a/be/src/testutil/certificates-info.txt b/be/src/testutil/certificates-info.txt index 2e290c572..cdf27528a 100644 --- a/be/src/testutil/certificates-info.txt +++ b/be/src/testutil/certificates-info.txt @@ -4,7 +4,7 @@ responsible for and how they were created: 1) wildcardCA.pem & wildcardCA.key: This is a root certificate and its key which was used to sign wildcard-cert.pem and - wildcard-san-cert.pem. (Added as a part of IMPALA-3159) + wildcard-san-cert.pem. (Added as a part of IMPALA-3159). The common name is "*" This was created using the following commands: @@ -18,14 +18,14 @@ responsible for and how they were created: 2) wildcard-cert.pem & wildcard-cert.key: This is a wildcard certificate and its corresponding key which has its commonName as - "*". This means it should match with any host. (Added as a part of IMPALA-3159) + "*.impala.test". (Added as a part of IMPALA-3159, updated related to IMPALA-13790) This was created using the following commands: openssl genrsa -out wildcard-cert.key 2048 - openssl req -new -key wildcard-cert.key -out wildcard-cert.csr - (Fill in all the details according to prompts) + openssl req -new -key wildcard-cert.key -out wildcard-cert.csr \ + -subj "/C=US/ST=CA/L=SF/O=Cloudera/CN=*.impala.test" openssl x509 -req -in wildcard-cert.csr -CA wildcardCA.pem -CAkey wildcardCA.key \ -CAcreateserial -out wildcard-cert.pem -days 10000 -sha256 @@ -34,8 +34,9 @@ responsible for and how they were created: 3) wildcard-san-cert.pem & wildcard-san-cert.key: This is a certificate and its corresponding key which has 2 SANs - (subjectAlternativeName). One is "localhost" and the other is a wildcard ("*"). - (Added as a part of IMPALA-3159) + (subjectAlternativeName). One is "alsoBad" and the other is a wildcard + ("*.impala.test"). + (Added as a part of IMPALA-3159, updated related to IMPALA-13790) This was created using the following commands: @@ -43,12 +44,12 @@ responsible for and how they were created: openssl req -new -sha256 -key wildcard-san-cert.key \ -subj "/C=US/ST=CA/L=SF/O=Cloudera/CN=badCN" -reqexts SAN \ - -config <(cat /etc/ssl/openssl.cnf <(printf "[SAN]\nsubjectAltName=DNS:localhost,DNS:*")) \ + -config <(cat /etc/ssl/openssl.cnf <(printf "[SAN]\nsubjectAltName=DNS:alsoBad,DNS:*.impala.test")) \ -out wildcard-san-cert.csr openssl x509 -req -in wildcard-san-cert.csr -CA wildcardCA.pem \ -CAkey wildcardCA.key -CAcreateserial \ - -extfile <(cat /etc/ssl/openssl.cnf <(printf "[SAN]\nsubjectAltName=DNS:localhost,DNS:*")) \ + -extfile <(cat /etc/ssl/openssl.cnf <(printf "[SAN]\nsubjectAltName=DNS:alsoBad,DNS:*.impala.test")) \ -extensions SAN -out wildcard-san-cert.pem -days 10000 -sha256 ------------- diff --git a/be/src/testutil/wildcard-cert.pem b/be/src/testutil/wildcard-cert.pem index 9425ee329..13e6370bb 100644 --- a/be/src/testutil/wildcard-cert.pem +++ b/be/src/testutil/wildcard-cert.pem @@ -1,21 +1,21 @@ -----BEGIN CERTIFICATE----- -MIIDhDCCAmwCCQC/dhp+NzzLKzANBgkqhkiG9w0BAQsFADCBizELMAkGA1UEBhMC -VVMxCzAJBgNVBAgMAkNBMQswCQYDVQQHDAJTRjERMA8GA1UECgwIQ2xvdWRlcmEx -DzANBgNVBAsMBkltcGFsYTEZMBcGA1UEAwwQV2lsZGNhcmQgUm9vdCBDQTEjMCEG -CSqGSIb3DQEJARYUc2FpbGVzaEBjbG91ZGVyYS5jb20wHhcNMTYwNzExMjMwMDIx -WhcNNDMxMTI3MjMwMDIxWjB8MQswCQYDVQQGEwJVUzELMAkGA1UECAwCQ0ExCzAJ -BgNVBAcMAlNGMREwDwYDVQQKDAhDbG91ZGVyYTEPMA0GA1UECwwGSW1wYWxhMQow -CAYDVQQDDAEqMSMwIQYJKoZIhvcNAQkBFhRzYWlsZXNoQGNsb3VkZXJhLmNvbTCC -ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAPQbyUwU2Ac5yg7r8hs/qmZa -jlXAuJWw1atqY1JG6NPTQo/6xDlS99PlxzlIc81441B5JgRTCzEIx0INKSHQ8SM1 -e1Kl5eHeOlgtUNFuHyTV6efQ6WMyu2PeWgqeRzaIHGN8WXoTWz4KyL/G1mAqHwa0 -aHvcu+milrjNh8Si/vSntVn7R/KlL3rAHJTKcsKuVxDQgo3ZPmn9fVVuVcwk1ncp -q5A9sV5weAl7/TI6tCmBuHIsWDj8llz1aLvaHvWaBEmQEcljFJGiXc2BJNiyiLRp -sm7d03D1huWbq+KgrU0b9lBltXpabO99peHARgO5L3MV/1RLamk6dtjbN//YXpsC -AwEAATANBgkqhkiG9w0BAQsFAAOCAQEAfeA/dt2eIlLPvOTqowwXyS8Fm69G09wJ -eR3cPZELJONbIRs8AicxNyglj3d2QkwLc+kMPighEqjBaGjXFxEMnY/nncw+DTtN -eLKu+QtXtKMaVxigmXx7fkdITy3OHUiEJcwzNjTj68XycgvSol5QQ3GtjvECGQgG -6bGle+kHrkYRRMqnnLAzoRTeTvkbGHNyoszl5Ix6iPWzCBfP/Vo+Swa+BUlDTLZs -4I+c/ORT69vIbU8b8EI+3DA1hf+6m8Pf8gXGCSPJyB8guUBuUPdd6wUemjLCbvM9 -R6gTEr1XDEAdjPryDL3pE84AF2u4bkoPTYosM9YOO2ZGYW2/wHT/0A== +MIIDZzCCAk8CFAc6GyhEDE9ccWUpVVGbpOnJ4PwVMA0GCSqGSIb3DQEBCwUAMIGL +MQswCQYDVQQGEwJVUzELMAkGA1UECAwCQ0ExCzAJBgNVBAcMAlNGMREwDwYDVQQK +DAhDbG91ZGVyYTEPMA0GA1UECwwGSW1wYWxhMRkwFwYDVQQDDBBXaWxkY2FyZCBS +b290IENBMSMwIQYJKoZIhvcNAQkBFhRzYWlsZXNoQGNsb3VkZXJhLmNvbTAgFw0y +NTAzMDMyMTU3NDhaGA8yMDUyMDcxOTIxNTc0OFowUjELMAkGA1UEBhMCVVMxCzAJ +BgNVBAgMAkNBMQswCQYDVQQHDAJTRjERMA8GA1UECgwIQ2xvdWRlcmExFjAUBgNV +BAMMDSouaW1wYWxhLnRlc3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB +AQD0G8lMFNgHOcoO6/IbP6pmWo5VwLiVsNWramNSRujT00KP+sQ5UvfT5cc5SHPN +eONQeSYEUwsxCMdCDSkh0PEjNXtSpeXh3jpYLVDRbh8k1enn0OljMrtj3loKnkc2 +iBxjfFl6E1s+Csi/xtZgKh8GtGh73Lvpopa4zYfEov70p7VZ+0fypS96wByUynLC +rlcQ0IKN2T5p/X1VblXMJNZ3KauQPbFecHgJe/0yOrQpgbhyLFg4/JZc9Wi72h71 +mgRJkBHJYxSRol3NgSTYsoi0abJu3dNw9Yblm6vioK1NG/ZQZbV6WmzvfaXhwEYD +uS9zFf9US2ppOnbY2zf/2F6bAgMBAAEwDQYJKoZIhvcNAQELBQADggEBABLaSKUz +csm6xdy6j5LDnBQLK2lviUSrW1kEXD0du93hzrAfE7tO+pYx4ojKoJDBm6oqAhwr +Od4e5q4fn/cP4vSL+K655qGxtDuus22dGqQXnPszF6pQjK0N2znZKI9AOdJZndiI +afw45PGXafixhwg19qFlGk7NNtQaGE6DbzQqDFonb7x3MhFV/ATsNUREKRKh5JLt +f9v9cGhky2YKV2Ljw794ujSa569XcKsvC658qpYyyKA7iyov02eSsu53DAgw0X/z +OpPP8N885KJfSeRCvlmopipDS2JekYLC/KFd8bV8rNx1NkEfsINZOClyf86PkDnp +9gWUE+YlpMfLSOg= -----END CERTIFICATE----- diff --git a/be/src/testutil/wildcard-san-cert.pem b/be/src/testutil/wildcard-san-cert.pem index c64b34224..1a7e58831 100644 --- a/be/src/testutil/wildcard-san-cert.pem +++ b/be/src/testutil/wildcard-san-cert.pem @@ -1,21 +1,21 @@ -----BEGIN CERTIFICATE----- -MIIDdDCCAlygAwIBAgIJAKQejNj+hCJGMA0GCSqGSIb3DQEBCwUAMIGLMQswCQYD -VQQGEwJVUzELMAkGA1UECAwCQ0ExCzAJBgNVBAcMAlNGMREwDwYDVQQKDAhDbG91 -ZGVyYTEPMA0GA1UECwwGSW1wYWxhMRkwFwYDVQQDDBBXaWxkY2FyZCBSb290IENB -MSMwIQYJKoZIhvcNAQkBFhRzYWlsZXNoQGNsb3VkZXJhLmNvbTAeFw0xNjA3MTIw -MjQwNDdaFw00MzExMjgwMjQwNDdaMEoxCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJD -QTELMAkGA1UEBwwCU0YxETAPBgNVBAoMCENsb3VkZXJhMQ4wDAYDVQQDDAViYWRD -TjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAKnWmgGAG/cHdfXmW7oq -T3E88q+MDi9gBakvI22M2+eSKkW/xsgJutGgZy/dUfyxvlRUxrj1oXMVsr7K4sdJ -Su2IbGUuIZDjuYIFtmlJpJcQ7M07I0L1Y+ya+km/G15zqmJfcwPHhFbGkaKJKb3h -idISptmaMwRe9XqUsk0swMQRsd8EvLYY/jSWwLvd4FluZBzmuOuexyQJifTM/KxH -VpzZLGYUeg7XFjkBeDMaafuT84J4BCp/tf3JiJ8xbNg6HoStS5Irk1Z+X13jGeLW -MWSgJ/cM5PlNlbkRMnFaf2YkARq+94lOZpAEA87VlEGuFkZStQOgl/G+JmYGpsdN -DMkCAwEAAaMbMBkwFwYDVR0RBBAwDoIJbG9jYWxob3N0ggEqMA0GCSqGSIb3DQEB -CwUAA4IBAQCbKVWv0j0JmK6dCbmWlEdbjpyr0ABgGCggvHmzjJeSIA+stmyDp/JD -BNO8bQTydc6EWMhOS8+9egVTevbYO6Kv9u4up/ZJ/noaEz3UnNeKAW7qWdElwM5q -GRm29g0wQ+tz63KMzWLGLMngM3gH3Omy0xtJ9sOZgV0SWxppujaK9RNkJaikLiIw -4uQ2WbJTiDG9U/5itOoQIroXOQF4+RugWJgkfXcuzb3fsOh9LRIOczW097E6lmWh -QYnBeGyYUuRBvD28xTXMeAUnR3dRyUrSbzUmUI2XvLyeeEC5uvAgsvr60bvluzZ+ -msvMXVQ/eq5FNnL9eRVVNCKz+Z+zrfHt +MIIDizCCAnOgAwIBAgIUBzobKEQMT1xxZSlVUZuk6cng/BQwDQYJKoZIhvcNAQEL +BQAwgYsxCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJDQTELMAkGA1UEBwwCU0YxETAP +BgNVBAoMCENsb3VkZXJhMQ8wDQYDVQQLDAZJbXBhbGExGTAXBgNVBAMMEFdpbGRj +YXJkIFJvb3QgQ0ExIzAhBgkqhkiG9w0BCQEWFHNhaWxlc2hAY2xvdWRlcmEuY29t +MCAXDTI1MDMwMzIxNTQzOFoYDzIwNTIwNzE5MjE1NDM4WjBKMQswCQYDVQQGEwJV +UzELMAkGA1UECAwCQ0ExCzAJBgNVBAcMAlNGMREwDwYDVQQKDAhDbG91ZGVyYTEO +MAwGA1UEAwwFYmFkQ04wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCp +1poBgBv3B3X15lu6Kk9xPPKvjA4vYAWpLyNtjNvnkipFv8bICbrRoGcv3VH8sb5U +VMa49aFzFbK+yuLHSUrtiGxlLiGQ47mCBbZpSaSXEOzNOyNC9WPsmvpJvxtec6pi +X3MDx4RWxpGiiSm94YnSEqbZmjMEXvV6lLJNLMDEEbHfBLy2GP40lsC73eBZbmQc +5rjrnsckCYn0zPysR1ac2SxmFHoO1xY5AXgzGmn7k/OCeAQqf7X9yYifMWzYOh6E +rUuSK5NWfl9d4xni1jFkoCf3DOT5TZW5ETJxWn9mJAEavveJTmaQBAPO1ZRBrhZG +UrUDoJfxviZmBqbHTQzJAgMBAAGjJTAjMCEGA1UdEQQaMBiCB2Fsc29CYWSCDSou +aW1wYWxhLnRlc3QwDQYJKoZIhvcNAQELBQADggEBAGISTBD0ucpRWWiPOfQ8q94G +SFuSDP8Lbi/ckwOXGMRp+znBv8aECe40MezVFi7/E/rZPisgVL29sQY0ytFRzsnI +BUz9rZjAHpZEnqCrIghoxmuEl3Ljo5K1wXZt/2DWUGhzrvFFQ87VerKb7P7vJ24p +uuVrqrkEvEoMAL6jeeZn90v4USjL8mKw5DAciLLPoVF7Na1fdtjXr2SrdLNryVbj +r5gdvicTHfTrzWBaKT61sRn9Ga0riiDb3QfUwsUnNrWSR3J7606pZo/8g8mwV9wy +24gkWjZzPiInS30c7rmFoMNpcdDX8A/Jc507bwR/4YHiFErkSxn8o+z1dLAFSso= -----END CERTIFICATE----- diff --git a/bin/bootstrap_system.sh b/bin/bootstrap_system.sh index 2661138f8..9a9996868 100755 --- a/bin/bootstrap_system.sh +++ b/bin/bootstrap_system.sh @@ -480,7 +480,16 @@ ssh localhost whoami # ... # ...ConnectionError: ('Connection aborted.', error(111, 'Connection refused')) # Prefer the FQDN first for rpc-mgr-kerberized-test as newer krb5 requires FQDN. -echo -e "\n127.0.0.1 $(hostname) $(hostname -s)" | sudo tee -a /etc/hosts +add_if_not_there() { + grep -q "$2" $1 || echo "$2" | sudo tee -a $1 +} +add_if_not_there "/etc/hosts" "127.0.0.1 $(hostname) $(hostname -s)" + +# Add hostnames with multiple labels to allow matching wildcard TLS certificates. +# Create names that map to v4/v6/dual localhost to help ipv6 testing. +add_if_not_there "/etc/hosts" "127.0.0.1 ip4.impala.test ip46.impala.test" +add_if_not_there "/etc/hosts" "::1 ip6.impala.test ip46.impala.test" + # # In Docker, one can change /etc/hosts as above but not with sed -i. The error message is # "sed: cannot rename /etc/sedc3gPj8: Device or resource busy". The following lines are diff --git a/tests/custom_cluster/test_client_ssl.py b/tests/custom_cluster/test_client_ssl.py index 51f3b2afe..dd2b05e93 100644 --- a/tests/custom_cluster/test_client_ssl.py +++ b/tests/custom_cluster/test_client_ssl.py @@ -65,21 +65,27 @@ class TestClientSsl(CustomClusterTestSuite): # Deprecation warnings should not be seen. DEPRECATION_WARNING = "DeprecationWarning" - SSL_WILDCARD_ARGS = ("--ssl_client_ca_certificate=%s/wildcardCA.pem " - "--ssl_server_certificate=%s/wildcard-cert.pem " - "--ssl_private_key=%s/wildcard-cert.key" - % (CERT_DIR, CERT_DIR, CERT_DIR)) - - SSL_WILDCARD_SAN_ARGS = ("--ssl_client_ca_certificate=%s/wildcardCA.pem " - "--ssl_server_certificate=%s/wildcard-san-cert.pem " - "--ssl_private_key=%s/wildcard-san-cert.key" - % (CERT_DIR, CERT_DIR, CERT_DIR)) - - SSL_ARGS = ("--ssl_client_ca_certificate=%s/server-cert.pem " - "--ssl_server_certificate=%s/server-cert.pem " - "--ssl_private_key=%s/server-key.pem " + SSL_WILDCARD_ARGS = ("--ssl_client_ca_certificate={0}/wildcardCA.pem " + "--ssl_server_certificate={0}/wildcard-cert.pem " + "--ssl_private_key={0}/wildcard-cert.key " + "--hostname={1} " + "--state_store_host={1} " + "--catalog_service_host={1} " + ).format(CERT_DIR, "ip4.impala.test") + + SSL_WILDCARD_SAN_ARGS = ("--ssl_client_ca_certificate={0}/wildcardCA.pem " + "--ssl_server_certificate={0}/wildcard-san-cert.pem " + "--ssl_private_key={0}/wildcard-san-cert.key " + "--hostname={1} " + "--state_store_host={1} " + "--catalog_service_host={1} " + ).format(CERT_DIR, "ip4.impala.test") + + SSL_ARGS = ("--ssl_client_ca_certificate={0}/server-cert.pem " + "--ssl_server_certificate={0}/server-cert.pem " + "--ssl_private_key={0}/server-key.pem " "--hostname=localhost " # Required to match hostname in certificate - % (CERT_DIR, CERT_DIR, CERT_DIR)) + ).format(CERT_DIR) @classmethod def setup_class(cls): @@ -202,21 +208,20 @@ class TestClientSsl(CustomClusterTestSuite): statestored_args=SSL_WILDCARD_ARGS, catalogd_args=SSL_WILDCARD_ARGS) @pytest.mark.skipif(SKIP_SSL_MSG is not None, reason=SKIP_SSL_MSG) - @pytest.mark.xfail(run=True, reason="Inconsistent wildcard support on target platforms") def test_wildcard_ssl(self, vector): """ Test for IMPALA-3159: Test with a certificate which has a wildcard for the CommonName. """ - self._verify_negative_cases(vector) + self._verify_negative_cases(vector, host="ip4.impala.test") - self._validate_positive_cases(vector, "%s/wildcardCA.pem" % CERT_DIR) + self._validate_positive_cases(vector, "%s/wildcardCA.pem" % CERT_DIR, + host="ip4.impala.test") @pytest.mark.execute_serially @CustomClusterTestSuite.with_args(impalad_args=SSL_WILDCARD_SAN_ARGS, statestored_args=SSL_WILDCARD_SAN_ARGS, catalogd_args=SSL_WILDCARD_SAN_ARGS) @pytest.mark.skipif(SKIP_SSL_MSG is not None, reason=SKIP_SSL_MSG) - @pytest.mark.xfail(run=True, reason="Inconsistent wildcard support on target platforms") def test_wildcard_san_ssl(self, vector): """ Test for IMPALA-3159: Test with a certificate which has a wildcard as a SAN. """ @@ -229,24 +234,31 @@ class TestClientSsl(CustomClusterTestSuite): "cannot retrieve SAN from certificate: " "https://bugzilla.redhat.com/show_bug.cgi?id=928390";) - self._verify_negative_cases(vector) + self._verify_negative_cases(vector, host="ip4.impala.test") - self._validate_positive_cases(vector, "%s/wildcardCA.pem" % CERT_DIR) + self._validate_positive_cases(vector, "%s/wildcardCA.pem" % CERT_DIR, + host="ip4.impala.test") - def _verify_negative_cases(self, vector): + def _verify_negative_cases(self, vector, host=""): # Expect the shell to not start successfully if we point --ca_cert to an incorrect # certificate. args = ["--ssl", "-q", "select 1 + 2", "--ca_cert=%s/incorrect-commonname-cert.pem" % CERT_DIR] + if host: + args.extend(["-i", host]) run_impala_shell_cmd(vector, args, expect_success=False) # Expect the shell to not start successfully if we don't specify the --ssl option args = ["-q", "select 1 + 2"] + if host: + args.extend(["-i", host]) run_impala_shell_cmd(vector, args, expect_success=False) - def _validate_positive_cases(self, vector, ca_cert=""): + def _validate_positive_cases(self, vector, ca_cert="", host=None): python3_10_version_re = re.compile(r"using Python 3\.1[0-9]") shell_options = ["--ssl", "-q", "select 1 + 2"] + if host: + shell_options.extend(["-i", host]) result = run_impala_shell_cmd(vector, shell_options) for msg in [self.SSL_ENABLED, self.CONNECTED, self.FETCHED]: assert msg in result.stderr
