This is an automated email from the ASF dual-hosted git repository. joemcdonnell pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/impala.git
commit 4e2189391c6cefa118d9f7b915eabd19f4a38189 Author: Joe McDonnell <[email protected]> AuthorDate: Thu Feb 12 14:39:01 2026 -0800 IMPALA-14770: Upgrade impala-python3 to 3.11.14 This upgrades the impala-python3 virtualenv to use Python 3.11.14. Upgrading to Python 3.11 allows us to use the most recent version of most packages. Python 3.11 delivered performance improvements over previous versions. It seems to reduce the runtime of precommit by about 15-20 minutes (~4:40 -> ~4:25). This fixes code using removed library functions: - XML's getiterator() was removed and code needs to use iter() - thread's isAlive() was removed and code needs to use is_alive() This fixes some pieces of code to avoid warnings: - The "pipes" package is deprecated, so this changes code using pipes.quote() to use shlex.quote() instead. - This changes some regex patterns to use raw strings to avoid warnings about invalid escape sequences. This upgrades a variety of packages (some of them required, many of them optional): 1. Upgrades setuptools to a version from about a month ago 2. Upgrades requests to the latest version - Addresses some warnings that show for pytest - Requests got stricter about certificates and server-cert.pem is no longer considered valid. - This regenerates the server-key.pem/server-key-password.pem/server-cert.pem to add a subject alternative name of localhost. 3. Upgrades python-magic to the latest and fixes dump_breakpad_symbols.py 4. Upgrades psutil to the latest and fixes tests/common/impala_cluster.py to handle the changes in how it represents zombie processes 5. Upgrades several other Python packages to more recent versions without needing code changes (allpairspy, flake8, hdfs, kazoo, prometheus-client, six, sasl, avro, distro, pytz, Cython). Removes unused pg8000 package. There were issues upgrading some other packages, so this does not upgrade them: - Newer versions of sqlparse have a regression in split.split() that makes it inoperable for us. - Newer versions of pexpect would require adjustment in the interactive shell tests. - New versions of pytest are not difficult but require additional changes that would be better suited to a separate commit Testing: - Ran precommit - Ran exhaustive release jobs Change-Id: I9383ebfb310a4ee30134e02b195c332ffbb6a7ec Reviewed-on: http://gerrit.cloudera.org:8080/23976 Tested-by: Impala Public Jenkins <[email protected]> Reviewed-by: Michael Smith <[email protected]> --- be/src/testutil/certificates-info.txt | 19 +++++-- be/src/testutil/server-cert.pem | 46 ++++++++-------- be/src/testutil/server-key-password.pem | 60 ++++++++++----------- be/src/testutil/server-key.pem | 55 +++++++++---------- bin/dump_breakpad_symbols.py | 2 +- bin/impala-config.sh | 2 +- infra/python/deps/requirements.txt | 64 ++++++++++++----------- infra/python/deps/setuptools-requirements.txt | 8 +-- tests/common/custom_cluster_test_suite.py | 4 +- tests/common/impala_cluster.py | 17 ++++-- tests/common/resource_pool_config.py | 2 +- tests/common/test_result_verifier.py | 8 +-- tests/comparison/cluster.py | 2 +- tests/comparison/leopard/controller.py | 2 +- tests/custom_cluster/test_admission_controller.py | 2 +- tests/stress/query_retries_stress_runner.py | 4 +- tests/util/auto_scaler.py | 4 +- 17 files changed, 164 insertions(+), 137 deletions(-) diff --git a/be/src/testutil/certificates-info.txt b/be/src/testutil/certificates-info.txt index cdf27528a..72ae886da 100644 --- a/be/src/testutil/certificates-info.txt +++ b/be/src/testutil/certificates-info.txt @@ -53,12 +53,25 @@ responsible for and how they were created: -extensions SAN -out wildcard-san-cert.pem -days 10000 -sha256 ------------- - 4) server-cert.pem & server-cert.key: + 4) server-key.pem & server-key-password.pem & server-cert.key: This is a self-signed certificate and its corresponding key which has the commonName - as "localhost". + as "localhost" and subject alternative name "DNS:localhost,IP:192.168.1.1". It also + specifies CA:FALSE to be clear that this is a leaf certificate. - This was created the same as 1) with a different commonName. + This was created using the following commands: + + openssl genrsa -out server-key.pem 2048 + + openssl req -x509 -new -nodes -key server-key.pem -sha256 -days 10000 -out server-cert.pem \ + -addext 'basicConstraints=CA:FALSE' \ + -addext 'subjectAltName=DNS:localhost,IP:192.168.1.1' + + (Fill in all the details according to prompts with common name set to 'localhost') + + openssl rsa -aes256 -in server-key.pem -out server-key-password.pem + + (Specify "password" for the pass phrase) ------------- 5) incorrect-commonname-cert.pem & incorrect-commonname-cert.key: diff --git a/be/src/testutil/server-cert.pem b/be/src/testutil/server-cert.pem index 08ff98d5e..f8dd0a895 100644 --- a/be/src/testutil/server-cert.pem +++ b/be/src/testutil/server-cert.pem @@ -1,26 +1,24 @@ -----BEGIN CERTIFICATE----- -MIIEejCCA2KgAwIBAgIJAKMdvDR5PL82MA0GCSqGSIb3DQEBBQUAMIGEMQswCQYD -VQQGEwJVUzETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UEBxMNU2FuIEZyYW5j -aXNjbzERMA8GA1UEChMIQ2xvdWRlcmExEjAQBgNVBAMTCWxvY2FsaG9zdDEhMB8G -CSqGSIb3DQEJARYSaGVucnlAY2xvdWRlcmEuY29tMB4XDTEzMDkyMjAwMjUxOFoX -DTQxMDIwNzAwMjUxOFowgYQxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpDYWxpZm9y -bmlhMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMREwDwYDVQQKEwhDbG91ZGVyYTES -MBAGA1UEAxMJbG9jYWxob3N0MSEwHwYJKoZIhvcNAQkBFhJoZW5yeUBjbG91ZGVy -YS5jb20wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCoUj3pMQ2ELkrz -zq+koixljVFBAEEqwUWSjA+GEKwfFb/UPRjeO/wrKndp2r83jc6KRt66rvAIl8cr -b54yTOsJ/ZcARrjTwG3IG8Tely/54ZQyH0ImdJyEbCSoI04zX3ovjlppz3g5xanj -WmpAh6pzPgBOTfisCLMPD70xQ8F//QWZdNatoly54STkTWoJv/Oll/UpXcBY8JOR -+ytX82eGgG4F8YoQqmbjrrN5JAmqLRiBAkr3WUena6ekqJBalJRzex/Wh8a9XEV7 -9HFVVngBhezsOJgf81hzBzzhULKfxuXl8uaUj3Z9cZg39CDsyz+ULYbsPm8VoMUI -VCf7MUVTAgMBAAGjgewwgekwHQYDVR0OBBYEFK94kea7jIKQawAIb+0DqsA1rf6n -MIG5BgNVHSMEgbEwga6AFK94kea7jIKQawAIb+0DqsA1rf6noYGKpIGHMIGEMQsw -CQYDVQQGEwJVUzETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UEBxMNU2FuIEZy -YW5jaXNjbzERMA8GA1UEChMIQ2xvdWRlcmExEjAQBgNVBAMTCWxvY2FsaG9zdDEh -MB8GCSqGSIb3DQEJARYSaGVucnlAY2xvdWRlcmEuY29tggkAox28NHk8vzYwDAYD -VR0TBAUwAwEB/zANBgkqhkiG9w0BAQUFAAOCAQEAEtkPPncCnN2IFVJvz04K+VsX -b6w3qwPynQKc67+++JkNb3TYKrh/0UVM1NrEOu3TGplqOrKgAlITuaWNqNOSBu1R -WJtrz85YkonED5awjjuALVEY82+c7pOXkuv5G5421RINfRn2hNzgw8VFb5CEvxHH -jER80Vx6UGKr/S649qTQ8AzVzTwWS86VsGI2azAD7D67G/IDGf+0P7FsXonKY+vl -vKzkfaO1+qEOLtDHV9mwlsxl3Re/MNym4ExWHi9txynCNiRZHqWoZUS+KyYqIR2q -seCrQwgi1Fer9Ekd5XNjWjigC3VC3SjMqWaxeKbZ2/AuABJMz5xSiRkgwphXEQ== +MIIEDTCCAvWgAwIBAgIUN9l41BNIHYdX2T4eYOWDMHFKaCAwDQYJKoZIhvcNAQEL +BQAwgYkxCzAJBgNVBAYTAlVTMRMwEQYDVQQIDApDYWxpZm9ybmlhMRQwEgYDVQQH +DAtTYW50YSBDbGFyYTERMA8GA1UECgwIQ2xvdWRlcmExEjAQBgNVBAMMCWxvY2Fs +aG9zdDEoMCYGCSqGSIb3DQEJARYZam9lbWNkb25uZWxsQGNsb3VkZXJhLmNvbTAg +Fw0yNjAyMjIyMTE3MjNaGA8yMDUzMDcxMDIxMTcyM1owgYkxCzAJBgNVBAYTAlVT +MRMwEQYDVQQIDApDYWxpZm9ybmlhMRQwEgYDVQQHDAtTYW50YSBDbGFyYTERMA8G +A1UECgwIQ2xvdWRlcmExEjAQBgNVBAMMCWxvY2FsaG9zdDEoMCYGCSqGSIb3DQEJ +ARYZam9lbWNkb25uZWxsQGNsb3VkZXJhLmNvbTCCASIwDQYJKoZIhvcNAQEBBQAD +ggEPADCCAQoCggEBANzvliRecMz/AMwIqK8e5xDh4Hv8eGKp5/zdKVkV7KU+ecGA +P68gnEmbX50pPGvYJEA7IkzdkRqR3CspUT4bO/Cx9hPfJlD/AP6p1Osezmqi1eO3 +Dw9VlNle3sZzXWXR2Jzqud6JxxEZhTRz6QbHLwBVhgOS7FOEUhVZfOu2yvmHUswW +sTNmUxLkiCMofhplPtFpfi3e+dE15bRLXjEOWOm4kPAKNxhS3PI0rxtKI7xDhvvU +fkuoFBJ0qulT/fzsR+wawfb+kebLEFV5K2KvEpfs4lv1wiH5VPB8QjIYNgJYak9w +HqHQyN9nkbsTUD+1txLvi8qk8taN19xJ/ocFPyMCAwEAAaNpMGcwHQYDVR0OBBYE +FGHsxHRxSDnqfXMCbp68lkW7yLNZMB8GA1UdIwQYMBaAFGHsxHRxSDnqfXMCbp68 +lkW7yLNZMAkGA1UdEwQCMAAwGgYDVR0RBBMwEYIJbG9jYWxob3N0hwTAqAEBMA0G +CSqGSIb3DQEBCwUAA4IBAQBKcf2AqM2jiOFA2YqeRSnYr9/rCgcBQN9XlU7GF8E1 +GzRtbpIwhUzK8D5yPKDlk1j+nQzeK7lrM1KqTb/lNNqnXQCcnSmZQ0aTeYquN7ig +etvlujskHDoa38QDVeQk1TmGeB3Dxm/uOFIZiMt3ko1VmuHWe8VQRnIaZL8xeTlH +cIsii4hVcy9dlwAh8weerKqBlnSZOFF0FoVy7bwVAdhFPLSZTYzeodNGSNo3k+Z7 +SNw3NHaotimL6oP16uN1HyMl5qSQFtoS0RIYQp9Hh8rYRlsqrugdiOSKr0fNlK+v +PLNC2U5MkB6IPoUIwEW6bv6cPYu+iD8Y7Rr6SeUGTtvN -----END CERTIFICATE----- diff --git a/be/src/testutil/server-key-password.pem b/be/src/testutil/server-key-password.pem index bb2a434f3..ff940e15d 100644 --- a/be/src/testutil/server-key-password.pem +++ b/be/src/testutil/server-key-password.pem @@ -1,30 +1,30 @@ ------BEGIN RSA PRIVATE KEY----- -Proc-Type: 4,ENCRYPTED -DEK-Info: DES-EDE3-CBC,34270C6CD1623E14 - -j2xGHUZ/lPkLPPAgnvdMcpkpHstFAIIyDw8KQHD3y4+J9B9ZX24XP34wEdit8eDF -Tm/ODX5eccn7t9gjkRU+rsF7L1JT+OoLkGwSI8KdUlVhncFUCGYWspSeVYLUl5Qa -ucbutzo0ael+zvOBXkc4ns/tvsIk3NYVz5qgcn1iw/87GN0Nqd1U2dwmClv8TP5m -4o641PEv7ozydD4+/yt4qc++UEgbZCNUfJYTG1dh2vnstCJgcGysS6f2Z6XiXLDW -PCpVfG81cDWh/pRJq7oa4neQXf/NP18OMBoUtJO7BPPjZtfuD/SbSuhaIGan16hT -RuVCl1ycJ2lZe428xTz2qbvBFli1RRMLtiV/qp8L8Db4hTeUWS4Sa3GonebYqJws -pe/r5/rr7Ph6YcK9nUCWfhAljwZ8aMSFO0LFKSYWgFHTlcnfEGucljJplsl7YzhV -bRkP7VL9p6ewIB6FuiIzcc64XSdflXal5PUkDUdIBig37T0GIHrqZPuWbJFbhiYk -//8c9W2J8Vl4o1kUHhwsxaXbWIUmmFN2im6zMOCVDt5TAkBeAfbr9IiUxlNg5Yb0 -QGRoi8ykGJL3TFBtHVE5hn0OVW41eNDPKmMdlsjGYfU5ZaX2Qefg6uOrll/wI6nM -2vvc2VwPtcRbYh8lK+oGShPSaCUqK650UXkcmYAQgCXR+UpJEl5Yp+jAq7UsvkyM -43vBx0mfvRH2Q1m3TTJ2ka32lFjzaUd+jsy1Mub/CcIsfJhSMppghFJ/W8SeEOg/ -viwyB7JoGD8ZhwEGL0asxF8PV2CFKy5AsbhKtmXmrMaPPomdarpheSX36CmH0gcv -K/nfsm1meWIBEKcTh05yedc4XuQLvKwHrrCLqlcrwmvaNAtRSt1ZrHb/rYfRmPX5 -hasseOY64SqKCSj2rx6ssuElMFk+xAeC7rJgrR1E3lwW881GoFGWYwMlbdFqEtyO -2SN/YKtzLRzzPJ5K36PzS7OhJG+UOMCVYBWvNQ51BpQpHp25ZxUgGHqtKeO55FAI -HDfjObOyqQPavVxqlis6TGLsWmD9OrXM3dqTRia4AmXjhJ/qVsvFYOlMpSgk8Q9i -Wx7dvVwqH4+/SrrasXkzPgKcexueyMf2iK18lesr6jM5er7ZWxvMfeKaIaS76Grs -STt1Ykabd0eJ0tJVKF2lO9KYmwSWqhq+PVOWGZmHHeJ8jMXvJmxU+cymNMiLhqWH -E0skEnM4xaaU1vHGzh8Clcb3BGbh2gi6XoaACxe6HgSHeGeglcBQiO9IULkUS12w -DDudhZ8/7nyYVPW5916aavQ8gxLo4VDhHDMFf7Jd7xV9eHhWulExwB7B33u4G4Ea -niV+Gj+n1WFHaylPKLLZV6H34zGzkG0adnr3KMT81TuknErv2rb41/tAeJPT2YG9 -fINyGX8+S7TPsRdH1a76qa9VGO5lKtDjZgTzBTXuj4PbT8OYwOO52OjSbCYK70Km -4xoe8Ylyte5OYdhNbYSMoV+pEO2xJ7WyjQzHhosbcXxDFmXveLy6y9H2qNcgGRYn -GFz+KH2108J0ENStYrPwkirejBlqSStsjtyWgDbyO/X8yJI0kRRik58lmUZ8A3ip ------END RSA PRIVATE KEY----- +-----BEGIN ENCRYPTED PRIVATE KEY----- +MIIFLTBXBgkqhkiG9w0BBQ0wSjApBgkqhkiG9w0BBQwwHAQIgciu2QPrOLACAggA +MAwGCCqGSIb3DQIJBQAwHQYJYIZIAWUDBAEqBBCybw7STzZjpGVYgCJ9t9r7BIIE +0DjMs2x4QlJYLLMdBmxLWXxJaZ+0pKvAJLg5w5tBI66/gNsheJR7HlsjdBtdUeV0 +Df2VSLfiW05ZWKCclZ8K3Q5GyAxbaLwrPIP28k7zDUSSUx9a4XHNozug3AzKHOWP +Pxq7H8T+xNu+fuJqUevUrj3xM4EakfAIyiMdUU3rCJPlsiS3b9VLGze1XUiM2584 +8nyhiyICGN9w31ZJeFhuOmiYOW02rAmL44Z+kVijshPoa4i7QNDkdFFXiXebrCZZ +sT1vJ5/zdINUVbWTbofNjhbB3xrkRkgdQHFJm6a7qHiEe5R+XL7ztziHLFG2OeKb +fA8RVNDPInZwZX3qYzfcPy/PWiT15Rsib/oHO+Xl/X0Bchf4QnnYUj2wJ55dW3VF +stVqrskxiB6+zo4eDIU2cKDsS1XwJLzHqDzsSvMFMr/8dtCrm6/L0pVA2QSdHbD2 +OHVs7CHTTxZVkTJ4Q2+ZWIQHQy1TAv8UXmSfgQLGKjaE5L+4Kd9QVV3V8kr2u27W +klxb41stpaHTdDureHQMQ8zoi8WPq6cGTRtAf1d7fvv0ONPedubieDGX3y4F/r1z +JQHAULqA2HHUFn4zSCsGEbcsiPc32HDtmO4uaN5bHOvgF3vwVAFAKEEWFOFQW8Zj +qESJ6MByIbcLW3mfLeg9mBzSV/LZmobG0l1GzBDhoot6P30nScBjnXQISZcN5/s0 +YnWJqVpQASQ6a3MojVVcFWlO6lGyXtbfICMrwgAnScBm4veG7rSJl1QFikQzOks2 +vtAsP29M5Rozb2XRQb7WFuDhc12Dsol+2vBXah7eAvUxgEeYptQT0yp3pc5l8BP9 +ohuGQcBZfDnNIYo7ScHjKE79Gxlb1+BT8cbh3wsEj5PTeFU87EGkZ5zcHIWDOMOv +r3U9RCixIs4Vo0bMU6t2GEt9TwlpommGTAqHe8TGF6PVQOn+lh4e7Rq+I44Zz1ix +TwL3eceMAp+ANpCpV0OhVmVh4YyhK6/irOYd72fbZIgeTH5VM/2Bq1Gbbex0G46q +J0W+rq1QcZOyX8JTHffmxkZykyMXkCvP6HIvh//qCCt8N81o0sR5aO9kgh/TeLOY +Ra+gmExiHFXGbsSBKgFzivL9Su3/GbMOhsyDWf+RWSXlYbBdEhQfSgn46I8MZ9mX +WGdzSZuAhdMBybLMoqNfz6HDzeBiof6XO7K+RGa20Rul95sOVlOx2OpPo1UnbhJH +MWuqvfzbEHoXpeB6/MJ8yg6x5uE+Xwm3g/49jGu6I5AgExpzTQ6fnxFfehQRbjrp +4yy3bnOe2KpgZk4jkSSXVOeMK+EjKKpH+m78RHWWlNliSWn2yEhs0+2+Qs/NKBIb +ZtkYsLs1n+ALoQ7f+28mJ3Spc+nKQUR3uSvG89kj/uLXIp1+/g4NkYkew/whVREe +MyCk6dm5UKTgVV/Sd699b1r5sDGwH9FhceEDaTZSGdQmpJy/cD5MqkVuTlCdan65 ++xyUTzYQTBKPPL+ijGiZ/o0wYkwiCp7ivlq2JOMMzZ+Pkp6DmLFi+omOjF4wgHf9 +j/RWeb2SOOZ5wD6FYHDXJGzaDdHMc9nye3Yh6lkFrRPA2RSrs7/HXxdQP0tGM+SI +N8Dz3ili+KqlFk6l1S5w78w+z+OklCEJnUvoPefhld7o +-----END ENCRYPTED PRIVATE KEY----- diff --git a/be/src/testutil/server-key.pem b/be/src/testutil/server-key.pem index 6012de7d3..657a742b4 100644 --- a/be/src/testutil/server-key.pem +++ b/be/src/testutil/server-key.pem @@ -1,27 +1,28 @@ ------BEGIN RSA PRIVATE KEY----- -MIIEpAIBAAKCAQEAqFI96TENhC5K886vpKIsZY1RQQBBKsFFkowPhhCsHxW/1D0Y -3jv8Kyp3adq/N43Oikbeuq7wCJfHK2+eMkzrCf2XAEa408BtyBvE3pcv+eGUMh9C -JnSchGwkqCNOM196L45aac94OcWp41pqQIeqcz4ATk34rAizDw+9MUPBf/0FmXTW -raJcueEk5E1qCb/zpZf1KV3AWPCTkfsrV/NnhoBuBfGKEKpm466zeSQJqi0YgQJK -91lHp2unpKiQWpSUc3sf1ofGvVxFe/RxVVZ4AYXs7DiYH/NYcwc84VCyn8bl5fLm -lI92fXGYN/Qg7Ms/lC2G7D5vFaDFCFQn+zFFUwIDAQABAoIBABNTpiIxbLDhs998 -uvQ3XsumR08kXVcwa/GgvWOSZIEJOUaAYWubDaBTNvTjlhMl6DI+YvKihZMnAkp9 -fXefF1nFUWJJvI0ryi8w6RD54RtbCG4c4raRqysVU7wumZsSenAdc0o09UQE6zXc -uth/+1VSKCzVjRkLwquXDg0rD3vHfQHWQvQlzwUh3OACA3LfLezVqzrEB02YVRxm -xwg5veeMg6Aod8vsvsajry9eE0hKeFGonRANerL9lwZxzD2ZjU2fSEJYY3xxKVgi -o+QVTKaAt9pivOs10YVZDcIDH0xmDpxAkaLb5ZAbnjwhf7WGYgEm8VruAHkJxyXX -yPf3rpkCgYEA0dp/Xv5KBIkD6JJao8hnhtP5x9U7o/pTzRxaO3WUflvTI6DtC0nk -cTOwFVs4WljT76T937q2x4stuX0woHzvIaZ6wwZ2vv81ikDY6pE8eLWsH/xFAmkx -HBfkSijFgJV6EpTqUnFD7QKU89tzWrh/kxaMO1WgFaBhxPPs3K1LDTUCgYEAzVW5 -3yjfVHNgjWTeAbnbToGvUihOidvIvS5cVo5q0Dhfabz0tiXFxAoQUGErUGPC8Nu2 -n/HxTI3b0PbCCwjBsDNqX2kzxTSe5aTGIrBUWbped1bxme8jggXuWYbg8vvLpsYf -wAJPxnGIxW/v/aftHUhbTIuVfZX2+UnilrwiwWcCgYEAg8paz4eXaH277KVtMwq6 -qZwac/tgNz0Qv/GcYVcYaLq2QNvhwoMnakhxvxfIrkS25PuTTJxwCaVIlAMhNMkB -TPrGghBfJtgUAb1z/Ow1NAG0FWpS1I7HfsMqZcBxOK2nOmA3QItNg11pujQJn+Ha -jL9OVj0SCkLs48nk6ToTtjkCgYEAh8YCtNwq6IWuN3CWGCAUMpIwIqxCWof48Zch -OZ7MZEiSVrG6QmMxpRJefTfzUyHUOj2eQZ7SxqMa0c8IuhEdOeyVjudaczD7TLAq -z68252oDovfbo8Tr/sL7OzmjryfuHqXtQqKEq5xRKvR8hYavlGhO7otx2uv5thcz -/CYE+UsCgYAsgewfzbcBqJwsAUs98DK99EU8VqKuaYlU5wNvAVb27O6sVeysSokM -G1TGIXJPphA3dSAe4Pf/j4ff/eGaS20FAFhs4BPpw0fAeCHpmD0BjIba0lxBS/gY -dc+JVPKL8Fe4a8fmsI6ndcZQ9qpOdZM5WOD0ldKRc+SsrYKkTmOOJQ== ------END RSA PRIVATE KEY----- +-----BEGIN PRIVATE KEY----- +MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQDc75YkXnDM/wDM +CKivHucQ4eB7/Hhiqef83SlZFeylPnnBgD+vIJxJm1+dKTxr2CRAOyJM3ZEakdwr +KVE+GzvwsfYT3yZQ/wD+qdTrHs5qotXjtw8PVZTZXt7Gc11l0dic6rneiccRGYU0 +c+kGxy8AVYYDkuxThFIVWXzrtsr5h1LMFrEzZlMS5IgjKH4aZT7RaX4t3vnRNeW0 +S14xDljpuJDwCjcYUtzyNK8bSiO8Q4b71H5LqBQSdKrpU/387EfsGsH2/pHmyxBV +eStirxKX7OJb9cIh+VTwfEIyGDYCWGpPcB6h0MjfZ5G7E1A/tbcS74vKpPLWjdfc +Sf6HBT8jAgMBAAECggEABnls5G7yDkfQvanpS4xzx/5ehjOdUYLaHgJbIlWupB8D +EWUROLATdGK9uAzLzPs+ALXa6Pa7OFSa55bj3xaMOJ72b5mj9U6ruErAxw0VjnFs +JCqqoc/nt0D89ULPM1slZl/Ih5lxAHLHgQR+iSjQG0nmTfeBvmbLwv1m2D94jjxX +gkaY/ugcu5fw4WM9CGzWisAJWwsNurJ8qLPumxac9zVC2Te+4ks7znyNx/4ZePmQ +/I0QrDDzlTH7BSFNciHZqo/jm1DQ5cusGnlncDLYLny9Vc2V5h31B+Wv6Y7NqxI/ +BKFAfXVCH0KABUGXlM9Jggb3STslAD4kZzuyZCc9cQKBgQDmep0eNhoINwF9Hha6 +7LY0iU0sVokAodaax70VXdYQWxuJBnGrFxaVsIm+VHlIsyVW/BNGrc0oQq6G5ZTC +rGzKAud2tCzDQHhwNkH9U47qRjRJBpvsLFkLOPrMRBCthGFd2JObIaMx9dIZajOl +yKdqg5QlibTjOa+FrMoI3uEUUwKBgQD1ZnSKDs/5DYR4nTbDbFnu8ZbfqQbTBzh0 +jffh6nOHCASpvWUUf1H+HOVbNZpXU6Bs5p7/EFeV4Y+XxQ28BeDCkkhEGix9Dlpe +yTyoJZ2Kcw5meWs0TtJ0hmw5lFt7LcwdV9Vdv8uJpZXVH84GhVYazDYZx9hem3Mh +JfSlXUbP8QKBgQCUsHW8j9qrghELzuWLIjtKGWoYOJuaNM5mfIWXyznWw3A4LlgW +sIZ5e7Lhf03fgPON8oHdDzNTzfEaRfJ6xRJKISWChyp5rJYGjnp0G/wPV8tTcVgW +PWYUv2XNcfvwdX77NEXFxyWpjZVFoCZBxfl71fc1YGm6TfTWoGqW3nMG0QKBgQDK +FKTDS49XuBnueQgvxXoJmkhzxe0oujlzQbVvBHI+MAwm+x/R0Ubjy0fT5EBEyUjC +ZOdpgzuc4TIt+MdmnjCICY4ZBrS8BqXEiFK65L7m0/h3ymTNnlsQOeHFSLZMqVn8 +j+fUW6xwnTrVVmuCUyqkxqWGdVtoDPqxwV/FDRrTAQKBgQCo9sHRi8MJXJdgRdUJ +aYzNk5Dr8K4NGXKOHqHtyAruEEomT6IEz6RKs15wLNQJ4wYpKCks9Vy1akYnRJVy +6u4w6G0QhyIv2jOBwSdaChWTe/W746zT9UagtULzmthpSpGxZj5dX+hk5I9UpiC5 +WE5MqsM3jatggZFzEeD8+Mzrug== +-----END PRIVATE KEY----- diff --git a/bin/dump_breakpad_symbols.py b/bin/dump_breakpad_symbols.py index 2b566845d..2321e17e1 100755 --- a/bin/dump_breakpad_symbols.py +++ b/bin/dump_breakpad_symbols.py @@ -178,7 +178,7 @@ def is_regular_file(path): def is_elf_file(path): """Check whether 'path' is an ELF file.""" - return is_regular_file(path) and b'ELF' in magic.from_file(path) + return is_regular_file(path) and 'ELF' in magic.from_file(path) def find_elf_files(path): diff --git a/bin/impala-config.sh b/bin/impala-config.sh index 7306e1bc2..9f923e426 100755 --- a/bin/impala-config.sh +++ b/bin/impala-config.sh @@ -189,7 +189,7 @@ export IMPALA_POSTGRES_JDBC_DRIVER_VERSION=42.5.6 unset IMPALA_POSTGRES_JDBC_DRIVER_URL export IMPALA_MYSQL_JDBC_DRIVER_VERSION=8.2.0 unset IMPALA_MYSQL_JDBC_DRIVER_URL -export IMPALA_PYTHON_VERSION=3.8.18 +export IMPALA_PYTHON_VERSION=3.11.14 unset IMPALA_PYTHON_URL export IMPALA_RAPIDJSON_VERSION=1.1.0-p1 unset IMPALA_RAPIDJSON_URL diff --git a/infra/python/deps/requirements.txt b/infra/python/deps/requirements.txt index eb0c8f575..6df6437da 100644 --- a/infra/python/deps/requirements.txt +++ b/infra/python/deps/requirements.txt @@ -20,60 +20,62 @@ # Dependents are indented. Dependents that have multiple parents are not listed # multiple times (though maybe they could be). -allpairspy == 2.5.0 -argparse == 1.4.0 -flake8==3.9.2 - mccabe == 0.6.1 - pyflakes==2.3.1 - pycodestyle==2.7.0 -flake8-unused-arguments==0.0.13 +allpairspy == 2.5.1 +flake8==7.3.0 + mccabe == 0.7.0 + pyflakes==3.4.0 + pycodestyle==2.14.0 +flake8-unused-arguments==0.0.14 future == 0.18.3 -hdfs == 2.0.2 +hdfs == 2.7.3 docopt == 0.6.2 - execnet == 1.9.0 impyla == 0.21a3 bitarray == 2.3.0 pure-sasl == 0.6.2 - # six == 1.14.0 (specified separately) + # six == 1.17.0 (specified separately) thrift_sasl == 0.4.3 isort == 5.13.2 poetry-core == 1.9.1 k5test==0.10.3 -kazoo==2.8.0 +kazoo==2.10.0 kerberos == 1.3.1 pexpect == 3.3 -pg8000 == 1.10.2 prettytable == 2.5.0 wcwidth == 0.2.14 -prometheus-client == 0.12.0 -psutil == 5.6.3 -pyparsing == 2.4.7 +prometheus-client == 0.24.1 +psutil == 7.2.2 +pyparsing == 3.3.2 pytest == 6.2.5 attrs == 19.2.0 iniconfig == 1.1.1 - pluggy == 0.13.1 - py == 1.8.2 + pluggy == 1.6.0 + py == 1.11.0 pytest-forked == 1.6.0 pytest-reportlog == 0.4.0 pytest-timeout == 2.2.0 pytest-xdist == 2.4.0 + execnet == 2.1.2 + hatchling == 1.28.0 + pathspec == 1.0.4 + trove-classifiers == 2026.1.14.14 + calver == 2025.10.20 + hatch-vcs == 0.5.0 toml == 0.10.2 -python-magic == 0.4.11 +python-magic == 0.4.27 pywebhdfs == 0.3.2 - pbr == 3.1.1 -requests == 2.21.0 - chardet == 3.0.4 - idna == 2.8 - urllib3 == 1.24.2 - certifi == 2020.12.5 -sasl == 0.2.1 -six == 1.14.0 + pbr == 7.0.3 +requests == 2.32.5 + charset_normalizer == 3.4.4 + idna == 3.11 + urllib3 == 2.6.3 + certifi == 2026.1.4 +sasl == 0.4a1 +six == 1.17.0 sqlparse == 0.3.1 -texttable == 0.8.3 -avro==1.10.2 +avro==1.12.1 thrift==0.16.0 -distro==1.6.0 +distro==1.9.0 # Required for Kudu: - Cython == 0.29.14 - pytz == 2020.1 + Cython == 0.29.37 + pytz == 2025.2 diff --git a/infra/python/deps/setuptools-requirements.txt b/infra/python/deps/setuptools-requirements.txt index 713bfa0af..2882a8f14 100644 --- a/infra/python/deps/setuptools-requirements.txt +++ b/infra/python/deps/setuptools-requirements.txt @@ -16,6 +16,8 @@ # under the License. # Newer versions of setuptools don't support Python 2.7 -setuptools == 44.1.1 - wheel == 0.35.1 -setuptools-scm == 5.0.2 +setuptools == 80.10.2 + wheel == 0.46.3 + flit-core == 3.12.0 + packaging == 26.0 +setuptools-scm == 9.2.2 diff --git a/tests/common/custom_cluster_test_suite.py b/tests/common/custom_cluster_test_suite.py index f597cd8fa..155a9f169 100644 --- a/tests/common/custom_cluster_test_suite.py +++ b/tests/common/custom_cluster_test_suite.py @@ -23,8 +23,8 @@ import inspect import logging import os import os.path -import pipes import pytest +import shlex import subprocess from glob import glob @@ -657,7 +657,7 @@ class CustomClusterTestSuite(ImpalaTestSuite): options.append("--impalad_args=--default_query_options={0}".format( ','.join(["{0}={1}".format(k, v) for k, v in default_query_option_kvs]))) - cmd_str = " ".join(pipes.quote(arg) for arg in cmd + options) + cmd_str = " ".join(shlex.quote(arg) for arg in cmd + options) # If the cluster is already started, we don't need to start it again, unless # force_restart is set to True. NOTE: reordering tests into classes with class-level diff --git a/tests/common/impala_cluster.py b/tests/common/impala_cluster.py index 7f422c2cf..41abc41c6 100644 --- a/tests/common/impala_cluster.py +++ b/tests/common/impala_cluster.py @@ -22,11 +22,11 @@ from builtins import map, range import json import logging import os -import pipes import psutil +import requests +import shlex import socket import time -import requests from getpass import getuser from random import choice from signal import SIGKILL, SIGRTMIN @@ -487,6 +487,15 @@ class Process(object): # In non-containerised case, search for process based on matching command lines. procs = [] for process in psutil.process_iter(['pid', 'ppid', 'cmdline']): + # psutil changed behavior for process_iter() so that it no longer checks + # for reused pids. Use is_running() to do that. + if not process.is_running(): + continue + + # psutil returns None for fields from a zombie process + if not process.info['cmdline']: + continue + # Use info because it won't throw NoSuchProcess exceptions. if set(self.cmd) == set(process.info['cmdline']): procs.append(process.info) @@ -804,9 +813,9 @@ def run_daemon(daemon_binary, args, build_type="latest", env_vars={}, output_fil # cannot cleanly kill it until py.test exits. In theory, Popen(shell=True) should # achieve the same thing but it doesn't work on some platforms for some reasons. sys_cmd = ("{set_cmds} {cmd} {redirect} &".format( - set_cmds=''.join(["export {0}={1};".format(k, pipes.quote(v)) + set_cmds=''.join(["export {0}={1};".format(k, shlex.quote(v)) for k, v in env_vars.items()]), - cmd=' '.join([pipes.quote(tok) for tok in cmd]), + cmd=' '.join([shlex.quote(tok) for tok in cmd]), redirect=redirect)) os.system(sys_cmd) diff --git a/tests/common/resource_pool_config.py b/tests/common/resource_pool_config.py index cd72340e7..c4e8f651f 100644 --- a/tests/common/resource_pool_config.py +++ b/tests/common/resource_pool_config.py @@ -100,7 +100,7 @@ class ResourcePoolConfig(object): def __find_xml_node(self, xml_root, pool_name, pool_attribute): """Returns the xml node corresponding to the 'pool_attribute' for the 'pool_name'""" - for property in xml_root.getiterator('property'): + for property in xml_root.iter('property'): try: name = property.find('name').text # eg. of name = impala.admission-control.max-query-mem-limit-bytes.root.pool_name diff --git a/tests/common/test_result_verifier.py b/tests/common/test_result_verifier.py index 305cccb05..be5a6df00 100644 --- a/tests/common/test_result_verifier.py +++ b/tests/common/test_result_verifier.py @@ -576,13 +576,15 @@ def parse_result_rows(exec_result, escape_strings=True): result.append(','.join(new_cols)) return result + # Special syntax for basic aggregation over fields in the runtime profile. # The syntax is: # aggregation(function, field_name): expected_value # Currently, the only implemented function is SUM and only integers are supported. -AGGREGATION_PREFIX_PATTERN = 'aggregation\(' +AGGREGATION_PREFIX_PATTERN = r'aggregation\(' AGGREGATION_PREFIX = re.compile(AGGREGATION_PREFIX_PATTERN) -AGGREGATION_SYNTAX_MATCH_PATTERN = 'aggregation\((\w+)[ ]*,[ ]*([^)]+)\)([:><])[ ]*(\d+)' +AGGREGATION_SYNTAX_MATCH_PATTERN = r'aggregation\((\w+)[ ]*,[ ]*([^)]+)\)([:><])[ ]*(\d+)' + def try_compile_aggregation(row_string): """ @@ -615,7 +617,7 @@ def compute_aggregation(function, field, runtime_profile): # These lines are printed by 'be/src/util/pretty-printer.h' with verbose=true. # 'field_regex' also captures the accurate value of the field which is the number # in parenthesis. It means we can retrieve this value with 're.findall()'. - field_regex = "{0}: \d+(?:\.\d+[KMB])? \((\d+)\)".format(field) + field_regex = r"{0}: \d+(?:\.\d+[KMB])? \((\d+)\)".format(field) field_regex_re = re.compile(field_regex) inside_avg_fragment = False avg_fragment_indent = None diff --git a/tests/comparison/cluster.py b/tests/comparison/cluster.py index ef39e2390..61d84ad71 100644 --- a/tests/comparison/cluster.py +++ b/tests/comparison/cluster.py @@ -90,7 +90,7 @@ class Cluster(with_metaclass(ABCMeta, object)): if not file_name.lower().endswith(".xml"): continue xml_doc = parse_xml(os.path.join(self.local_hadoop_conf_dir, file_name)) - for property in xml_doc.getiterator("property"): + for property in xml_doc.iter("property"): name = property.find("name") if name is None or name.text is None: continue diff --git a/tests/comparison/leopard/controller.py b/tests/comparison/leopard/controller.py index 531abcc4d..6af7422f2 100755 --- a/tests/comparison/leopard/controller.py +++ b/tests/comparison/leopard/controller.py @@ -154,7 +154,7 @@ class Controller(object): ''' while True: self.schedule_items = dict([(run_id, thread) for run_id, thread - in self.schedule_items.items() if self.schedule_items[run_id].isAlive()]) + in self.schedule_items.items() if self.schedule_items[run_id].is_alive()]) LOG.info('Number of Active Threads: {0}'.format(len(self.schedule_items))) self.generate_schedule_item() self.start_new_jobs() diff --git a/tests/custom_cluster/test_admission_controller.py b/tests/custom_cluster/test_admission_controller.py index 4be8f7a76..fcc97d2ef 100644 --- a/tests/custom_cluster/test_admission_controller.py +++ b/tests/custom_cluster/test_admission_controller.py @@ -2651,7 +2651,7 @@ class TestAdmissionControllerStress(TestAdmissionControllerBase): for thread in self.all_threads: thread.join(5) LOG.info("Join thread for query num %s %s", thread.query_num, - "TIMED OUT" if thread.isAlive() else "") + "TIMED OUT" if thread.is_alive() else "") super(TestAdmissionControllerStress, self).teardown_method(method) def should_run(self): diff --git a/tests/stress/query_retries_stress_runner.py b/tests/stress/query_retries_stress_runner.py index 3f80d6741..437fc2c13 100755 --- a/tests/stress/query_retries_stress_runner.py +++ b/tests/stress/query_retries_stress_runner.py @@ -27,9 +27,9 @@ from __future__ import absolute_import, division, print_function from builtins import map, range import logging -import pipes import os import random +import shlex import subprocess import sys import threading @@ -94,7 +94,7 @@ completed_queries_latch = None def configured_call(cmd): """Call a command in a shell with config-impala.sh.""" if type(cmd) is list: - cmd = " ".join([pipes.quote(arg) for arg in cmd]) + cmd = " ".join([shlex.quote(arg) for arg in cmd]) cmd = "source {0}/bin/impala-config.sh && {1}".format(IMPALA_HOME, cmd) return subprocess.check_call(["bash", "-c", cmd]) diff --git a/tests/util/auto_scaler.py b/tests/util/auto_scaler.py index b39e53665..d7ee34f6e 100755 --- a/tests/util/auto_scaler.py +++ b/tests/util/auto_scaler.py @@ -22,7 +22,7 @@ import argparse import time import logging import os -import pipes +import shlex from subprocess import check_call from tests.common.impala_cluster import ImpalaCluster from threading import Event, Thread @@ -258,7 +258,7 @@ class AutoScaler(object): options += ["--impalad_args=%s" % a for a in impalad_args] logging.debug("Starting cluster with command: %s" % - " ".join(pipes.quote(arg) for arg in cmd + options)) + " ".join(shlex.quote(arg) for arg in cmd + options)) log_debug = logging.getLogger().getEffectiveLevel() == logging.DEBUG log_file = None if not log_debug:
