This is an automated email from the ASF dual-hosted git repository. joemcdonnell pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/impala.git
commit 9fb1274867a59b467617e6292a54fd1f0bb937ad Author: Joe McDonnell <[email protected]> AuthorDate: Mon Apr 3 21:15:52 2023 -0700 IMPALA-12117: Use separate cache dirs for shell pip installs Pip sporadically hits an error when installing impala-shell into a virtualenv. An example symptom is this (though the issue is not specific to thrift): WARNING: Skipping page https://pypi.org/simple/thrift/ because the GET request got Content-Type: Unknown. The only supported Content-Types are application/vnd.pypi.simple.v1+json, application/vnd.pypi.simple.v1+html, and text/html ERROR: Could not find a version that satisfies the requirement thrift==0.16.0 (from impala-shell) (from versions: none) ERROR: No matching distribution found for thrift==0.16.0 It appears that this error can occur when two pip processes are installing into virtualenvs simultaneously and share a cache directory. This happens for our impala-shell build, because we are doing pip install for Python 2 and Python 3 simultaneously. The impala-python/impala-python3 virtualenvs do not use a cache directory and are not impacted. This changes the shell's pip install to give the Python 2 and Python 3 separate cache directories. The cache directories are placed in ~/.cache like the regular pip cache. These do not consume much space (a couple MB). Testing: - Ran all-build-options-ub2004 ten times without seeing the failure Change-Id: I3f834b9f8c8cbc09830745ad132677a2fe17e07b Reviewed-on: http://gerrit.cloudera.org:8080/19813 Tested-by: Impala Public Jenkins <[email protected]> Reviewed-by: Daniel Becker <[email protected]> Reviewed-by: Michael Smith <[email protected]> --- shell/CMakeLists.txt | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/shell/CMakeLists.txt b/shell/CMakeLists.txt index e41e72d0f..104125bc4 100644 --- a/shell/CMakeLists.txt +++ b/shell/CMakeLists.txt @@ -39,13 +39,18 @@ add_custom_target(shell_pypi_test_package DEPENDS shell_tarball impala_python set(PYTHON2_VENV "${CMAKE_SOURCE_DIR}/shell/build/py2_venv") set(PYTHON3_VENV "${CMAKE_SOURCE_DIR}/shell/build/py3_venv") +# IMPALA-12117: Use separate pip cache directories to avoid concurrency +# issues. The standard location is in ~/.cache/pip, so this uses directories +# inside ~/.cache. These typical consume a couple MB each. +set(PYTHON2_PIP_CACHE "~/.cache/impala_py2_pip") +set(PYTHON3_PIP_CACHE "~/.cache/impala_py3_pip") add_custom_command(OUTPUT "${PYTHON2_VENV}" DEPENDS impala_python COMMAND impala-virtualenv --python "$ENV{IMPALA_SYSTEM_PYTHON2}" "${PYTHON2_VENV}" ) add_custom_target(shell_python2_install DEPENDS "${PYTHON2_VENV}" shell_pypi_test_package - COMMAND "${PYTHON2_VENV}/bin/pip" install "${SHELL_TEST_PKG}" + COMMAND "${PYTHON2_VENV}/bin/pip" install --cache-dir "${PYTHON2_PIP_CACHE}" "${SHELL_TEST_PKG}" ) add_custom_command(OUTPUT "${PYTHON3_VENV}" DEPENDS impala_python @@ -53,5 +58,5 @@ add_custom_command(OUTPUT "${PYTHON3_VENV}" DEPENDS impala_python ) add_custom_target(shell_python3_install DEPENDS "${PYTHON3_VENV}" shell_pypi_test_package - COMMAND "${PYTHON3_VENV}/bin/pip" install "${SHELL_TEST_PKG}" + COMMAND "${PYTHON3_VENV}/bin/pip" install --cache-dir "${PYTHON3_PIP_CACHE}" "${SHELL_TEST_PKG}" )
