github-actions[bot] commented on code in PR #66546:
URL: https://github.com/apache/doris/pull/66546#discussion_r3878580776
##########
build.sh:
##########
@@ -491,12 +503,27 @@ fi
# shellcheck source=thirdparty/arrow-paimon-vars.sh
. "${DORIS_HOME}/thirdparty/arrow-paimon-vars.sh"
NEED_ARROW_PAIMON_THIRDPARTY=false
-if [[ "${BUILD_BE}" -eq 1 || "${BUILD_CLOUD}" -eq 1 ||
- "${BUILD_META_TOOL}" == "ON" || "${BUILD_FILE_CACHE_MICROBENCH_TOOL}" ==
"ON" ||
+if [[ "${BUILD_BE}" -eq 1 || "${BUILD_META_TOOL}" == "ON" ||
+ "${BUILD_FILE_CACHE_MICROBENCH_TOOL}" == "ON" ||
"${BUILD_INDEX_TOOL}" == "ON" ]]; then
NEED_ARROW_PAIMON_THIRDPARTY=true
fi
+if [[ "${NEED_ARROW_PAIMON_THIRDPARTY}" == "true" ]]; then
+
DEFAULT_ARROW_PAIMON_HOME="${DORIS_THIRDPARTY}/installed/${ARROW_INSTALL_SUBDIR}"
Review Comment:
[P1] Keep the previous official prebuilt usable during rollout
This makes `installed/arrow-24.0.0` mandatory, but the current automation
archive is stamped `1731787677f0` and contains Arrow/Paimon 24 only at the
unversioned root. Official compilation images expose only that `installed` tree
as `DORIS_THIRDPARTY`, so a default build on the new master reaches
`rebuild_thirdparty_libraries()` and exits because the install-only prefix has
no `build-thirdparty.sh`. Publishing the dual layout first is not safe either:
it replaces root 24 with 17 for live pre-merge master and the current
branch-4.1 consumer (the latter is the existing unresolved thread); the manual
arbitrary-ref workflow still clobbers this same shared release. Please add a
transitional, fingerprint/version-validated root-24 fallback that prefers the
versioned prefix, or stage a consumer/artifact rollout with no mismatched
window, and cover the immediately previous official archive in the lifecycle
test.
##########
thirdparty/arrow-paimon-vars.sh:
##########
@@ -357,3 +557,108 @@ require_arrow_prebuilt_for_paimon() {
return 1
fi
}
+
+invalidate_arrow_17_prebuilt_marker() {
+ local install_dir="$1"
+ rm -f "${install_dir}/arrow-17-build-fingerprint.txt" \
+ "${install_dir}/arrow-paimon-17-build-fingerprint.txt"
+}
+
+publish_arrow_17_prebuilt_marker() {
+ local install_dir="$1"
+ arrow_17_artifacts_valid "${install_dir}"
+ arrow_17_build_fingerprint >"${install_dir}/arrow-17-build-fingerprint.txt"
+}
+
+arrow_17_prebuilt_valid() {
+ local install_dir="$1"
+ local fingerprint_mark="${install_dir}/arrow-17-build-fingerprint.txt"
+ local expected_fingerprint
+
+ if [[ ! -f "${fingerprint_mark}" ]]; then
+ echo "Missing Arrow 17 build fingerprint: ${fingerprint_mark}" >&2
+ return 1
+ fi
+ expected_fingerprint="$(arrow_17_build_fingerprint)"
+ if [[ "$(<"${fingerprint_mark}")" != "${expected_fingerprint}" ]]; then
+ echo "Arrow 17 build fingerprint does not match selected inputs" >&2
+ return 1
+ fi
+ arrow_17_artifacts_valid "${install_dir}"
+}
+
+require_arrow_17_prebuilt_for_paimon() {
+ local install_dir="$1"
+ if ! arrow_17_prebuilt_valid "${install_dir}"; then
+ echo "Paimon for pre-upgrade branch-4.1 requires Arrow 17 to be built
first" >&2
+ return 1
+ fi
+}
+
+invalidate_paimon_17_prebuilt_marker() {
+ local install_dir="$1"
+ rm -f "${install_dir}/paimon-arrow-17-build-fingerprint.txt" \
+ "${install_dir}/arrow-paimon-17-build-fingerprint.txt"
+}
+
+# A legacy-prefix Arrow downgrade must first remove Paimon built against the
+# previous Arrow ABI. If the Arrow build is interrupted, the incomplete prefix
+# then fails at link time instead of exposing a mixed Arrow/Paimon SDK.
+prepare_arrow_17_install_prefix() {
+ local install_dir="$1"
+
+ rm -f "${install_dir}/arrow-build-fingerprint.txt" \
+ "${install_dir}/paimon-build-fingerprint.txt" \
+ "${install_dir}/arrow-paimon-build-fingerprint.txt"
+ invalidate_paimon_17_prebuilt_marker "${install_dir}"
+ clean_paimon_artifacts_in "${install_dir}"
+ invalidate_arrow_17_prebuilt_marker "${install_dir}"
+ clean_arrow_artifacts_in "${install_dir}"
+}
+
+publish_paimon_17_prebuilt_marker() {
+ local install_dir="$1"
+ paimon_artifacts_valid_in "${install_dir}"
+ paimon_17_build_fingerprint
>"${install_dir}/paimon-arrow-17-build-fingerprint.txt"
+}
+
+paimon_17_prebuilt_valid() {
+ local install_dir="$1"
+ local
fingerprint_mark="${install_dir}/paimon-arrow-17-build-fingerprint.txt"
+ local expected_fingerprint
+
+ if [[ ! -f "${fingerprint_mark}" ]]; then
+ echo "Missing Paimon Arrow 17 build fingerprint: ${fingerprint_mark}"
>&2
+ return 1
+ fi
+ expected_fingerprint="$(paimon_17_build_fingerprint)"
+ if [[ "$(<"${fingerprint_mark}")" != "${expected_fingerprint}" ]]; then
+ echo "Paimon Arrow 17 build fingerprint does not match selected
inputs" >&2
+ return 1
+ fi
+ paimon_artifacts_valid_in "${install_dir}"
+}
+
+arrow_paimon_17_prebuilt_valid() {
+ local install_dir="$1"
+ arrow_17_prebuilt_valid "${install_dir}" &&
+ paimon_17_prebuilt_valid "${install_dir}"
+}
+
+shared_arrow_paimon_prebuilt_valid() {
+ local install_dir="$1"
+ arrow_paimon_17_prebuilt_valid "${install_dir}" &&
+ arrow_paimon_prebuilt_valid "${install_dir}"
+}
+
+select_arrow_paimon_rebuild_packages() {
+ local install_dir="$1"
+ ARROW_PAIMON_REBUILD_PACKAGES=()
+
+ if ! arrow_paimon_17_prebuilt_valid "${install_dir}"; then
Review Comment:
[P2] Do not rebuild valid Arrow 17 for a Paimon-only change
This composite check appends `arrow_17 paimon_cpp_17` even when
`arrow_17_prebuilt_valid` succeeds and only the Paimon 17 marker/input is
stale. For example, a Paimon-only cache change or retry after `paimon_cpp_17`
fails leaves Arrow 17 valid, yet the new selector repeats that large build; a
fixture with valid Arrow and invalid Paimon markers selects all four packages.
The patch already publishes independent component fingerprints, and
`build_paimon_cpp_17()` validates/reuses Arrow through
`require_arrow_17_prebuilt_for_paimon()`, so select only `paimon_cpp_17` in
this case and add the component-only recovery case to the lifecycle test.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
