github-actions[bot] commented on code in PR #68562:
URL: https://github.com/apache/doris/pull/68562#discussion_r4124047331
##########
build.sh:
##########
@@ -486,7 +486,12 @@ else
LAST_THIRDPARTY_LIB='hadoop_hdfs_3_4/native/libhdfs.a'
fi
-if [[ ! -f "${DORIS_THIRDPARTY}/installed/lib/${LAST_THIRDPARTY_LIB}" ]]; then
+# Old installs can contain the sentinel but predate Paimon. Rebuild the whole
+# dependency set so Lance and Paimon also use the same selected Rust toolchain.
+if [[ ! -f "${DORIS_THIRDPARTY}/installed/lib/${LAST_THIRDPARTY_LIB}" ||
+ ! -f "${DORIS_THIRDPARTY}/installed/lib64/liblance_c.a" ||
+ ! -f "${DORIS_THIRDPARTY}/installed/lib64/libpaimon_c.a" ||
Review Comment:
[P2] Check the Rust artifacts only when BE CMake will run. With a valid
pre-PR install, `./build.sh --fe`, `--cloud`, or a Java-only selection reaches
this unconditional gate without a Paimon archive/header, deletes the entire
`installed` tree, and rebuilds every dependency (including Rust 1.94 and
cbindgen) before its own work. Those targets do not link `paimon_c`; keep the
old platform sentinel for them. When gating the new checks, include
`--compile-bench`: it sets `BUILD_BE=1` only after this gate.
##########
thirdparty/build-thirdparty.sh:
##########
@@ -2440,11 +2464,132 @@ build_lance_c() {
mkdir -p "${TP_INSTALL_DIR}/include" "${TP_INSTALL_DIR}/lib64"
rm -rf "${TP_INSTALL_DIR}/include/lance"
cp -av include/lance "${TP_INSTALL_DIR}/include/"
- cp -v "${BUILD_DIR}/release/liblance_c.a" "${TP_INSTALL_DIR}/lib64/"
+ install_rust_archive "${BUILD_DIR}/release/liblance_c.a"
+}
+
+build_paimon_rust() {
+ check_if_source_exist "${PAIMON_RUST_SOURCE}"
+ cd "${TP_SOURCE_DIR}/${PAIMON_RUST_SOURCE}"
+
+ rm -rf "${BUILD_DIR}"
+ mkdir -p "${BUILD_DIR}"
+
+ local cargo_bin="${PAIMON_RUST_CARGO:-${CARGO:-cargo}}"
+ if ! command -v "${cargo_bin}" >/dev/null 2>&1; then
+ echo "cargo is required to build paimon-rust. Install Rust 1.94.0 or
set PAIMON_RUST_CARGO."
+ exit 1
+ fi
+
+ local required_rust_version="1.94.0"
+ local cargo_env=(
+ "CARGO_BUILD_JOBS=${PARALLEL}"
+ "CARGO_TARGET_DIR=${PWD}/${BUILD_DIR}"
+ )
+ if command -v rustup >/dev/null 2>&1 && [[ -z "${RUSTUP_TOOLCHAIN}" ]];
then
+ # The presence check must look for the toolchain the minimum actually
+ # requires, not a literal: with only an older toolchain installed the
+ # stale check would skip the install below and then force
+ # RUSTUP_TOOLCHAIN to a version rustup cannot dispatch, failing the
+ # build before any archive is produced.
+ local required_rust_regex="${required_rust_version//./\\.}"
+ if ! rustup toolchain list | grep -Eq
"^${required_rust_regex}([[:space:]-]|$)"; then
+ rustup toolchain install "${required_rust_version}" --profile
minimal
+ fi
+ cargo_env+=("RUSTUP_TOOLCHAIN=${required_rust_version}")
+ fi
- if [[ "${STRIP_TP_LIB}" = "ON" && "${KERNEL}" != 'Darwin' ]]; then
- strip --strip-debug --strip-unneeded
"${TP_INSTALL_DIR}/lib64/liblance_c.a"
+ local cargo_version
+ if ! cargo_version="$(env "${cargo_env[@]}" "${cargo_bin}" --version | awk
'{print $2}')"; then
+ echo "failed to get cargo version for paimon-rust. Install Rust
${required_rust_version} or set PAIMON_RUST_CARGO/RUSTUP_TOOLCHAIN."
+ exit 1
fi
+ # Rust 1.94.0 is the minimum supported version. Allow newer toolchains when
+ # callers explicitly select one or rustup is unavailable on the system.
+ # NOTE: paimon_c and lance_c are both Rust staticlibs linked into the same
+ # BE binary; they must be built with the SAME rustc toolchain so the linker
+ # resolves both crates' std references against a single std copy. Mixing
+ # toolchains makes the precompiled std hashes differ and the linker pulls
+ # both std copies in, colliding on the unmangled `rust_eh_personality`
+ # (duplicate symbol). Rebuild both lance_c and paimon_rust whenever the
+ # toolchain changes, using the same RUSTUP_TOOLCHAIN for both builds.
+ if ! awk -v required="${required_rust_version}" -v
actual="${cargo_version}" 'BEGIN {
+ split(required, r, ".");
+ split(actual, a, ".");
+ for (i = 1; i <= 3; i++) {
+ if ((a[i] + 0) > (r[i] + 0)) {
+ exit 0;
+ }
+ if ((a[i] + 0) < (r[i] + 0)) {
+ exit 1;
+ }
+ }
+ exit 0;
+ }'; then
+ echo "paimon-rust requires Rust/Cargo ${required_rust_version} or
newer, but found ${cargo_version}."
+ echo "Install Rust ${required_rust_version} or set
PAIMON_RUST_CARGO/RUSTUP_TOOLCHAIN."
+ exit 1
+ fi
+
+ if [[ "${KERNEL}" != 'Darwin' ]]; then
+ cargo_env+=("CFLAGS=${CFLAGS:-} -std=gnu17")
+ fi
+
+ local cargo_args=(build --release --locked -p paimon-c --features
paimon/storage-hdfs)
+ # cbindgen invokes cargo metadata itself; command-line flags on the build
+ # and install calls do not propagate to that child process.
+ cargo_env+=("CARGO=${cargo_bin}")
+ if [[ "$(echo "${PAIMON_RUST_CARGO_OFFLINE}" | tr '[:lower:]'
'[:upper:]')" == "ON" ]]; then
+ cargo_args+=(--offline)
+ cargo_env+=("CARGO_NET_OFFLINE=true")
+ fi
+ env "${cargo_env[@]}" "${cargo_bin}" "${cargo_args[@]}"
+
+ # Generate the C header from the Rust extern "C" surface via cbindgen.
+ # cbindgen is a pinned, Doris-controlled input: an unpinned "current"
+ # release would regenerate paimon.h differently between builds. The
+ # pinned version installs under a Doris-controlled --root and its
+ # resolved absolute path is invoked directly (a custom CARGO_HOME does
+ # not necessarily put cargo-installed binaries on PATH). Offline builds
+ # pass --offline to the install command, exactly like the fetch/build
+ # handling above — cargo fails on a missing local crate cache instead
+ # of reaching for the network.
+ local cbindgen_version="0.29.4"
+ local cbindgen_bin="${PAIMON_RUST_CBINDGEN:-}"
+ if [[ -z "${cbindgen_bin}" ]]; then
+ local
cbindgen_root="${TP_SOURCE_DIR}/.doris-cbindgen-${cbindgen_version}"
+ cbindgen_bin="${cbindgen_root}/bin/cbindgen"
+ if [[ ! -x "${cbindgen_bin}" ]]; then
+ local cbindgen_install_args=(install cbindgen
+ --version "${cbindgen_version}" --locked --root
"${cbindgen_root}")
+ if [[ "$(echo "${PAIMON_RUST_CARGO_OFFLINE}" | tr '[:lower:]'
'[:upper:]')" == "ON" ]]; then
+ cbindgen_install_args+=(--offline)
+ fi
+ echo "cbindgen not found; installing pinned ${cbindgen_version}
via cargo install ..."
+ env "${cargo_env[@]}" "${cargo_bin}" "${cbindgen_install_args[@]}"
+ fi
+ elif [[ ! -x "${cbindgen_bin}" ]]; then
+ echo "PAIMON_RUST_CBINDGEN=${cbindgen_bin} is not an executable file."
+ exit 1
+ fi
+ # Write a temporary cbindgen.toml so the generated header carries our
+ # include-guard / cpp-compat settings without touching the upstream tree.
+ local cbindgen_toml="${BUILD_DIR}/cbindgen.toml"
+ mkdir -p "${BUILD_DIR}"
+ cat >"${cbindgen_toml}" <<'EOF'
+language = "C"
+include_guard = "PAIMON_C_H"
+pragma_once = true
+cpp_compat = true
+EOF
+ env "${cargo_env[@]}" "${cbindgen_bin}" bindings/c \
+ --config "${cbindgen_toml}" \
+ --output "${BUILD_DIR}/release/paimon.h"
+
+ mkdir -p "${TP_INSTALL_DIR}/include" "${TP_INSTALL_DIR}/lib64"
+ rm -rf "${TP_INSTALL_DIR}/include/paimon_rust"
+ mkdir -p "${TP_INSTALL_DIR}/include/paimon_rust"
+ cp -v "${BUILD_DIR}/release/paimon.h"
"${TP_INSTALL_DIR}/include/paimon_rust/"
Review Comment:
[P2] Publish the generated header atomically. During a supported
`build-thirdparty.sh paimon_rust` rebuild of an existing install, this `cp` can
create `installed/include/paimon_rust/paimon.h` and then fail or be interrupted
before it finishes. The old archive remains, and a later `./build.sh --be` sees
all four files in its `-f` gate and skips repair, leaving a truncated header
for the Paimon reader. Stage the header beside its final path and rename it
after a successful copy, as the archive installer already does.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
