This is an automated email from the ASF dual-hosted git repository.
gavinchou pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new c14320cb3c5 [enhance](Azure) Make azure dependency compile one config
(#38239)
c14320cb3c5 is described below
commit c14320cb3c54ee56fc8c010114d27a20275f0747
Author: AlexYue <[email protected]>
AuthorDate: Thu Jul 25 23:44:38 2024 +0800
[enhance](Azure) Make azure dependency compile one config (#38239)
Previously the azure sdk would be compiled into BE by default. But
sometimes the BE would run in environment which only require AWS
s3-compatible functionality. So user can use `DISABLE_BUILD_AZURE` to
control whether build azure sdk into BE or not.
`export DISABLE_BUILD_AZURE=ON` to disable build with azure support
---
be/CMakeLists.txt | 8 ++++++++
be/cmake/thirdparty.cmake | 10 ++++++----
be/src/io/CMakeLists.txt | 3 +++
be/src/util/s3_util.cpp | 9 +++++++++
be/test/io/fs/azure_test.cpp | 4 ++++
build.sh | 8 ++++++++
cloud/CMakeLists.txt | 6 ++++++
cloud/src/recycler/CMakeLists.txt | 4 ++++
cloud/src/recycler/s3_accessor.cpp | 9 +++++++++
common/cpp/obj_retry_strategy.cpp | 3 ++-
common/cpp/obj_retry_strategy.h | 4 ++++
run-be-ut.sh | 6 ++++++
run-cloud-ut.sh | 7 +++++++
thirdparty/build-thirdparty.sh | 34 ++++++++++++++++++++++------------
14 files changed, 98 insertions(+), 17 deletions(-)
diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt
index e7d0849c5ff..8ad11cf84ef 100644
--- a/be/CMakeLists.txt
+++ b/be/CMakeLists.txt
@@ -162,6 +162,14 @@ else()
find_package(Boost ${BOOST_VERSION} COMPONENTS system container)
endif()
+# Set if use libazure or not
+option(BUILD_AZURE "ON for building azure support for BE or OFF for not" OFF)
+message(STATUS "build azure: ${BUILD_AZURE}")
+if(BUILD_AZURE STREQUAL "ON")
+ add_definitions(-DUSE_AZURE)
+endif()
+
+
set(GPERFTOOLS_HOME "${THIRDPARTY_DIR}/gperftools")
include (cmake/thirdparty.cmake)
diff --git a/be/cmake/thirdparty.cmake b/be/cmake/thirdparty.cmake
index 502a22bbdf4..19f2a00012a 100644
--- a/be/cmake/thirdparty.cmake
+++ b/be/cmake/thirdparty.cmake
@@ -141,10 +141,12 @@ if (NOT OS_MACOSX)
add_thirdparty(aws-s2n LIBNAME "lib/libs2n.a")
endif()
-add_thirdparty(azure-core)
-add_thirdparty(azure-identity)
-add_thirdparty(azure-storage-blobs)
-add_thirdparty(azure-storage-common)
+if(BUILD_AZURE STREQUAL "ON")
+ add_thirdparty(azure-core)
+ add_thirdparty(azure-identity)
+ add_thirdparty(azure-storage-blobs)
+ add_thirdparty(azure-storage-common)
+endif()
add_thirdparty(minizip LIB64)
add_thirdparty(simdjson LIB64)
diff --git a/be/src/io/CMakeLists.txt b/be/src/io/CMakeLists.txt
index 09b5bbd94e9..02b34f2f0ea 100644
--- a/be/src/io/CMakeLists.txt
+++ b/be/src/io/CMakeLists.txt
@@ -22,6 +22,9 @@ set(LIBRARY_OUTPUT_PATH "${BUILD_DIR}/src/io")
set(EXECUTABLE_OUTPUT_PATH "${BUILD_DIR}/src/io")
file(GLOB_RECURSE IO_FILES CONFIGURE_DEPENDS *.cpp)
+if(BUILD_AZURE STREQUAL "OFF")
+ list(REMOVE_ITEM IO_FILES
"${CMAKE_CURRENT_SOURCE_DIR}/fs/azure_obj_storage_client.cpp")
+endif()
list(REMOVE_ITEM IO_FILES
"${CMAKE_CURRENT_SOURCE_DIR}/fs/benchmark/fs_benchmark_tool.cpp")
add_library(IO STATIC ${IO_FILES})
diff --git a/be/src/util/s3_util.cpp b/be/src/util/s3_util.cpp
index ffb93c2d9d9..24e11b03b0b 100644
--- a/be/src/util/s3_util.cpp
+++ b/be/src/util/s3_util.cpp
@@ -29,7 +29,9 @@
#include <util/string_util.h>
#include <atomic>
+#ifdef USE_AZURE
#include <azure/storage/blobs/blob_container_client.hpp>
+#endif
#include <cstdlib>
#include <filesystem>
#include <functional>
@@ -41,7 +43,9 @@
#include "common/logging.h"
#include "common/status.h"
#include "cpp/sync_point.h"
+#ifdef USE_AZURE
#include "io/fs/azure_obj_storage_client.h"
+#endif
#include "io/fs/obj_storage_client.h"
#include "io/fs/s3_obj_storage_client.h"
#include "runtime/exec_env.h"
@@ -218,6 +222,7 @@ std::shared_ptr<io::ObjStorageClient>
S3ClientFactory::create(const S3ClientConf
std::shared_ptr<io::ObjStorageClient> S3ClientFactory::_create_azure_client(
const S3ClientConf& s3_conf) {
+#ifdef USE_AZURE
auto cred =
std::make_shared<Azure::Storage::StorageSharedKeyCredential>(s3_conf.ak,
s3_conf.sk);
@@ -228,6 +233,10 @@ std::shared_ptr<io::ObjStorageClient>
S3ClientFactory::_create_azure_client(
auto containerClient =
std::make_shared<Azure::Storage::Blobs::BlobContainerClient>(uri, cred);
LOG_INFO("create one azure client with {}", s3_conf.to_string());
return
std::make_shared<io::AzureObjStorageClient>(std::move(containerClient));
+#else
+ LOG_FATAL("BE is not compiled with azure support, export BUILD_AZURE=ON
before building");
+ return nullptr;
+#endif
}
std::shared_ptr<io::ObjStorageClient> S3ClientFactory::_create_s3_client(
diff --git a/be/test/io/fs/azure_test.cpp b/be/test/io/fs/azure_test.cpp
index f158cf0a7b4..c745bdb01f5 100644
--- a/be/test/io/fs/azure_test.cpp
+++ b/be/test/io/fs/azure_test.cpp
@@ -18,10 +18,12 @@
#include <fmt/core.h>
#include <gtest/gtest.h>
+#ifdef USE_AZURE
#include <azure/storage/blobs.hpp>
#include <azure/storage/blobs/blob_client.hpp>
#include <azure/storage/blobs/blob_container_client.hpp>
#include <azure/storage/common/storage_credential.hpp>
+#endif
#include <cstdio>
#include <iostream>
#include <stdexcept>
@@ -46,6 +48,7 @@ std::string GetConnectionString() {
TEST(AzureTest, Write) {
GTEST_SKIP() << "Skipping Azure test, because this test it to test the
compile and linkage";
+#ifdef USE_AZURE
using namespace Azure::Storage::Blobs;
std::string accountName = config::test_s3_ak;
@@ -91,6 +94,7 @@ TEST(AzureTest, Write) {
blobClient.DownloadTo(buffer.data(), buffer.size());
std::cout << std::string(buffer.begin(), buffer.end()) << std::endl;
+#endif
}
} // namespace doris
diff --git a/build.sh b/build.sh
index 14b1244ead7..e2a7f33bd52 100755
--- a/build.sh
+++ b/build.sh
@@ -63,6 +63,7 @@ Usage: $0 <options>
STRIP_DEBUG_INFO If set STRIP_DEBUG_INFO=ON, the debug
information in the compiled binaries will be stored separately in the
'be/lib/debug_info' directory. Default is OFF.
DISABLE_BE_JAVA_EXTENSIONS If set DISABLE_BE_JAVA_EXTENSIONS=ON, we will
do not build binary with java-udf,hudi-scanner,jdbc-scanner and so on Default
is OFF.
DISABLE_JAVA_CHECK_STYLE If set DISABLE_JAVA_CHECK_STYLE=ON, it will
skip style check of java code in FE.
+ DISABLE_BUILD_AZURE If set DISABLE_BUILD_AZURE=ON, it will not
build azure into BE.
Eg.
$0 build all
$0 --be build Backend
@@ -158,6 +159,7 @@ HELP=0
PARAMETER_COUNT="$#"
PARAMETER_FLAG=0
DENABLE_CLANG_COVERAGE='OFF'
+BUILD_AZURE='ON'
BUILD_UI=1
if [[ "$#" == 1 ]]; then
# default
@@ -432,6 +434,10 @@ if [[ -z "${DISABLE_JAVA_CHECK_STYLE}" ]]; then
DISABLE_JAVA_CHECK_STYLE='OFF'
fi
+if [[ -n "${DISABLE_BUILD_AZURE}" ]]; then
+ BUILD_AZURE='OFF'
+fi
+
if [[ -z "${ENABLE_INJECTION_POINT}" ]]; then
ENABLE_INJECTION_POINT='OFF'
fi
@@ -587,6 +593,7 @@ if [[ "${BUILD_BE}" -eq 1 ]]; then
-DEXTRA_CXX_FLAGS="${EXTRA_CXX_FLAGS}" \
-DENABLE_CLANG_COVERAGE="${DENABLE_CLANG_COVERAGE}" \
-DDORIS_JAVA_HOME="${JAVA_HOME}" \
+ -DBUILD_AZURE="${BUILD_AZURE}" \
"${DORIS_HOME}/be"
if [[ "${OUTPUT_BE_BINARY}" -eq 1 ]]; then
@@ -625,6 +632,7 @@ if [[ "${BUILD_CLOUD}" -eq 1 ]]; then
-DUSE_DWARF="${USE_DWARF}" \
-DUSE_JEMALLOC="${USE_JEMALLOC}" \
-DEXTRA_CXX_FLAGS="${EXTRA_CXX_FLAGS}" \
+ -DBUILD_AZURE="${BUILD_AZURE}" \
-DBUILD_CHECK_META="${BUILD_CHECK_META:-OFF}" \
"${DORIS_HOME}/cloud/"
"${BUILD_SYSTEM}" -j "${PARALLEL}"
diff --git a/cloud/CMakeLists.txt b/cloud/CMakeLists.txt
index 576dac9f8d5..801bf26d135 100644
--- a/cloud/CMakeLists.txt
+++ b/cloud/CMakeLists.txt
@@ -347,6 +347,12 @@ else ()
set(MALLOCLIB tcmalloc)
endif()
+option(BUILD_AZURE "ON for building azure support for BE or OFF for not" OFF)
+message(STATUS "build azure: ${BUILD_AZURE}")
+if(BUILD_AZURE STREQUAL "ON")
+ add_definitions(-DUSE_AZURE)
+endif()
+
if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU")
set(ASAN_LIBS -static-libasan)
set(LSAN_LIBS -static-liblsan)
diff --git a/cloud/src/recycler/CMakeLists.txt
b/cloud/src/recycler/CMakeLists.txt
index 882dc8e5429..6dbb8a0d696 100644
--- a/cloud/src/recycler/CMakeLists.txt
+++ b/cloud/src/recycler/CMakeLists.txt
@@ -9,6 +9,10 @@ set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -lfdb_c
-L${THIRDPARTY_DIR
file(GLOB_RECURSE SRC_LIST CONFIGURE_DEPENDS *.cpp)
+if(BUILD_AZURE STREQUAL "OFF")
+ list(REMOVE_ITEM SRC_LIST
"${CMAKE_CURRENT_SOURCE_DIR}/azure_obj_client.cpp")
+endif()
+
if(BUILD_CHECK_META STREQUAL "OFF")
list(REMOVE_ITEM SRC_LIST ${CMAKE_CURRENT_SOURCE_DIR}/meta_checker.cpp)
endif ()
diff --git a/cloud/src/recycler/s3_accessor.cpp
b/cloud/src/recycler/s3_accessor.cpp
index e05ec7c72eb..924b3cbc98c 100644
--- a/cloud/src/recycler/s3_accessor.cpp
+++ b/cloud/src/recycler/s3_accessor.cpp
@@ -25,8 +25,10 @@
#include <gen_cpp/cloud.pb.h>
#include <algorithm>
+#ifdef USE_AZURE
#include <azure/storage/blobs/blob_container_client.hpp>
#include <azure/storage/common/storage_credential.hpp>
+#endif
#include <execution>
#include <memory>
#include <utility>
@@ -39,7 +41,9 @@
#include "common/util.h"
#include "cpp/obj_retry_strategy.h"
#include "cpp/s3_rate_limiter.h"
+#ifdef USE_AZURE
#include "recycler/azure_obj_client.h"
+#endif
#include "recycler/obj_storage_client.h"
#include "recycler/s3_obj_client.h"
#include "recycler/storage_vault_accessor.h"
@@ -212,6 +216,7 @@ int S3Accessor::init() {
});
switch (conf_.provider) {
case S3Conf::AZURE: {
+#ifdef USE_AZURE
Azure::Storage::Blobs::BlobClientOptions options;
options.Retry.StatusCodes.insert(Azure::Core::Http::HttpStatusCode::TooManyRequests);
options.Retry.MaxRetries = config::max_s3_client_retry;
@@ -231,6 +236,10 @@ int S3Accessor::init() {
uri_ = uri_ + '/' + conf_.prefix;
obj_client_ =
std::make_shared<AzureObjClient>(std::move(container_client));
return 0;
+#else
+ LOG_FATAL("BE is not compiled with azure support, export
BUILD_AZURE=ON before building");
+ return 0;
+#endif
}
default: {
uri_ = conf_.endpoint + '/' + conf_.bucket + '/' + conf_.prefix;
diff --git a/common/cpp/obj_retry_strategy.cpp
b/common/cpp/obj_retry_strategy.cpp
index 8461ab39f74..4daa31dc588 100644
--- a/common/cpp/obj_retry_strategy.cpp
+++ b/common/cpp/obj_retry_strategy.cpp
@@ -36,7 +36,7 @@ bool S3CustomRetryStrategy::ShouldRetry(const
Aws::Client::AWSError<Aws::Client:
}
return Aws::Client::DefaultRetryStrategy::ShouldRetry(error,
attemptedRetries);
}
-
+#ifdef USE_AZURE
AzureRetryRecordPolicy::AzureRetryRecordPolicy(int retry_cnt) :
retry_cnt(retry_cnt) {}
AzureRetryRecordPolicy::~AzureRetryRecordPolicy() = default;
@@ -58,4 +58,5 @@ std::unique_ptr<AzureRetryRecordPolicy::HttpPolicy>
AzureRetryRecordPolicy::Clon
ret->retry_cnt = 0;
return ret;
}
+#endif
} // namespace doris
\ No newline at end of file
diff --git a/common/cpp/obj_retry_strategy.h b/common/cpp/obj_retry_strategy.h
index 181762ffdf2..b081ca91a22 100644
--- a/common/cpp/obj_retry_strategy.h
+++ b/common/cpp/obj_retry_strategy.h
@@ -20,7 +20,9 @@
#include <aws/core/client/AWSError.h>
#include <aws/core/client/DefaultRetryStrategy.h>
+#ifdef USE_AZURE
#include <azure/core/http/policies/policy.hpp>
+#endif
namespace doris {
class S3CustomRetryStrategy final : public Aws::Client::DefaultRetryStrategy {
@@ -32,6 +34,7 @@ public:
long attemptedRetries) const override;
};
+#ifdef USE_AZURE
class AzureRetryRecordPolicy final : public
Azure::Core::Http::Policies::HttpPolicy {
public:
AzureRetryRecordPolicy(int retry_cnt);
@@ -45,4 +48,5 @@ public:
private:
mutable int retry_cnt;
};
+#endif
} // namespace doris
\ No newline at end of file
diff --git a/run-be-ut.sh b/run-be-ut.sh
index fe3e9dbd4a5..5f73f6f0ee6 100755
--- a/run-be-ut.sh
+++ b/run-be-ut.sh
@@ -80,6 +80,7 @@ CLEAN=0
RUN=0
BUILD_BENCHMARK_TOOL='OFF'
DENABLE_CLANG_COVERAGE='OFF'
+BUILD_AZURE='ON'
FILTER=""
if [[ "$#" != 1 ]]; then
while true; do
@@ -163,6 +164,10 @@ if [[ "_${DENABLE_CLANG_COVERAGE}" == "_ON" ]]; then
echo "export DORIS_TOOLCHAIN=clang" >>custom_env.sh
fi
+if [[ -n "${DISABLE_BUILD_AZURE}" ]]; then
+ BUILD_AZURE='OFF'
+fi
+
if [[ -z ${CMAKE_BUILD_DIR} ]]; then
CMAKE_BUILD_DIR="${DORIS_HOME}/be/ut_build_${CMAKE_BUILD_TYPE}"
fi
@@ -245,6 +250,7 @@ cd "${CMAKE_BUILD_DIR}"
${CMAKE_USE_CCACHE:+${CMAKE_USE_CCACHE}} \
-DENABLE_PCH="${ENABLE_PCH}" \
-DDORIS_JAVA_HOME="${JAVA_HOME}" \
+ -DBUILD_AZURE="${BUILD_AZURE}" \
"${DORIS_HOME}/be"
"${BUILD_SYSTEM}" -j "${PARALLEL}"
diff --git a/run-cloud-ut.sh b/run-cloud-ut.sh
index 07d4fc4bb5b..60455dd19fb 100755
--- a/run-cloud-ut.sh
+++ b/run-cloud-ut.sh
@@ -86,6 +86,8 @@ RUN=0
FILTER=""
FDB=""
ENABLE_CLANG_COVERAGE=OFF
+BUILD_AZURE="ON"
+
echo "===================== filter: ${FILTER}"
if [[ $# != 1 ]]; then
while true; do
@@ -164,6 +166,10 @@ if [[ -z "${USE_DWARF}" ]]; then
USE_DWARF=OFF
fi
+if [[ -n "${DISABLE_BUILD_AZURE}" ]]; then
+ BUILD_AZURE='OFF'
+fi
+
MAKE_PROGRAM="$(command -v "${BUILD_SYSTEM}")"
echo "-- Make program: ${MAKE_PROGRAM}"
@@ -182,6 +188,7 @@ find . -name "*.gcda" -exec rm {} \;
-DSTRICT_MEMORY_USE=OFF \
-DENABLE_CLANG_COVERAGE="${ENABLE_CLANG_COVERAGE}" \
"${CMAKE_USE_CCACHE}" \
+ -DBUILD_AZURE="${BUILD_AZURE}" \
"${DORIS_HOME}/cloud/"
"${BUILD_SYSTEM}" -j "${PARALLEL}"
"${BUILD_SYSTEM}" install
diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh
index c45fe62e08f..cf0d7576d0d 100755
--- a/thirdparty/build-thirdparty.sh
+++ b/thirdparty/build-thirdparty.sh
@@ -73,6 +73,8 @@ else
PARALLEL="$(($(nproc) / 4 + 1))"
fi
+BUILD_AZURE="ON"
+
while true; do
case "$1" in
-j)
@@ -119,6 +121,10 @@ if [[ "${HELP}" -eq 1 ]]; then
usage
fi
+if [[ -n "${DISABLE_BUILD_AZURE}" ]]; then
+ BUILD_AZURE='OFF'
+fi
+
echo "Get params:
PARALLEL -- ${PARALLEL}
CLEAN -- ${CLEAN}
@@ -1780,21 +1786,25 @@ build_base64() {
# azure blob storage
build_azure() {
- check_if_source_exist "${AZURE_SOURCE}"
- cd "${TP_SOURCE_DIR}/${AZURE_SOURCE}"
- azure_dir=$(pwd)
+ if [[ "${BUILD_AZURE}" == "OFF" ]]; then
+ echo "Skip build azure"
+ else
+ check_if_source_exist "${AZURE_SOURCE}"
+ cd "${TP_SOURCE_DIR}/${AZURE_SOURCE}"
+ azure_dir=$(pwd)
- rm -rf "${BUILD_DIR}"
- mkdir -p "${BUILD_DIR}"
- cd "${BUILD_DIR}"
+ rm -rf "${BUILD_DIR}"
+ mkdir -p "${BUILD_DIR}"
+ cd "${BUILD_DIR}"
- # We need use openssl 1.1.1n, which is already carried in
vcpkg-custom-ports
- AZURE_PORTS="vcpkg-custom-ports"
- AZURE_MANIFEST_DIR="."
+ # We need use openssl 1.1.1n, which is already carried in
vcpkg-custom-ports
+ AZURE_PORTS="vcpkg-custom-ports"
+ AZURE_MANIFEST_DIR="."
- "${CMAKE_CMD}" -G "${GENERATOR}" -DVCPKG_MANIFEST_MODE=ON
-DVCPKG_OVERLAY_PORTS="${azure_dir}/${AZURE_PORTS}"
-DVCPKG_MANIFEST_DIR="${azure_dir}/${AZURE_MANIFEST_DIR}"
-DWARNINGS_AS_ERRORS=FALSE -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}"
-DCMAKE_BUILD_TYPE=Release ..
- "${BUILD_SYSTEM}" -j "${PARALLEL}"
- "${BUILD_SYSTEM}" install
+ "${CMAKE_CMD}" -G "${GENERATOR}" -DVCPKG_MANIFEST_MODE=ON
-DVCPKG_OVERLAY_PORTS="${azure_dir}/${AZURE_PORTS}"
-DVCPKG_MANIFEST_DIR="${azure_dir}/${AZURE_MANIFEST_DIR}"
-DWARNINGS_AS_ERRORS=FALSE -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}"
-DCMAKE_BUILD_TYPE=Release ..
+ "${BUILD_SYSTEM}" -j "${PARALLEL}"
+ "${BUILD_SYSTEM}" install
+ fi
}
# dragonbox
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
