This is an automated email from the ASF dual-hosted git repository. dataroaring pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push: new 4f218b461a9 [Fix](compile) Fix clang-18 and libc++ compile problems (#34715) 4f218b461a9 is described below commit 4f218b461a9c4b775b1ea02d16d9f53c55d34d71 Author: sparrow <38098988+biohazard4...@users.noreply.github.com> AuthorDate: Wed Jul 31 17:59:52 2024 +0800 [Fix](compile) Fix clang-18 and libc++ compile problems (#34715) ## Proposed changes Issue Number: close #xxx Fix clang-18 and libc++ compile problems unique() of shared_ptr is deprecated in clang-18, it's not a strict implementation, can use use_count() instead. And fixed some other compie errors in both using clang-18 and libc++. ## Further comments If this is a relatively large or complex change, kick off the discussion at [d...@doris.apache.org](mailto:d...@doris.apache.org) by explaining why you chose the solution you did and what alternatives you considered, etc... --- be/CMakeLists.txt | 12 +++++++----- be/src/cloud/cloud_tablet_hotspot.cpp | 16 ++++++++-------- be/src/io/cache/block_file_cache.cpp | 2 +- be/test/util/threadpool_test.cpp | 2 ++ cloud/CMakeLists.txt | 6 ++++-- cloud/src/common/encryption_util.cpp | 4 ++-- cloud/src/recycler/checker.cpp | 5 +++-- cloud/src/recycler/checker.h | 3 +++ cloud/src/recycler/s3_accessor.cpp | 4 ++++ cloud/test/codec_test.cpp | 4 ++-- common/cpp/s3_rate_limiter.cpp | 6 ++---- run-cloud-ut.sh | 11 +++++++++++ thirdparty/build-thirdparty.sh | 4 ++-- 13 files changed, 51 insertions(+), 28 deletions(-) diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt index 00d084bbc25..f554ba6053a 100644 --- a/be/CMakeLists.txt +++ b/be/CMakeLists.txt @@ -157,6 +157,7 @@ set(BOOST_VERSION "1.81.0") if (NOT APPLE) find_package(Boost ${BOOST_VERSION} REQUIRED COMPONENTS system date_time) + find_package(Boost ${BOOST_VERSION} REQUIRED COMPONENTS system container) else() find_package(Boost ${BOOST_VERSION} COMPONENTS system date_time) find_package(Boost ${BOOST_VERSION} COMPONENTS system container) @@ -298,12 +299,11 @@ if (COMPILER_CLANG) -Wno-implicit-float-conversion -Wno-implicit-int-conversion -Wno-sign-conversion + -Wno-missing-field-initializers + -Wno-unused-const-variable -Wno-shorten-64-to-32) if (USE_LIBCPP) add_compile_options($<$<COMPILE_LANGUAGE:CXX>:-stdlib=libc++>) - if (NOT OS_MACOSX) - add_compile_options($<$<COMPILE_LANGUAGE:CXX>:-lstdc++>) - endif() add_definitions(-DUSE_LIBCPP) endif() endif () @@ -517,6 +517,7 @@ find_package(absl) # add it here first. set(COMMON_THIRDPARTY Boost::date_time + Boost::container ${COMMON_THIRDPARTY} ) @@ -559,7 +560,6 @@ endif() if (OS_MACOSX) set(COMMON_THIRDPARTY ${COMMON_THIRDPARTY} - Boost::container bfd iberty intl @@ -603,9 +603,11 @@ if (NOT OS_MACOSX) ${DORIS_DEPENDENCIES} -static-libstdc++ -static-libgcc - -lstdc++fs -lresolv ) + if (NOT (USE_LIBCPP AND COMPILER_CLANG)) + set(DORIS_LINK_LIBS ${DORIS_LINK_LIBS} -lstdc++fs) + endif() else() set(DORIS_LINK_LIBS ${DORIS_LINK_LIBS} diff --git a/be/src/cloud/cloud_tablet_hotspot.cpp b/be/src/cloud/cloud_tablet_hotspot.cpp index ae8b3a54d2b..dd197268646 100644 --- a/be/src/cloud/cloud_tablet_hotspot.cpp +++ b/be/src/cloud/cloud_tablet_hotspot.cpp @@ -89,20 +89,20 @@ void TabletHotspot::get_top_n_hot_partition(std::vector<THotTableMessage>* hot_t hot_partition.qpd = std::max(hot_partition.qpd, counter->qpd()); hot_partition.qpw = std::max(hot_partition.qpw, counter->qpw()); hot_partition.last_access_time = - std::max(hot_partition.last_access_time, - std::chrono::duration_cast<std::chrono::seconds>( - counter->last_access_time.time_since_epoch()) - .count()); + std::max<int64_t>(hot_partition.last_access_time, + std::chrono::duration_cast<std::chrono::seconds>( + counter->last_access_time.time_since_epoch()) + .count()); } else if (counter->qpw() != 0) { auto& hot_partition = week_hot_partitions[std::make_pair( counter->table_id, counter->index_id)][counter->partition_id]; hot_partition.qpd = 0; hot_partition.qpw = std::max(hot_partition.qpw, counter->qpw()); hot_partition.last_access_time = - std::max(hot_partition.last_access_time, - std::chrono::duration_cast<std::chrono::seconds>( - counter->last_access_time.time_since_epoch()) - .count()); + std::max<int64_t>(hot_partition.last_access_time, + std::chrono::duration_cast<std::chrono::seconds>( + counter->last_access_time.time_since_epoch()) + .count()); } } }); diff --git a/be/src/io/cache/block_file_cache.cpp b/be/src/io/cache/block_file_cache.cpp index 33858e9ac53..cef9ad17520 100644 --- a/be/src/io/cache/block_file_cache.cpp +++ b/be/src/io/cache/block_file_cache.cpp @@ -157,7 +157,7 @@ void BlockFileCache::remove_query_context(const TUniqueId& query_id) { std::lock_guard cache_lock(_mutex); const auto& query_iter = _query_map.find(query_id); - if (query_iter != _query_map.end() && query_iter->second.unique()) { + if (query_iter != _query_map.end() && query_iter->second.use_count() <= 1) { _query_map.erase(query_iter); } } diff --git a/be/test/util/threadpool_test.cpp b/be/test/util/threadpool_test.cpp index cd33aaaa644..3859639539d 100644 --- a/be/test/util/threadpool_test.cpp +++ b/be/test/util/threadpool_test.cpp @@ -332,6 +332,8 @@ TEST_F(ThreadPoolTest, TestDeadlocks) { const char* death_msg = "doris::ThreadPool::check_not_pool_thread_unlocked()"; #elif defined(__APPLE__) const char* death_msg = "pthread_start"; +#elif defined(__clang__) && defined(USE_LIBCPP) + const char* death_msg = "doris::ThreadPool::check_not_pool_thread_unlocked()"; #else const char* death_msg = "_ZNSt5_BindIFMN5doris10ThreadPoolEFvvEPS1_EE6__callIvJEJLm0EEEET_OSt5tupleIJDpT0_" diff --git a/cloud/CMakeLists.txt b/cloud/CMakeLists.txt index 801bf26d135..ddcf8aab5f9 100644 --- a/cloud/CMakeLists.txt +++ b/cloud/CMakeLists.txt @@ -165,7 +165,7 @@ if (NOT CUSTUM_LINKER_COMMAND STREQUAL "ld") endif() if (USE_LIBCPP AND COMPILER_CLANG) - set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -stdlib=libc++ -lstdc++") + set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -stdlib=libc++") add_definitions(-DUSE_LIBCPP) endif() @@ -335,11 +335,13 @@ set(DORIS_LINK_LIBS ${DORIS_LINK_LIBS} ${DORIS_DEPENDENCIES} -static-libstdc++ -static-libgcc - -lstdc++fs -lresolv -L${DORIS_JAVA_HOME}/lib/server -ljvm ) +if (NOT (USE_LIBCPP AND COMPILER_CLANG)) + set(DORIS_LINK_LIBS ${DORIS_LINK_LIBS} -lstdc++fs) +endif() if (USE_JEMALLOC) set(MALLOCLIB jemalloc) diff --git a/cloud/src/common/encryption_util.cpp b/cloud/src/common/encryption_util.cpp index c28a882b1f4..84cbfc13054 100644 --- a/cloud/src/common/encryption_util.cpp +++ b/cloud/src/common/encryption_util.cpp @@ -632,8 +632,8 @@ static int generate_random_root_key(TxnKv* txn_kv, KmsClient* kms_client, std::s // 3. otherwise, generate a random data key in memory std::mt19937 rnd(time(nullptr)); - std::uniform_int_distribution<char> dist(std::numeric_limits<char>::min(), - std::numeric_limits<char>::max()); + std::uniform_int_distribution<short> dist(std::numeric_limits<char>::min(), + std::numeric_limits<char>::max()); std::string root_key_plaintext(32, '0'); for (char& i : root_key_plaintext) { i = (char)dist(rnd); diff --git a/cloud/src/recycler/checker.cpp b/cloud/src/recycler/checker.cpp index c8f7a955514..49421f97ca0 100644 --- a/cloud/src/recycler/checker.cpp +++ b/cloud/src/recycler/checker.cpp @@ -127,8 +127,9 @@ int Checker::start() { long enqueue_time_s = 0; { std::unique_lock lock(mtx_); - pending_instance_cond_.wait( - lock, [&]() { return !pending_instance_queue_.empty() || stopped(); }); + pending_instance_cond_.wait(lock, [&]() -> bool { + return !pending_instance_queue_.empty() || stopped(); + }); if (stopped()) { return; } diff --git a/cloud/src/recycler/checker.h b/cloud/src/recycler/checker.h index 66bc6912764..4cd851d5218 100644 --- a/cloud/src/recycler/checker.h +++ b/cloud/src/recycler/checker.h @@ -17,6 +17,9 @@ #pragma once +#if defined(USE_LIBCPP) && _LIBCPP_ABI_VERSION <= 1 +#define _LIBCPP_ABI_INCOMPLETE_TYPES_IN_DEQUE +#endif #include <atomic> #include <condition_variable> #include <deque> diff --git a/cloud/src/recycler/s3_accessor.cpp b/cloud/src/recycler/s3_accessor.cpp index 3a051c5d46b..2c983a5fa06 100644 --- a/cloud/src/recycler/s3_accessor.cpp +++ b/cloud/src/recycler/s3_accessor.cpp @@ -415,7 +415,11 @@ int GcsAccessor::delete_prefix_impl(const std::string& path_prefix, int64_t expi int GcsAccessor::delete_files(const std::vector<std::string>& paths) { std::vector<int> delete_rets(paths.size()); +#ifdef USE_LIBCPP + std::transform(paths.begin(), paths.end(), delete_rets.begin(), +#else std::transform(std::execution::par, paths.begin(), paths.end(), delete_rets.begin(), +#endif [this](const std::string& path) { LOG_INFO("delete file").tag("uri", to_uri(path)); return delete_file(path); diff --git a/cloud/test/codec_test.cpp b/cloud/test/codec_test.cpp index 86702efbd8f..94c7a9912d7 100644 --- a/cloud/test/codec_test.cpp +++ b/cloud/test/codec_test.cpp @@ -35,8 +35,8 @@ TEST(CodecTest, StringCodecTest) { std::mt19937 gen(std::random_device("/dev/urandom")()); const int max_len = (2 << 16) + 10086; std::uniform_int_distribution<int> rd_len(0, max_len); - std::uniform_int_distribution<char> rd_char(std::numeric_limits<char>::min(), - std::numeric_limits<char>::max()); + std::uniform_int_distribution<short> rd_char(std::numeric_limits<char>::min(), + std::numeric_limits<char>::max()); int ret = -1; diff --git a/common/cpp/s3_rate_limiter.cpp b/common/cpp/s3_rate_limiter.cpp index 64ee4ce19d8..2d3b21f4a56 100644 --- a/common/cpp/s3_rate_limiter.cpp +++ b/common/cpp/s3_rate_limiter.cpp @@ -25,10 +25,8 @@ #if defined(__APPLE__) #include <ctime> -#define CURRENT_TIME std::chrono::system_clock::now() -#else -#define CURRENT_TIME std::chrono::high_resolution_clock::now() #endif +#define CURRENT_TIME std::chrono::system_clock::now() namespace doris { // Just 10^6. @@ -154,4 +152,4 @@ S3RateLimitType string_to_s3_rate_limit_type(std::string_view value) { } return S3RateLimitType::UNKNOWN; } -} // namespace doris \ No newline at end of file +} // namespace doris diff --git a/run-cloud-ut.sh b/run-cloud-ut.sh index 60455dd19fb..16473b1a2a8 100755 --- a/run-cloud-ut.sh +++ b/run-cloud-ut.sh @@ -162,6 +162,14 @@ if [[ -z "${GLIBC_COMPATIBILITY}" ]]; then GLIBC_COMPATIBILITY=ON fi +if [[ -z "${USE_LIBCPP}" ]]; then + if [[ "$(uname -s)" != 'Darwin' ]]; then + USE_LIBCPP='OFF' + else + USE_LIBCPP='ON' + fi +fi + if [[ -z "${USE_DWARF}" ]]; then USE_DWARF=OFF fi @@ -182,6 +190,7 @@ find . -name "*.gcda" -exec rm {} \; -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \ -DMAKE_TEST=ON \ -DGLIBC_COMPATIBILITY="${GLIBC_COMPATIBILITY}" \ + -DUSE_LIBCPP="${USE_LIBCPP}" \ -DUSE_DWARF="${USE_DWARF}" \ -DUSE_MEM_TRACKER=ON \ -DUSE_JEMALLOC=OFF \ @@ -212,6 +221,8 @@ echo "**********************************" echo " Running MetaService Unit Test " echo "**********************************" +export ASAN_OPTIONS=detect_container_overflow=0 + # test binary output dir cd test # FILTER: binary_name:gtest_filter diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh index 07acd21e44d..55aad76f16c 100755 --- a/thirdparty/build-thirdparty.sh +++ b/thirdparty/build-thirdparty.sh @@ -633,7 +633,7 @@ build_bzip() { check_if_source_exist "${BZIP_SOURCE}" cd "${TP_SOURCE_DIR}/${BZIP_SOURCE}" - make -j "${PARALLEL}" install PREFIX="${TP_INSTALL_DIR}" + make -j "${PARALLEL}" install PREFIX="${TP_INSTALL_DIR}" CFLAGS="-fPIC" } # lzo2 @@ -1744,7 +1744,7 @@ build_libuuid() { check_if_source_exist "${LIBUUID_SOURCE}" cd "${TP_SOURCE_DIR}/${LIBUUID_SOURCE}" CC=gcc ./configure --prefix="${TP_INSTALL_DIR}" --disable-shared --enable-static - make -j "${PARALLEL}" + make -j "${PARALLEL}" CFLAGS="-fPIC" make install } --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org