(doris) branch master updated: [chore](Azure) Print Azure request failed message (#36794)
This is an automated email from the ASF dual-hosted git repository.
plat1ko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 728923a5866 [chore](Azure) Print Azure request failed message (#36794)
728923a5866 is described below
commit 728923a5866357ddbd91b9d8ddef8eb198cb6e30
Author: AlexYue
AuthorDate: Wed Jun 26 13:57:39 2024 +0800
[chore](Azure) Print Azure request failed message (#36794)
Print azure's failed request's http code along with more details.
---
be/src/io/fs/azure_obj_storage_client.cpp | 21 ++---
be/src/io/fs/s3_file_writer.cpp | 16 ++--
2 files changed, 28 insertions(+), 9 deletions(-)
diff --git a/be/src/io/fs/azure_obj_storage_client.cpp
b/be/src/io/fs/azure_obj_storage_client.cpp
index 4bd0d1b7009..231113350f2 100644
--- a/be/src/io/fs/azure_obj_storage_client.cpp
+++ b/be/src/io/fs/azure_obj_storage_client.cpp
@@ -22,6 +22,7 @@
#include
#include
+#include
#include
#include
#include
@@ -60,8 +61,10 @@ ObjectStorageResponse do_azure_client_call(Func f, const
ObjectStoragePathOption
try {
f();
} catch (Azure::Core::RequestFailedException& e) {
-auto msg = fmt::format("Azure request failed because {}, error msg {},
path msg {}",
- e.what(), e.Message,
wrap_object_storage_path_msg(opts));
+auto msg = fmt::format(
+"Azure request failed because {}, error msg {}, http code {},
path msg {}",
+e.what(), e.Message, static_cast(e.StatusCode),
+wrap_object_storage_path_msg(opts));
LOG_WARNING(msg);
return {.status =
convert_to_obj_response(Status::InternalError(std::move(msg))),
.http_code = static_cast(e.StatusCode),
@@ -99,10 +102,12 @@ ObjectStorageUploadResponse
AzureObjStorageClient::upload_part(const ObjectStora
Azure::Core::IO::MemoryBodyStream memory_body(
reinterpret_cast(stream.data()),
stream.size());
// The blockId must be base64 encoded
-auto resp = client.StageBlock(base64_encode_part_num(part_num),
memory_body);
+client.StageBlock(base64_encode_part_num(part_num), memory_body);
} catch (Azure::Core::RequestFailedException& e) {
-auto msg = fmt::format("Azure request failed because {}, error msg {},
path msg {}",
- e.what(), e.Message,
wrap_object_storage_path_msg(opts));
+auto msg = fmt::format(
+"Azure request failed because {}, error msg {}, http code {},
path msg {}",
+e.what(), e.Message, static_cast(e.StatusCode),
+wrap_object_storage_path_msg(opts));
LOG_WARNING(msg);
// clang-format off
return {
@@ -142,8 +147,10 @@ ObjectStorageHeadResponse
AzureObjStorageClient::head_object(const ObjectStorage
.request_id = std::move(e.RequestId)},
};
}
-auto msg = fmt::format("Failed to head azure blob due to {}, path msg
{}", e.Message,
- wrap_object_storage_path_msg(opts));
+auto msg = fmt::format(
+"Azure request failed because {}, error msg {}, http code {},
path msg {}",
+e.what(), e.Message, static_cast(e.StatusCode),
+wrap_object_storage_path_msg(opts));
return ObjectStorageHeadResponse {
.resp = {.status = convert_to_obj_response(
Status::InternalError(std::move(msg))),
diff --git a/be/src/io/fs/s3_file_writer.cpp b/be/src/io/fs/s3_file_writer.cpp
index 9af34ea8ca8..20c616ef90a 100644
--- a/be/src/io/fs/s3_file_writer.cpp
+++ b/be/src/io/fs/s3_file_writer.cpp
@@ -290,15 +290,21 @@ Status S3FileWriter::appendv(const Slice* data, size_t
data_cnt) {
void S3FileWriter::_upload_one_part(int64_t part_num, UploadFileBuffer& buf) {
if (buf.is_cancelled()) {
+LOG_INFO("file {} skip part {} because previous failure {}",
+ _obj_storage_path_opts.path.native(), part_num, _st);
return;
}
const auto& client = _obj_client->get();
if (nullptr == client) {
+LOG_WARNING("failed at key: {}, load part {} bacause of invalid obj
client",
+_obj_storage_path_opts.key, part_num);
buf.set_status(Status::InternalError("invalid obj storage
client"));
return;
}
auto resp = client->upload_part(_obj_storage_path_opts,
buf.get_string_view_data(), part_num);
if (resp.resp.status.code != ErrorCode::OK) {
+LOG_INFO("failed at key: {}, load part {}, st {}",
_obj_storage_path_opts.key, part_num,
+ resp.resp.stat
(doris) branch master updated: [feature](Cloud) Introduce obj storage client interface to recycler (#35447)
This is an automated email from the ASF dual-hosted git repository.
plat1ko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 45661f941e8 [feature](Cloud) Introduce obj storage client interface to
recycler (#35447)
45661f941e8 is described below
commit 45661f941e80265039429b26bf8bb6ef8ec410d5
Author: AlexYue
AuthorDate: Mon Jun 3 20:12:14 2024 +0800
[feature](Cloud) Introduce obj storage client interface to recycler (#35447)
Extract basic interface to suite different kinds of ObjectStorage.
---
be/src/common/status.h | 18 +-
be/src/io/fs/err_utils.cpp | 7 +
be/src/io/fs/obj_storage_client.h | 24 +-
be/src/io/fs/s3_file_reader.cpp| 5 +-
be/src/io/fs/s3_file_system.cpp| 73 ++--
be/src/io/fs/s3_file_writer.cpp| 16 +-
be/src/io/fs/s3_obj_storage_client.cpp | 112 --
be/src/io/fs/s3_obj_storage_client.h | 7 +-
be/test/io/fs/s3_file_writer_test.cpp | 17 +-
cloud/src/recycler/obj_store_accessor.h| 57 +++
cloud/src/recycler/s3_accessor.cpp | 389 -
cloud/src/recycler/s3_accessor.h | 6 +-
cloud/src/recycler/s3_obj_client.cpp | 371
.../src/recycler/s3_obj_client.h | 48 ++-
cloud/test/s3_accessor_test.cpp| 18 +
15 files changed, 715 insertions(+), 453 deletions(-)
diff --git a/be/src/common/status.h b/be/src/common/status.h
index 137a268b5aa..34e13749165 100644
--- a/be/src/common/status.h
+++ b/be/src/common/status.h
@@ -25,6 +25,14 @@
namespace doris {
+namespace io {
+struct ObjectStorageStatus;
+}
+
+class Status;
+
+extern io::ObjectStorageStatus convert_to_obj_response(Status st);
+
class PStatus;
namespace ErrorCode {
@@ -352,11 +360,11 @@ public:
Status() : _code(ErrorCode::OK), _err_msg(nullptr) {}
// used to convert Exception to Status
-Status(int code, std::string msg, std::string stack) : _code(code) {
+Status(int code, std::string msg, std::string stack = "") : _code(code) {
_err_msg = std::make_unique();
-_err_msg->_msg = msg;
+_err_msg->_msg = std::move(msg);
#ifdef ENABLE_STACKTRACE
-_err_msg->_stack = stack;
+_err_msg->_stack = std::move(stack);
#endif
}
@@ -529,6 +537,10 @@ public:
std::string_view msg() const { return _err_msg ? _err_msg->_msg :
std::string_view(""); }
+std::pair retrieve_error_msg() { return {_code,
std::move(_err_msg->_msg)}; }
+
+friend io::ObjectStorageStatus convert_to_obj_response(Status st);
+
private:
int _code;
struct ErrMsg {
diff --git a/be/src/io/fs/err_utils.cpp b/be/src/io/fs/err_utils.cpp
index 8552c647cdd..6552d454824 100644
--- a/be/src/io/fs/err_utils.cpp
+++ b/be/src/io/fs/err_utils.cpp
@@ -27,10 +27,17 @@
#include "common/status.h"
#include "io/fs/hdfs.h"
+#include "io/fs/obj_storage_client.h"
namespace doris {
using namespace ErrorCode;
+io::ObjectStorageStatus convert_to_obj_response(Status st) {
+int code = st._code;
+std::string msg = st._err_msg == nullptr ? "" :
std::move(st._err_msg->_msg);
+return io::ObjectStorageStatus {.code = code, .msg = std::move(msg)};
+}
+
namespace io {
std::string errno_to_str() {
diff --git a/be/src/io/fs/obj_storage_client.h
b/be/src/io/fs/obj_storage_client.h
index 40e0ff9a8fe..3ab0a8e2dea 100644
--- a/be/src/io/fs/obj_storage_client.h
+++ b/be/src/io/fs/obj_storage_client.h
@@ -46,14 +46,26 @@ struct ObjectStoragePathOptions {
struct ObjectCompleteMultiParts {};
+struct ObjectStorageStatus {
+int code = 0;
+std::string msg = std::string();
+};
+
+// We only store error code along with err_msg instead of Status to unify BE
and recycler's error handle logic
struct ObjectStorageResponse {
-Status status = Status::OK();
+ObjectStorageStatus status {};
+int http_code {200};
+std::string request_id = std::string();
+};
+
+struct ObjectStorageUploadResponse {
+ObjectStorageResponse resp {};
std::optional upload_id = std::nullopt;
std::optional etag = std::nullopt;
};
struct ObjectStorageHeadResponse {
-Status status = Status::OK();
+ObjectStorageResponse resp {};
long long file_size {0};
};
@@ -62,7 +74,8 @@ public:
virtual ~ObjStorageClient() = default;
// Create a multi-part upload request. On AWS-compatible systems, it will
return an upload ID, but not on Azure.
// The input parameters should include the bucket and key for the object
storage.
-virtual ObjectStorageResponse create_multipart_upload(const
ObjectStoragePathOptions& opts)
(doris) branch master updated (43d0f191d93 -> 10ad1c73d9b)
This is an automated email from the ASF dual-hosted git repository. plat1ko pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/doris.git from 43d0f191d93 [opt](mtmv) Materialized view partition track supports date_trunc and optimize the fail reason (#35562) add 10ad1c73d9b [enhance](BE) Avoid memcpy in `OlapMeta::iterate` (#35850) No new revisions were added by this update. Summary of changes: be/src/http/action/restore_tablet_action.cpp | 2 +- be/src/olap/data_dir.cpp | 23 ++--- be/src/olap/olap_meta.cpp| 23 +++-- be/src/olap/olap_meta.h | 13 +++- be/src/olap/rowset/rowset_meta.cpp | 15 - be/src/olap/rowset/rowset_meta.h | 4 +-- be/src/olap/rowset/rowset_meta_manager.cpp | 48 +--- be/src/olap/rowset/rowset_meta_manager.h | 4 +-- be/src/olap/storage_engine.cpp | 12 +++ be/src/olap/tablet.cpp | 4 +-- be/src/olap/tablet_manager.cpp | 3 +- be/src/olap/tablet_manager.h | 3 +- be/src/olap/tablet_meta.cpp | 4 +-- be/src/olap/tablet_meta.h| 2 +- be/src/olap/tablet_meta_manager.cpp | 30 - be/src/olap/tablet_meta_manager.h| 25 --- be/src/olap/utils.h | 8 ++--- be/test/olap/olap_meta_test.cpp | 2 +- be/test/olap/tablet_meta_manager_test.cpp| 4 +-- 19 files changed, 113 insertions(+), 116 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
(doris) branch master updated: [enhance](S3) Don't use aws's multithread utility in S3 FS to suite newer C++ compiler (#38539)
This is an automated email from the ASF dual-hosted git repository.
plat1ko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new bb433c73257 [enhance](S3) Don't use aws's multithread utility in S3 FS
to suite newer C++ compiler (#38539)
bb433c73257 is described below
commit bb433c73257bca314ce29f5cfa47d4a184fef0eb
Author: AlexYue
AuthorDate: Wed Jul 31 13:23:28 2024 +0800
[enhance](S3) Don't use aws's multithread utility in S3 FS to suite newer
C++ compiler (#38539)
---
be/src/common/config.cpp | 3 ++-
be/src/common/config.h | 3 ++-
be/src/io/fs/s3_file_system.cpp | 23 ++-
be/src/runtime/exec_env.h| 2 ++
be/src/runtime/exec_env_init.cpp | 6 ++
5 files changed, 22 insertions(+), 15 deletions(-)
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index f984621ec85..d0a2a5fa7e3 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -938,7 +938,8 @@ DEFINE_mInt32(cold_data_compaction_interval_sec, "1800");
DEFINE_String(tmp_file_dir, "tmp");
-DEFINE_Int32(s3_transfer_executor_pool_size, "2");
+DEFINE_Int32(min_s3_file_system_thread_num, "16");
+DEFINE_Int32(max_s3_file_system_thread_num, "64");
DEFINE_Bool(enable_time_lut, "true");
DEFINE_mBool(enable_simdjson_reader, "true");
diff --git a/be/src/common/config.h b/be/src/common/config.h
index fcfce74e7be..e117c824329 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -995,7 +995,8 @@ DECLARE_mInt32(confirm_unused_remote_files_interval_sec);
DECLARE_Int32(cold_data_compaction_thread_num);
DECLARE_mInt32(cold_data_compaction_interval_sec);
-DECLARE_Int32(s3_transfer_executor_pool_size);
+DECLARE_Int32(min_s3_file_system_thread_num);
+DECLARE_Int32(max_s3_file_system_thread_num);
DECLARE_Bool(enable_time_lut);
DECLARE_mBool(enable_simdjson_reader);
diff --git a/be/src/io/fs/s3_file_system.cpp b/be/src/io/fs/s3_file_system.cpp
index 93f36429485..3905c4ddb1e 100644
--- a/be/src/io/fs/s3_file_system.cpp
+++ b/be/src/io/fs/s3_file_system.cpp
@@ -18,9 +18,8 @@
#include "io/fs/s3_file_system.h"
#include
-#include
-#include
+#include
#include "common/compiler_util.h" // IWYU pragma: keep
// IWYU pragma: no_include
@@ -32,7 +31,6 @@
#include // IWYU pragma: keep
#include
#include
-#include
#include "common/config.h"
#include "common/logging.h"
@@ -46,7 +44,7 @@
#include "io/fs/s3_file_reader.h"
#include "io/fs/s3_file_writer.h"
#include "io/fs/s3_obj_storage_client.h"
-#include "util/bvar_helper.h"
+#include "runtime/exec_env.h"
#include "util/s3_uri.h"
#include "util/s3_util.h"
@@ -69,13 +67,6 @@ Result get_key(const Path& full_path) {
return uri.get_key();
}
-// TODO(plat1ko): AwsTransferManager will be deprecated
-std::shared_ptr&
default_executor() {
-static auto executor =
Aws::MakeShared(
-"default", config::s3_transfer_executor_pool_size);
-return executor;
-}
-
} // namespace
ObjClientHolder::ObjClientHolder(S3ClientConf conf) : _conf(std::move(conf)) {}
@@ -383,13 +374,19 @@ Status S3FileSystem::batch_upload_impl(const
std::vector& local_files,
return Status::OK();
};
+Status s = Status::OK();
std::vector> futures;
for (int i = 0; i < local_files.size(); ++i) {
auto task = std::make_shared>(upload_task);
futures.emplace_back(task->get_future());
-default_executor()->Submit([t = std::move(task), idx = i]() mutable {
(*t)(idx); });
+auto st =
ExecEnv::GetInstance()->s3_file_system_thread_pool()->submit_func(
+[t = std::move(task), idx = i]() mutable { (*t)(idx); });
+// We shouldn't return immediately since the previous submitted tasks
might still be running in the thread pool
+if (!st.ok()) {
+s = st;
+break;
+}
}
-Status s = Status::OK();
for (auto&& f : futures) {
auto cur_s = f.get();
if (!cur_s.ok()) {
diff --git a/be/src/runtime/exec_env.h b/be/src/runtime/exec_env.h
index 89e5593c84b..65cf70bf568 100644
--- a/be/src/runtime/exec_env.h
+++ b/be/src/runtime/exec_env.h
@@ -204,6 +204,7 @@ public:
ThreadPool* join_node_thread_pool() { return _join_node_thread_pool.get();
}
ThreadPool* lazy_release_obj_pool() { return _lazy_release_obj_pool.get();
}
ThreadPool* non_block_close_thread_pool();
+ThreadPool* s3_file_system_thread_pool() { return
_s3_file_system_thread_pool.get(); }
Status init_pipeline_task_scheduler();
void init_file_cache_factory();
@@ -381,6 +382,7 @@ private:
// Pool t
(doris) branch master updated: [enhance](Vault) Refresh default storage vault info when doing instance check in FE (#36104)
This is an automated email from the ASF dual-hosted git repository.
plat1ko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new c24891069b1 [enhance](Vault) Refresh default storage vault info when
doing instance check in FE (#36104)
c24891069b1 is described below
commit c24891069b19747167ea445b49879539d43a130b
Author: AlexYue
AuthorDate: Wed Jun 12 20:57:25 2024 +0800
[enhance](Vault) Refresh default storage vault info when doing instance
check in FE (#36104)
---
.../org/apache/doris/cloud/catalog/CloudInstanceStatusChecker.java | 5 +
1 file changed, 5 insertions(+)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudInstanceStatusChecker.java
b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudInstanceStatusChecker.java
index 2fcb5c57cd2..9342c705e9d 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudInstanceStatusChecker.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudInstanceStatusChecker.java
@@ -17,9 +17,11 @@
package org.apache.doris.cloud.catalog;
+import org.apache.doris.catalog.Env;
import org.apache.doris.cloud.proto.Cloud;
import org.apache.doris.cloud.system.CloudSystemInfoService;
import org.apache.doris.common.Config;
+import org.apache.doris.common.Pair;
import org.apache.doris.common.util.MasterDaemon;
import org.apache.logging.log4j.LogManager;
@@ -47,6 +49,9 @@ public class CloudInstanceStatusChecker extends MasterDaemon {
+ "cloud_unique_id={}, response={}",
Config.cloud_unique_id, response);
} else {
cloudSystemInfoService.setInstanceStatus(response.getInstance().getStatus());
+
Env.getCurrentEnv().getStorageVaultMgr().setDefaultStorageVault(
+
Pair.of(response.getInstance().getDefaultStorageVaultName(),
+
response.getInstance().getDefaultStorageVaultId()));
}
} catch (Exception e) {
LOG.warn("get instance from ms exception", e);
-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
(doris) branch master updated: Fix use-after-free during graceful shutdown (#33457)
This is an automated email from the ASF dual-hosted git repository.
plat1ko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 6d441baf70c Fix use-after-free during graceful shutdown (#33457)
6d441baf70c is described below
commit 6d441baf70c0c2f5968e36209c67221c9964daaf
Author: plat1ko
AuthorDate: Tue Apr 16 22:50:13 2024 +0800
Fix use-after-free during graceful shutdown (#33457)
---
be/src/olap/olap_server.cpp | 27 ---
be/src/olap/storage_engine.h | 3 +++
2 files changed, 15 insertions(+), 15 deletions(-)
diff --git a/be/src/olap/olap_server.cpp b/be/src/olap/olap_server.cpp
index aaa1e1dd7ac..73bd0e37f81 100644
--- a/be/src/olap/olap_server.cpp
+++ b/be/src/olap/olap_server.cpp
@@ -1304,9 +1304,6 @@ void StorageEngine::do_remove_unused_remote_files() {
}
void StorageEngine::_cold_data_compaction_producer_callback() {
-std::unordered_set tablet_submitted;
-std::mutex tablet_submitted_mtx;
-
while (!_stop_background_threads_latch.wait_for(
std::chrono::seconds(config::cold_data_compaction_interval_sec))) {
if (config::disable_auto_compaction ||
@@ -1316,8 +1313,8 @@ void
StorageEngine::_cold_data_compaction_producer_callback() {
std::unordered_set copied_tablet_submitted;
{
-std::lock_guard lock(tablet_submitted_mtx);
-copied_tablet_submitted = tablet_submitted;
+std::lock_guard lock(_cold_compaction_tablet_submitted_mtx);
+copied_tablet_submitted = _cold_compaction_tablet_submitted;
}
int n = config::cold_data_compaction_thread_num -
copied_tablet_submitted.size();
if (n <= 0) {
@@ -1326,7 +1323,7 @@ void
StorageEngine::_cold_data_compaction_producer_callback() {
auto tablets =
_tablet_manager->get_all_tablet([&copied_tablet_submitted](Tablet* t) {
return t->tablet_meta()->cooldown_meta_id().initialized() &&
t->is_used() &&
t->tablet_state() == TABLET_RUNNING &&
- !copied_tablet_submitted.count(t->tablet_id()) &&
+ !copied_tablet_submitted.contains(t->tablet_id()) &&
!t->tablet_meta()->tablet_schema()->disable_auto_compaction();
});
std::vector> tablet_to_compact;
@@ -1351,7 +1348,7 @@ void
StorageEngine::_cold_data_compaction_producer_callback() {
// else, need to follow
{
std::lock_guard lock(_running_cooldown_mutex);
-if (_running_cooldown_tablets.count(t->table_id())) {
+if (_running_cooldown_tablets.contains(t->table_id())) {
// already in cooldown queue
continue;
}
@@ -1374,12 +1371,12 @@ void
StorageEngine::_cold_data_compaction_producer_callback() {
[&, t = std::move(tablet), this]() {
auto compaction =
std::make_shared(*this, t);
{
-std::lock_guard lock(tablet_submitted_mtx);
-tablet_submitted.insert(t->tablet_id());
+std::lock_guard
lock(_cold_compaction_tablet_submitted_mtx);
+
_cold_compaction_tablet_submitted.insert(t->tablet_id());
}
Defer defer {[&] {
-std::lock_guard lock(tablet_submitted_mtx);
-tablet_submitted.erase(t->tablet_id());
+std::lock_guard
lock(_cold_compaction_tablet_submitted_mtx);
+
_cold_compaction_tablet_submitted.erase(t->tablet_id());
}};
std::unique_lock
cold_compaction_lock(t->get_cold_compaction_lock(),
std::try_to_lock);
@@ -1412,13 +1409,13 @@ void
StorageEngine::_cold_data_compaction_producer_callback() {
t = std::move(
tablet)]() {
{
-std::lock_guard lock(tablet_submitted_mtx);
-tablet_submitted.insert(t->tablet_id());
+std::lock_guard
lock(_cold_compaction_tablet_submitted_mtx);
+_cold_compaction_tablet_submitted.insert(t->tablet_id());
}
auto st = t->cooldown();
{
-std::lock_guard lock(tablet_submitted_mtx);
-tablet_submitted.erase(t->tablet_id());
+std::lock_guar
(doris) branch master updated: [enhance](cloud_p0) morecases for cloud_p0 (#32652)
This is an automated email from the ASF dual-hosted git repository. plat1ko pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/doris.git The following commit(s) were added to refs/heads/master by this push: new 031a4c039a6 [enhance](cloud_p0) morecases for cloud_p0 (#32652) 031a4c039a6 is described below commit 031a4c039a6d7ca10fa4f8b393d6dc95a908f09a Author: Yongqiang YANG <98214048+dataroar...@users.noreply.github.com> AuthorDate: Thu Apr 4 00:52:47 2024 +0800 [enhance](cloud_p0) morecases for cloud_p0 (#32652) --- .../pipeline/cloud_p0/conf/regression-conf-custom.groovy | 8 +--- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/regression-test/pipeline/cloud_p0/conf/regression-conf-custom.groovy b/regression-test/pipeline/cloud_p0/conf/regression-conf-custom.groovy index 501d5c74e9b..ec3c2fd8719 100644 --- a/regression-test/pipeline/cloud_p0/conf/regression-conf-custom.groovy +++ b/regression-test/pipeline/cloud_p0/conf/regression-conf-custom.groovy @@ -16,23 +16,20 @@ // under the License. testGroups = "p0" + // exclude groups and exclude suites is more prior than include groups and include suites. // keep them in lexico order(add/remove cases between the sentinals and sort): // * sort lines in vim: select lines and then type :sort // * sort lines in vscode: https://ulfschneider.io/2023-09-01-sort-in-vscode/ excludeSuites = "000_the_start_sentinel_do_not_touch," + // keep this line as the first line -"set_and_unset_variable," + "set_replica_status," + // not a case for cloud mode, no need to run "test_be_inject_publish_txn_fail," + // not a case for cloud mode, no need to run "test_bitmap_filter," + "test_compaction_uniq_cluster_keys_with_delete," + "test_compaction_uniq_keys_cluster_key," + -"test_disable_move_memtable," + "test_dump_image," + "test_index_failure_injection," + "test_information_schema_external," + -"test_insert_move_memtable," + -"test_materialized_view_move_memtable," + "test_pk_uk_case_cluster," + "test_point_query_cluster_key," + "test_profile," + @@ -42,14 +39,11 @@ excludeSuites = "000_the_start_sentinel_do_not_touch," + // keep this line as th "test_set_partition_version," + "test_show_transaction," + // not supported yet "test_spark_load," + -"test_stream_load_move_memtable," + -"test_stream_load_new_move_memtable," + "test_array_index1," + "test_array_index2," + "test_index_lowercase_fault_injection," + "zzz_the_end_sentinel_do_not_touch" // keep this line as the last line - excludeDirectories = "000_the_start_sentinel_do_not_touch," + // keep this line as the first line "cloud/multi_cluster," + // run in specific regression pipeline "workload_manager_p1," + - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
(doris) branch master updated: [enhance](Cloud) Add case to test if vault is forbid for cloud mode without vault (#34006)
This is an automated email from the ASF dual-hosted git repository.
plat1ko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 3aa89110bc6 [enhance](Cloud) Add case to test if vault is forbid for
cloud mode without vault (#34006)
3aa89110bc6 is described below
commit 3aa89110bc6e459b81ead622a9ff794e8073ac2f
Author: AlexYue
AuthorDate: Thu Apr 25 14:17:48 2024 +0800
[enhance](Cloud) Add case to test if vault is forbid for cloud mode without
vault (#34006)
---
.../doris/analysis/ShowStorageVaultStmt.java | 12 ++
regression-test/suites/vaults/forbid/forbid.groovy | 50 ++
2 files changed, 62 insertions(+)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowStorageVaultStmt.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowStorageVaultStmt.java
index e4af07e43b1..ddf25292846 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowStorageVaultStmt.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowStorageVaultStmt.java
@@ -19,9 +19,12 @@ package org.apache.doris.analysis;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.StorageVault;
+import org.apache.doris.cloud.catalog.CloudEnv;
import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.Config;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.ErrorReport;
+import org.apache.doris.common.FeConstants;
import org.apache.doris.common.UserException;
import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.qe.ConnectContext;
@@ -41,6 +44,15 @@ public class ShowStorageVaultStmt extends ShowStmt {
@Override
public void analyze(Analyzer analyzer) throws AnalysisException,
UserException {
+if (Config.isNotCloudMode()) {
+throw new AnalysisException("Storage Vault is only supported for
cloud mode");
+}
+if (!FeConstants.runningUnitTest) {
+// In legacy cloud mode, some s3 back-ended storage does need to
use storage vault.
+if (!((CloudEnv) Env.getCurrentEnv()).getEnableStorageVault()) {
+throw new AnalysisException("Your cloud instance doesn't
support storage vault");
+}
+}
super.analyze(analyzer);
// check auth
if
(!Env.getCurrentEnv().getAccessManager().checkGlobalPriv(ConnectContext.get(),
PrivPredicate.ADMIN)) {
diff --git a/regression-test/suites/vaults/forbid/forbid.groovy
b/regression-test/suites/vaults/forbid/forbid.groovy
new file mode 100644
index 000..15fba18fc6d
--- /dev/null
+++ b/regression-test/suites/vaults/forbid/forbid.groovy
@@ -0,0 +1,50 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("forbid_vault") {
+if (enableStoragevault()) {
+logger.info("skip forbid storage vault case because storage vault
enabled")
+return
+}
+
+if (!isCloudMode()) {
+logger.info("skip forbid storage vault case because not cloud mode")
+return
+}
+
+expectExceptionLike({
+sql """
+set not_exist as default storage vault
+"""
+}, "Your cloud instance doesn't support storage vault")
+
+expectExceptionLike({
+sql """
+CREATE STORAGE VAULT IF NOT EXISTS hdfs_vault
+PROPERTIES (
+"type"="hdfs",
+"fs.defaultFS"="hdfs://127.0.0.1:8020"
+);
+"""
+}, "Your cloud instance doesn't support storage vault")
+
+expectExceptionLike({
+sql """
+show storage vault
+"""
+}, "Your cloud instance doesn't support storage vault")
+}
\ No newline at end of file
-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
(doris) branch master updated: decrypt sk (#34174)
This is an automated email from the ASF dual-hosted git repository.
plat1ko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new dc09fab881f decrypt sk (#34174)
dc09fab881f is described below
commit dc09fab881fd2d6145cd768ba351510ff19e6caf
Author: AlexYue
AuthorDate: Fri Apr 26 20:31:57 2024 +0800
decrypt sk (#34174)
---
be/src/util/s3_util.h| 4 ++--
cloud/src/meta-service/meta_service_resource.cpp | 8
2 files changed, 10 insertions(+), 2 deletions(-)
diff --git a/be/src/util/s3_util.h b/be/src/util/s3_util.h
index 62d8b4f1d64..46226b79359 100644
--- a/be/src/util/s3_util.h
+++ b/be/src/util/s3_util.h
@@ -132,8 +132,8 @@ struct S3Conf {
cloud::ObjectStoreInfoPB::Provider provider;
std::string to_string() const {
-return fmt::format("(bucket={}, prefix={}, client_conf={})", bucket,
prefix,
- client_conf.to_string());
+return fmt::format("(bucket={}, prefix={}, client_conf={},
sse_enabled={})", bucket, prefix,
+ client_conf.to_string(), sse_enabled);
}
};
diff --git a/cloud/src/meta-service/meta_service_resource.cpp
b/cloud/src/meta-service/meta_service_resource.cpp
index 1dd04617cc4..0adb3f0ac7e 100644
--- a/cloud/src/meta-service/meta_service_resource.cpp
+++ b/cloud/src/meta-service/meta_service_resource.cpp
@@ -277,6 +277,14 @@ void
MetaServiceImpl::get_obj_store_info(google::protobuf::RpcController* contro
storage_vault_start.push_back('\x00'); // Update to next smallest
key for iteration
} while (it->more());
}
+for (auto& vault : *response->mutable_storage_vault()) {
+if (vault.has_obj_info()) {
+if (auto ret = decrypt_and_update_ak_sk(*vault.mutable_obj_info(),
code, msg);
+ret != 0) {
+return;
+}
+}
+}
response->mutable_obj_info()->CopyFrom(instance.obj_info());
if (instance.has_default_storage_vault_id()) {
-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
(doris) branch master updated: [enhance](Cloud) Unify add storage vault http action (#34119)
This is an automated email from the ASF dual-hosted git repository.
plat1ko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 4b58fc95097 [enhance](Cloud) Unify add storage vault http action
(#34119)
4b58fc95097 is described below
commit 4b58fc95097f853e66c919065f63d14756549899
Author: AlexYue
AuthorDate: Fri Apr 26 20:32:42 2024 +0800
[enhance](Cloud) Unify add storage vault http action (#34119)
---
cloud/src/meta-service/meta_service_http.cpp | 8 +++-
cloud/src/meta-service/meta_service_resource.cpp | 3 +++
gensrc/proto/cloud.proto | 1 +
3 files changed, 11 insertions(+), 1 deletion(-)
diff --git a/cloud/src/meta-service/meta_service_http.cpp
b/cloud/src/meta-service/meta_service_http.cpp
index 80ae08ef251..4542e05e486 100644
--- a/cloud/src/meta-service/meta_service_http.cpp
+++ b/cloud/src/meta-service/meta_service_http.cpp
@@ -209,7 +209,10 @@ static HttpResponse
process_alter_obj_store_info(MetaServiceImpl* service, brpc:
static std::unordered_map operations {
{"add_obj_info", AlterObjStoreInfoRequest::ADD_OBJ_INFO},
{"legacy_update_ak_sk",
AlterObjStoreInfoRequest::LEGACY_UPDATE_AK_SK},
-{"drop_hdfs_vault", AlterObjStoreInfoRequest::DROP_HDFS_INFO}};
+{"drop_s3_vault", AlterObjStoreInfoRequest::DROP_S3_VAULT},
+{"add_s3_vault", AlterObjStoreInfoRequest::ADD_S3_VAULT},
+{"drop_hdfs_vault", AlterObjStoreInfoRequest::DROP_HDFS_INFO},
+{"add_hdfs_vault", AlterObjStoreInfoRequest::ADD_HDFS_INFO}};
auto& path = ctrl->http_request().unresolved_path();
auto it = operations.find(remove_version_prefix(path));
@@ -444,6 +447,9 @@ void
MetaServiceImpl::http(::google::protobuf::RpcController* controller,
{"v1/legacy_update_ak_sk", process_alter_obj_store_info},
{"v1/update_ak_sk", process_update_ak_sk},
{"show_storage_vaults", process_get_obj_store_info},
+{"add_hdfs_vault", process_alter_obj_store_info},
+{"add_s3_vault", process_alter_obj_store_info},
+{"drop_s3_vault", process_alter_obj_store_info},
{"drop_hdfs_vault", process_alter_obj_store_info},
// for tools
{"decode_key", process_decode_key},
diff --git a/cloud/src/meta-service/meta_service_resource.cpp
b/cloud/src/meta-service/meta_service_resource.cpp
index 0adb3f0ac7e..eefd01eb254 100644
--- a/cloud/src/meta-service/meta_service_resource.cpp
+++ b/cloud/src/meta-service/meta_service_resource.cpp
@@ -537,6 +537,7 @@ void
MetaServiceImpl::alter_obj_store_info(google::protobuf::RpcController* cont
switch (request->op()) {
case AlterObjStoreInfoRequest::ADD_OBJ_INFO:
case AlterObjStoreInfoRequest::ADD_S3_VAULT:
+case AlterObjStoreInfoRequest::DROP_S3_VAULT:
case AlterObjStoreInfoRequest::LEGACY_UPDATE_AK_SK:
case AlterObjStoreInfoRequest::UPDATE_AK_SK: {
if (!request->has_obj() && (!request->has_vault() ||
!request->vault().has_obj_info())) {
@@ -824,6 +825,8 @@ void
MetaServiceImpl::alter_obj_store_info(google::protobuf::RpcController* cont
instance.clear_default_storage_vault_name();
break;
}
+case AlterObjStoreInfoRequest::DROP_S3_VAULT:
+[[fallthrough]];
default: {
code = MetaServiceCode::INVALID_ARGUMENT;
ss << "invalid request op, op=" << request->op();
diff --git a/gensrc/proto/cloud.proto b/gensrc/proto/cloud.proto
index f132e2eab4d..ba9017a5ecc 100644
--- a/gensrc/proto/cloud.proto
+++ b/gensrc/proto/cloud.proto
@@ -775,6 +775,7 @@ message AlterObjStoreInfoRequest {
DROP_HDFS_INFO = 101;
ADD_BUILT_IN_VAULT = 102;
ADD_S3_VAULT = 103;
+DROP_S3_VAULT = 104;
SET_DEFAULT_VAULT = 200;
UNSET_DEFAULT_VAULT = 201;
-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
(doris) branch master updated: [chore](InjectPoint) Log when entering injection point #34286
This is an automated email from the ASF dual-hosted git repository.
plat1ko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new cb8f9e81291 [chore](InjectPoint) Log when entering injection point
#34286
cb8f9e81291 is described below
commit cb8f9e812913d4176da48a98730a2c6f28f97c44
Author: AlexYue
AuthorDate: Mon Apr 29 23:19:30 2024 +0800
[chore](InjectPoint) Log when entering injection point #34286
---
be/src/cloud/cloud_meta_mgr.cpp | 2 +-
be/src/common/sync_point.h | 12 ++--
be/src/http/action/config_action.cpp | 2 +-
be/src/io/fs/hdfs_file_writer.cpp| 10 +-
4 files changed, 13 insertions(+), 13 deletions(-)
diff --git a/be/src/cloud/cloud_meta_mgr.cpp b/be/src/cloud/cloud_meta_mgr.cpp
index 32f99228088..506c57672d9 100644
--- a/be/src/cloud/cloud_meta_mgr.cpp
+++ b/be/src/cloud/cloud_meta_mgr.cpp
@@ -131,7 +131,7 @@ bvar::LatencyRecorder
g_cloud_commit_txn_resp_redirect_latency("cloud_table_stat
class MetaServiceProxy {
public:
static Status get_client(std::shared_ptr* stub) {
-SYNC_POINT_RETURN_WITH_VALUE("MetaServiceProxy::get_client",
Status::OK(), stub);
+TEST_SYNC_POINT_RETURN_WITH_VALUE("MetaServiceProxy::get_client",
Status::OK(), stub);
return get_pooled_client(stub);
}
diff --git a/be/src/common/sync_point.h b/be/src/common/sync_point.h
index 9cddce96e4c..81764a09ee9 100644
--- a/be/src/common/sync_point.h
+++ b/be/src/common/sync_point.h
@@ -228,12 +228,12 @@ auto try_any_cast_ret(std::vector& any) {
namespace doris::config {
extern bool enable_injection_point;
}
-# define TEST_INJECTION_POINT(x) if (doris::config::enable_injection_point) {
SYNC_POINT(x); }
-# define TEST_IDX_INJECTION_POINT(x, index) if
(doris::config::enable_injection_point) { IDX_SYNC_POINT(x, index); }
-# define TEST_INJECTION_POINT_CALLBACK(x, ...) if
(doris::config::enable_injection_point) { SYNC_POINT_CALLBACK(x, __VA_ARGS__); }
-# define TEST_INJECTION_POINT_SINGLETON() if
(doris::config::enable_injection_point) { SYNC_POINT_SINGLETON(); }
-# define TEST_INJECTION_POINT_RETURN_WITH_VALUE(x, default_ret_val, ...) if
(doris::config::enable_injection_point) { SYNC_POINT_RETURN_WITH_VALUE(x,
default_ret_val, __VA_ARGS__); }
-# define TEST_INJECTION_POINT_RETURN_WITH_VOID(x, ...) if
(doris::config::enable_injection_point) { SYNC_POINT_RETURN_WITH_VOID(x,
__VA_ARGS__); }
+# define TEST_INJECTION_POINT(x) if (doris::config::enable_injection_point) {
LOG_INFO("enter inject point {}", x); SYNC_POINT(x); }
+# define TEST_IDX_INJECTION_POINT(x, index) if
(doris::config::enable_injection_point) { LOG_INFO("enter inject point {}", x);
IDX_SYNC_POINT(x, index); }
+# define TEST_INJECTION_POINT_CALLBACK(x, ...) if
(doris::config::enable_injection_point) { LOG_INFO("enter inject point {}", x);
SYNC_POINT_CALLBACK(x, __VA_ARGS__); }
+# define TEST_INJECTION_POINT_SINGLETON() if
(doris::config::enable_injection_point) { LOG_INFO("enter inject point {}", x);
SYNC_POINT_SINGLETON(); }
+# define TEST_INJECTION_POINT_RETURN_WITH_VALUE(x, default_ret_val, ...) if
(doris::config::enable_injection_point) { LOG_INFO("enter inject point {}", x);
SYNC_POINT_RETURN_WITH_VALUE(x, default_ret_val, __VA_ARGS__); }
+# define TEST_INJECTION_POINT_RETURN_WITH_VOID(x, ...) if
(doris::config::enable_injection_point) { LOG_INFO("enter inject point {}", x);
SYNC_POINT_RETURN_WITH_VOID(x, __VA_ARGS__); }
#endif // ENABLE_INJECTION_POINT
// clang-format on
diff --git a/be/src/http/action/config_action.cpp
b/be/src/http/action/config_action.cpp
index a9de9dc2406..b855de88100 100644
--- a/be/src/http/action/config_action.cpp
+++ b/be/src/http/action/config_action.cpp
@@ -62,7 +62,7 @@ void ConfigAction::handle_show_config(HttpRequest* req) {
writer.StartArray();
for (const auto& _config : config_info) {
-if (conf_item != nullptr || conf_item != "") {
+if (conf_item != nullptr || !conf_item.empty()) {
if (_config[0] == conf_item) {
writer.StartArray();
for (const std::string& config_filed : _config) {
diff --git a/be/src/io/fs/hdfs_file_writer.cpp
b/be/src/io/fs/hdfs_file_writer.cpp
index 9ea66ca4da1..54e609a040b 100644
--- a/be/src/io/fs/hdfs_file_writer.cpp
+++ b/be/src/io/fs/hdfs_file_writer.cpp
@@ -185,11 +185,11 @@ Status HdfsFileWriter::append_hdfs_file(std::string_view
content) {
written_bytes =
hdfsWrite(_hdfs_handler->hdfs_fs, _hdfs_file,
content.data(), content.size());
{
-[[maybe_unused]] Status error_ret = Status::InternalError(
-"write hdfs failed. fs_name: {}, path: {}, error: size
exceeds", _fs_name,
-
(doris) branch master updated: (cloud-merge) Fix the case timeout because the sql fetches too many datas (#33300)
This is an automated email from the ASF dual-hosted git repository.
plat1ko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 7c73df569f9 (cloud-merge) Fix the case timeout because the sql fetches
too many datas (#33300)
7c73df569f9 is described below
commit 7c73df569f9ddc5d0a6424200646f585afd7e05b
Author: Lightman <31928846+lchangli...@users.noreply.github.com>
AuthorDate: Sun Apr 7 16:12:51 2024 +0800
(cloud-merge) Fix the case timeout because the sql fetches too many datas
(#33300)
---
regression-test/suites/cloud_p0/cache/ttl/alter_ttl_4.groovy | 6 --
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/regression-test/suites/cloud_p0/cache/ttl/alter_ttl_4.groovy
b/regression-test/suites/cloud_p0/cache/ttl/alter_ttl_4.groovy
index 188bfd0a9cb..398ca843243 100644
--- a/regression-test/suites/cloud_p0/cache/ttl/alter_ttl_4.groovy
+++ b/regression-test/suites/cloud_p0/cache/ttl/alter_ttl_4.groovy
@@ -165,16 +165,18 @@ suite("alter_ttl_4") {
load_customer_once("customer")
sql """ ALTER TABLE customer_ttl SET ("file_cache_ttl_seconds"="3600") """
+// wait for fetching new tablet meta in BE
sleep(6)
// some datas in s3 and will download them
-sql """ select C_CUSTKEY from customer_ttl"""
-sql """ select C_NAME from customer_ttl"""
+sql """ select C_CUSTKEY from customer_ttl order by C_CUSTKEY limit 1"""
+sql """ select C_NAME from customer_ttl order by C_NAME limit 1"""
sql """ select C_ADDRESS from customer_ttl order by C_ADDRESS limit 1"""
sql """ select C_NATIONKEY from customer_ttl order by C_NATIONKEY limit
1"""
sql """ select C_PHONE from customer_ttl order by C_PHONE limit 1 """
sql """ select C_ACCTBAL from customer_ttl order by C_ACCTBAL limit 1"""
sql """ select C_MKTSEGMENT from customer_ttl order by C_MKTSEGMENT limit
1"""
sql """ select C_COMMENT from customer_ttl order by C_COMMENT limit 1"""
+// wait for updating file cache metrics
sleep(3)
getMetricsMethod.call() {
respCode, body ->
-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
(doris) branch branch-2.1 updated: [fix](cooldown) Fix hdfs path (#33315)
This is an automated email from the ASF dual-hosted git repository.
plat1ko pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 97850cf2bbf [fix](cooldown) Fix hdfs path (#33315)
97850cf2bbf is described below
commit 97850cf2bbf6d3fac993094e70731af5966f94f9
Author: plat1ko
AuthorDate: Tue Apr 9 12:55:53 2024 +0800
[fix](cooldown) Fix hdfs path (#33315)
---
be/src/io/fs/hdfs_file_system.cpp | 7 ---
be/src/util/hdfs_util.cpp | 22 --
2 files changed, 16 insertions(+), 13 deletions(-)
diff --git a/be/src/io/fs/hdfs_file_system.cpp
b/be/src/io/fs/hdfs_file_system.cpp
index 5b5dbba16b2..22b4784ea61 100644
--- a/be/src/io/fs/hdfs_file_system.cpp
+++ b/be/src/io/fs/hdfs_file_system.cpp
@@ -283,11 +283,12 @@ Status HdfsFileSystem::list_impl(const Path& path, bool
only_file, std::vectoremplace_back();
+std::string_view fname(file.mName);
+fname.remove_prefix(fname.rfind('/') + 1);
+file_info.file_name = fname;
file_info.file_size = file.mSize;
file_info.is_file = (file.mKind != kObjectKindDirectory);
-files->emplace_back(std::move(file_info));
}
hdfsFreeFileInfo(hdfs_file_info, numEntries);
return Status::OK();
diff --git a/be/src/util/hdfs_util.cpp b/be/src/util/hdfs_util.cpp
index 6e99fdea3d3..794c53f15e4 100644
--- a/be/src/util/hdfs_util.cpp
+++ b/be/src/util/hdfs_util.cpp
@@ -42,17 +42,19 @@ hdfsFS HDFSHandle::create_hdfs_fs(HDFSCommonBuilder&
hdfs_builder) {
}
Path convert_path(const Path& path, const std::string& namenode) {
-Path real_path(path);
-if (path.string().find(namenode) != std::string::npos) {
-std::string real_path_str = path.string().substr(namenode.size());
-if (!real_path_str.starts_with("/")) {
-// The real path must starts with "/"
-// Or the hadoop client will add a prefix like "/user/hadoop".
-real_path_str = "/" + real_path_str;
-}
-real_path = real_path_str;
+std::string fs_path;
+if (path.native().starts_with(namenode)) {
+// `path` is URI format, remove the namenode part in `path`
+fs_path = path.native().substr(namenode.size());
+} else {
+fs_path = path;
}
-return real_path;
+
+// Always use absolute path (start with '/') in hdfs
+if (fs_path.empty() || fs_path[0] != '/') {
+fs_path.insert(fs_path.begin(), '/');
+}
+return fs_path;
}
bool is_hdfs(const std::string& path_or_fs) {
-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
(doris) branch master updated: [feature](Cloud) Generate one suffix to distinguish different hdfs root path (#33411)
This is an automated email from the ASF dual-hosted git repository.
plat1ko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 7cd7d4902eb [feature](Cloud) Generate one suffix to distinguish
different hdfs root path (#33411)
7cd7d4902eb is described below
commit 7cd7d4902eb9d6a54f7cf1b5b95693c7d9df639e
Author: AlexYue
AuthorDate: Tue Apr 9 16:52:54 2024 +0800
[feature](Cloud) Generate one suffix to distinguish different hdfs root
path (#33411)
---
cloud/src/common/config.h| 2 ++
cloud/src/meta-service/meta_service_resource.cpp | 10 ++
2 files changed, 12 insertions(+)
diff --git a/cloud/src/common/config.h b/cloud/src/common/config.h
index 41e8529dbf0..5640aef2cf9 100644
--- a/cloud/src/common/config.h
+++ b/cloud/src/common/config.h
@@ -165,4 +165,6 @@ CONF_String(kerberos_ccache_path, "");
// set krb5.conf path, use "/etc/krb5.conf" by default
CONF_String(kerberos_krb5_conf_path, "/etc/krb5.conf");
+CONF_mBool(enable_distinguish_hdfs_path, "true");
+
} // namespace doris::cloud::config
diff --git a/cloud/src/meta-service/meta_service_resource.cpp
b/cloud/src/meta-service/meta_service_resource.cpp
index 82056a5172c..f4f8c944c7b 100644
--- a/cloud/src/meta-service/meta_service_resource.cpp
+++ b/cloud/src/meta-service/meta_service_resource.cpp
@@ -16,6 +16,7 @@
// under the License.
#include
+#include
#include
#include
@@ -384,6 +385,15 @@ static int add_hdfs_storage_vault(InstanceInfoPB&
instance, Transaction* txn,
msg = fmt::format("invalid prefix: ", *prefix);
return -1;
}
+if (config::enable_distinguish_hdfs_path) {
+auto uuid_suffix = butil::GenerateGUID();
+if (uuid_suffix.empty()) [[unlikely]] {
+code = MetaServiceCode::UNDEFINED_ERR;
+msg = fmt::format("failed to generate one suffix for hdfs prefix");
+return -1;
+}
+*prefix = fmt::format("{}_{}", *prefix, uuid_suffix);
+}
auto* fs_name =
hdfs_param.mutable_hdfs_info()->mutable_build_conf()->mutable_fs_name();
if (!normalize_hdfs_fs_name(*fs_name)) {
-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
(doris) branch master updated: add config enable_create_bitmap_index_as_inverted_index (#33434)
This is an automated email from the ASF dual-hosted git repository.
plat1ko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new c8c689714d0 add config enable_create_bitmap_index_as_inverted_index
(#33434)
c8c689714d0 is described below
commit c8c689714d0c9907752c7b80646242780982e06e
Author: Kang
AuthorDate: Tue Apr 9 21:04:14 2024 +0800
add config enable_create_bitmap_index_as_inverted_index (#33434)
---
fe/fe-common/src/main/java/org/apache/doris/common/Config.java | 3 +++
fe/fe-core/src/main/cup/sql_parser.cup | 7 ++-
.../java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java | 3 ++-
3 files changed, 11 insertions(+), 2 deletions(-)
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index bc478f43864..20ac31289b5 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -2632,6 +2632,9 @@ public class Config extends ConfigBase {
@ConfField(mutable = true, masterOnly = true)
public static boolean enable_light_index_change = true;
+@ConfField(mutable = true, masterOnly = true)
+public static boolean enable_create_bitmap_index_as_inverted_index = false;
+
// The original meta read lock is not enough to keep a snapshot of
partition versions,
// so the execution of `createScanRangeLocations` are delayed to
`Coordinator::exec`,
// to help to acquire a snapshot of partition versions.
diff --git a/fe/fe-core/src/main/cup/sql_parser.cup
b/fe/fe-core/src/main/cup/sql_parser.cup
index ac623e39ff4..08b793c4352 100644
--- a/fe/fe-core/src/main/cup/sql_parser.cup
+++ b/fe/fe-core/src/main/cup/sql_parser.cup
@@ -48,6 +48,7 @@ import org.apache.doris.catalog.StructType;
import org.apache.doris.catalog.TableIf.TableType;
import org.apache.doris.catalog.View;
import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.Config;
import org.apache.doris.common.FeConstants;
import org.apache.doris.common.Version;
import org.apache.doris.cloud.analysis.UseCloudClusterStmt;
@@ -3881,7 +3882,11 @@ opt_index_type ::=
:}
| KW_USING KW_BITMAP
{:
-RESULT = IndexDef.IndexType.INVERTED;
+if (Config.enable_create_bitmap_index_as_inverted_index) {
+RESULT = IndexDef.IndexType.INVERTED;
+} else {
+RESULT = IndexDef.IndexType.BITMAP;
+}
:}
| KW_USING KW_NGRAM_BF
{:
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index 874b6a0d432..793cc9e258f 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -2625,7 +2625,8 @@ public class LogicalPlanBuilder extends
DorisParserBaseVisitor {
String indexType = ctx.indexType != null ?
ctx.indexType.getText().toUpperCase() : null;
String comment = ctx.comment != null ? ctx.comment.getText() : "";
// change BITMAP index to INVERTED index
-if ("BITMAP".equalsIgnoreCase(indexType)) {
+if (Config.enable_create_bitmap_index_as_inverted_index
+&& "BITMAP".equalsIgnoreCase(indexType)) {
indexType = "INVERTED";
}
return new IndexDefinition(indexName, indexCols, indexType,
properties, comment);
-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
(doris) branch master updated: [fix](fs) Close local file writer when downloading finished (#33556)
This is an automated email from the ASF dual-hosted git repository.
plat1ko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new ab015d6379c [fix](fs) Close local file writer when downloading
finished (#33556)
ab015d6379c is described below
commit ab015d6379cff2e0251ce077e3c2bca238cbb9b2
Author: walter
AuthorDate: Fri Apr 12 11:10:15 2024 +0800
[fix](fs) Close local file writer when downloading finished (#33556)
---
be/src/io/fs/hdfs_file_system.cpp | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/be/src/io/fs/hdfs_file_system.cpp
b/be/src/io/fs/hdfs_file_system.cpp
index bd535066106..5ea742c20d5 100644
--- a/be/src/io/fs/hdfs_file_system.cpp
+++ b/be/src/io/fs/hdfs_file_system.cpp
@@ -320,8 +320,7 @@ Status HdfsFileSystem::download_impl(const Path&
remote_file, const Path& local_
RETURN_IF_ERROR(local_writer->append({read_buf.get(), read_len}));
}
-
-return Status::OK();
+return local_writer->close();
}
} // namespace doris::io
-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
(doris) branch master updated (806e2418a53 -> a33d2ee0f7e)
This is an automated email from the ASF dual-hosted git repository. plat1ko pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/doris.git from 806e2418a53 [fix](mtmv) Fix table id overturn and optimize get table qualifier method (#34768) add a33d2ee0f7e [bug](Cloud) Use value capture for done closure when alter vault sync No new revisions were added by this update. Summary of changes: be/src/cloud/cloud_internal_service.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
(doris) branch master updated: [chore](cloud) Use c++20 and unleash endian check (#34735)
This is an automated email from the ASF dual-hosted git repository.
plat1ko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 5aab04bbf51 [chore](cloud) Use c++20 and unleash endian check (#34735)
5aab04bbf51 is described below
commit 5aab04bbf51e6878a5a9e5b2d11533b8ef453b19
Author: Gavin Chou
AuthorDate: Sat May 25 13:36:57 2024 +0800
[chore](cloud) Use c++20 and unleash endian check (#34735)
Use c++20 and unleash endian check to prevent misuse.
---
cloud/CMakeLists.txt | 2 +-
cloud/src/meta-service/doris_txn.cpp | 5 -
cloud/src/meta-service/txn_kv.cpp| 8
3 files changed, 9 insertions(+), 6 deletions(-)
diff --git a/cloud/CMakeLists.txt b/cloud/CMakeLists.txt
index 9b9929ae1d5..22469ef4d37 100644
--- a/cloud/CMakeLists.txt
+++ b/cloud/CMakeLists.txt
@@ -137,7 +137,7 @@ check_function_exists(sched_getcpu HAVE_SCHED_GETCPU)
# -fno-omit-frame-pointers: Keep frame pointer for functions in register
set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wall -Wno-sign-compare -pthread
-Werror")
set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -fstrict-aliasing
-fno-omit-frame-pointer")
-set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -std=gnu++17 -D__STDC_FORMAT_MACROS")
+set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -std=gnu++20 -D__STDC_FORMAT_MACROS")
set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS}
-DBOOST_DATE_TIME_POSIX_TIME_STD_CONFIG")
set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -DBOOST_SYSTEM_NO_DEPRECATED")
# Enable the cpu and heap profile of brpc
diff --git a/cloud/src/meta-service/doris_txn.cpp
b/cloud/src/meta-service/doris_txn.cpp
index 7c185bbeb4a..eb01dd4d874 100644
--- a/cloud/src/meta-service/doris_txn.cpp
+++ b/cloud/src/meta-service/doris_txn.cpp
@@ -17,6 +17,8 @@
#include "doris_txn.h"
+#include
+
namespace doris::cloud {
int get_txn_id_from_fdb_ts(std::string_view fdb_vts, int64_t* txn_id) {
@@ -29,7 +31,8 @@ int get_txn_id_from_fdb_ts(std::string_view fdb_vts, int64_t*
txn_id) {
// byte addr 0 1 2 3 4 5 6 7 8 9
int64_t ver = *reinterpret_cast(fdb_vts.data());
-// static_assert(std::endian::little); // Since c++20
+// TODO(gavin): implementation for big-endian or make it endian-independent
+static_assert(std::endian::native == std::endian::little); // Since c++20
// Convert big endian to little endian
static auto to_little = [](int64_t v) {
v = ((v & 0x) >> 32) | ((v & 0x) <<
32);
diff --git a/cloud/src/meta-service/txn_kv.cpp
b/cloud/src/meta-service/txn_kv.cpp
index 4c732892be1..ebb63d095f8 100644
--- a/cloud/src/meta-service/txn_kv.cpp
+++ b/cloud/src/meta-service/txn_kv.cpp
@@ -19,11 +19,11 @@
#include
#include
-#include
#include
#include
#include
+#include
#include
#include
#include
@@ -420,9 +420,9 @@ bool Transaction::decode_atomic_int(std::string_view data,
int64_t* val) {
// ATTN: The FDB_MUTATION_TYPE_ADD stores integers in a little-endian
representation.
std::memcpy(val, data.data(), sizeof(*val));
-#if __BYTE_ORDER == __BIG_ENDIAN
-*val = bswap_64(*val);
-#endif
+if constexpr (std::endian::native == std::endian::big) {
+*val = bswap_64(*val);
+}
return true;
}
-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
(doris) branch master updated (99c5a805d08 -> 54773531c3c)
This is an automated email from the ASF dual-hosted git repository. plat1ko pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/doris.git from 99c5a805d08 [Feature](topn) support general expr with topn filter and some refactor (#35405) add 54773531c3c [fix](Nereids): avoid memory usage due to multiple iterations when eliminate func deps (#35408) No new revisions were added by this update. Summary of changes: .../src/main/java/org/apache/doris/nereids/properties/FuncDeps.java | 6 +- 1 file changed, 5 insertions(+), 1 deletion(-) - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
(doris) branch master updated (c6b766722be -> 401910c017f)
This is an automated email from the ASF dual-hosted git repository. plat1ko pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/doris.git from c6b766722be [Fix](planner) fix create view star except and modify cast to sql (#33726) add 401910c017f [feature](Cloud) Enable write into file cache for hdfs writer (#33796) No new revisions were added by this update. Summary of changes: be/src/common/config.cpp | 1 + be/src/common/config.h| 1 + be/src/io/cache/file_cache_common.cpp | 11 ++ be/src/io/cache/file_cache_common.h | 12 ++ be/src/io/fs/hdfs_file_system.cpp | 4 +- be/src/io/fs/hdfs_file_writer.cpp | 199 -- be/src/io/fs/hdfs_file_writer.h | 35 +- be/src/io/hdfs_util.cpp | 11 ++ be/src/io/hdfs_util.h | 12 ++ 9 files changed, 245 insertions(+), 41 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
(doris) branch branch-3.0 updated: [fix](checker) Clarified the semantics of the checker return value (#38785)
This is an automated email from the ASF dual-hosted git repository.
plat1ko pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 7131a3b39b3 [fix](checker) Clarified the semantics of the checker
return value (#38785)
7131a3b39b3 is described below
commit 7131a3b39b32bcafad3b4a6c062d24dfdb7bd5d8
Author: plat1ko
AuthorDate: Mon Aug 26 19:50:56 2024 +0800
[fix](checker) Clarified the semantics of the checker return value (#38785)
---
cloud/src/recycler/checker.cpp | 68 --
cloud/src/recycler/checker.h | 13
2 files changed, 53 insertions(+), 28 deletions(-)
diff --git a/cloud/src/recycler/checker.cpp b/cloud/src/recycler/checker.cpp
index 49421f97ca0..f8289160269 100644
--- a/cloud/src/recycler/checker.cpp
+++ b/cloud/src/recycler/checker.cpp
@@ -167,10 +167,20 @@ int Checker::start() {
duration_cast(system_clock::now().time_since_epoch()).count();
g_bvar_checker_enqueue_cost_s.put(instance_id, ctime_ms / 1000 -
enqueue_time_s);
ret = checker->do_check();
+
if (config::enable_inverted_check) {
-if (checker->do_inverted_check() != 0) ret = -1;
+if (ret == 0) {
+ret = checker->do_inverted_check();
+}
+}
+
+if (ret < 0) {
+// If ret < 0, it means that a temporary error occurred during
the check process.
+// The check job should not be considered finished, and the
next round of check job
+// should be retried as soon as possible.
+return;
}
-if (ret == -1) return;
+
// If instance checker has been aborted, don't finish this job
if (!checker->stopped()) {
finish_instance_recycle_job(txn_kv_.get(), check_job_key,
instance.instance_id(),
@@ -444,9 +454,10 @@ int InstanceChecker::init_storage_vault_accessors(const
InstanceInfoPB& instance
int InstanceChecker::do_check() {
TEST_SYNC_POINT("InstanceChecker.do_check");
LOG(INFO) << "begin to check instance objects instance_id=" <<
instance_id_;
+int check_ret = 0;
long num_scanned = 0;
long num_scanned_with_segment = 0;
-long num_check_failed = 0;
+long num_rowset_loss = 0;
long instance_volume = 0;
using namespace std::chrono;
auto start_time = steady_clock::now();
@@ -455,11 +466,11 @@ int InstanceChecker::do_check() {
LOG(INFO) << "check instance objects finished, cost=" << cost
<< "s. instance_id=" << instance_id_ << " num_scanned=" <<
num_scanned
<< " num_scanned_with_segment=" << num_scanned_with_segment
- << " num_check_failed=" << num_check_failed
+ << " num_rowset_loss=" << num_rowset_loss
<< " instance_volume=" << instance_volume;
g_bvar_checker_num_scanned.put(instance_id_, num_scanned);
g_bvar_checker_num_scanned_with_segment.put(instance_id_,
num_scanned_with_segment);
-g_bvar_checker_num_check_failed.put(instance_id_, num_check_failed);
+g_bvar_checker_num_check_failed.put(instance_id_, num_rowset_loss);
g_bvar_checker_check_cost_s.put(instance_id_, static_cast(cost));
// FIXME(plat1ko): What if some list operation failed?
g_bvar_checker_instance_volume.put(instance_id_, instance_volume);
@@ -490,7 +501,7 @@ int InstanceChecker::do_check() {
.tag("resource_id", rs_meta.resource_id())
.tag("tablet_id", rs_meta.tablet_id())
.tag("rowset_id", rs_meta.rowset_id_v2());
-++num_check_failed;
+check_ret = -1;
return;
}
@@ -498,7 +509,7 @@ int InstanceChecker::do_check() {
int ret =
find_it->second->list_directory(tablet_path_prefix(rs_meta.tablet_id()),
&list_iter);
if (ret != 0) { // No need to log, because S3Accessor has logged
this error
-++num_check_failed;
+check_ret = -1;
return;
}
@@ -510,6 +521,7 @@ int InstanceChecker::do_check() {
instance_volume += tablet_volume;
}
+bool data_loss = false;
for (int i = 0; i < rs_meta.num_segments(); ++i) {
auto path = segment_path(rs_meta.tablet_id(),
rs_meta.rowset_id_v2(), i);
if (tablet_files_c
