(inlong) branch master updated: [INLONG-10851][SDK] Support multiple protocols for DataProxy C++ SDK (#10855)
This is an automated email from the ASF dual-hosted git repository.
doleyzi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new 2bb4c0eaa4 [INLONG-10851][SDK] Support multiple protocols for
DataProxy C++ SDK (#10855)
2bb4c0eaa4 is described below
commit 2bb4c0eaa42ebf57c4fbd13d2214864d2e134408
Author: doleyzi <43397300+dole...@users.noreply.github.com>
AuthorDate: Thu Aug 22 17:17:10 2024 +0800
[INLONG-10851][SDK] Support multiple protocols for DataProxy C++ SDK
(#10855)
---
.../dataproxy-sdk-cpp/src/client/tcp_client.cc | 2 +-
.../dataproxy-sdk-cpp/src/config/sdk_conf.cc | 31 ++--
.../dataproxy-sdk-cpp/src/config/sdk_conf.h| 7 +-
.../dataproxy-sdk-cpp/src/group/recv_group.cc | 10 +-
.../dataproxy-sdk-cpp/src/group/recv_group.h | 3 +
.../src/manager/metric_manager.cc | 1 +
.../dataproxy-sdk-cpp/src/manager/proxy_manager.cc | 133 +++--
.../dataproxy-sdk-cpp/src/manager/proxy_manager.h | 3 +-
.../dataproxy-sdk-cpp/src/manager/send_manager.cc | 2 +-
.../dataproxy-sdk-cpp/src/metric/environment.h | 7 +-
.../dataproxy-sdk-cpp/src/metric/metric.h | 4 +-
.../dataproxy-sdk-cpp/src/utils/capi_constant.h| 1 +
.../dataproxy-sdk-cpp/src/utils/parse_json.cc | 160 +
.../dataproxy-sdk-cpp/src/utils/parse_json.h | 35 +
14 files changed, 248 insertions(+), 151 deletions(-)
diff --git
a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/client/tcp_client.cc
b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/client/tcp_client.cc
index c3b7692809..1e4d8bcedf 100644
--- a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/client/tcp_client.cc
+++ b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/client/tcp_client.cc
@@ -307,7 +307,7 @@ void TcpClient::UpdateMetric() {
stat.Update(it.second);
it.second.ResetStat();
}
- LOG_INFO(stat.ToString() << CLIENT_INFO);
+ LOG_INFO(stat.GetSendMetricInfo() << CLIENT_INFO);
}
void TcpClient::HeartBeat(bool only_heart_heat) {
diff --git
a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/config/sdk_conf.cc
b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/config/sdk_conf.cc
index 5f067528af..6ff642472f 100644
--- a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/config/sdk_conf.cc
+++ b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/config/sdk_conf.cc
@@ -121,8 +121,6 @@ void SdkConfig::defaultInit() {
// manager parameters
manager_url_ = constants::kManagerURL;
- enable_manager_url_from_cluster_ = constants::kEnableManagerFromCluster;
- manager_cluster_url_ = constants::kManagerClusterURL;
manager_update_interval_ = constants::kManagerUpdateInterval;
manager_url_timeout_ = constants::kManagerTimeout;
max_proxy_num_ = constants::kMaxProxyNum;
@@ -329,22 +327,7 @@ void SdkConfig::InitManagerParam(const rapidjson::Value
&doc) {
} else {
manager_url_ = constants::kManagerURL;
}
- // manager cluster url
- if (doc.HasMember("manager_cluster_url") &&
- doc["manager_cluster_url"].IsString()) {
-const rapidjson::Value &obj = doc["manager_cluster_url"];
-manager_cluster_url_ = obj.GetString();
- } else {
-manager_cluster_url_ = constants::kManagerClusterURL;
- }
- // enable manager from cluster
- if (doc.HasMember("enable_manager_url_from_cluster") &&
- doc["enable_manager_url_from_cluster"].IsBool()) {
-const rapidjson::Value &obj = doc["enable_manager_url_from_cluster"];
-enable_manager_url_from_cluster_ = obj.GetBool();
- } else {
-enable_manager_url_from_cluster_ = constants::kEnableManagerFromCluster;
- }
+
// manager update interval
if (doc.HasMember("manager_update_interval") &&
doc["manager_update_interval"].IsInt() &&
@@ -522,6 +505,13 @@ void SdkConfig::OthersParam(const rapidjson::Value &doc) {
} else {
max_instance_ = constants::kMaxInstance;
}
+
+ if (doc.HasMember("extend_report") && doc["extend_report"].IsBool()) {
+const rapidjson::Value &obj = doc["extend_report"];
+extend_report_ = obj.GetBool();
+ } else {
+extend_report_ = constants::kExtendReport;
+ }
}
bool SdkConfig::GetLocalIPV4Address(std::string &err_info, std::string
&localhost) {
@@ -589,11 +579,6 @@ void SdkConfig::ShowClientConfig() {
LOG_INFO("log_level: " << log_level_);
LOG_INFO("log_path: " << log_path_.c_str());
LOG_INFO("manager_url: " << manager_url_.c_str());
- LOG_INFO("manager_cluster_url: " << manager_cluster_url_.c_str());
- LOG_INFO(
- "enable_manager_url_from_cluster: " <<
(inlong-website) branch INLONG-945 created (now f3fac76a21)
This is an automated email from the ASF dual-hosted git repository. doleyzi pushed a change to branch INLONG-945 in repository https://gitbox.apache.org/repos/asf/inlong-website.git at f3fac76a21 Update inlong audit usage documentation. This branch includes the following new commits: new f3fac76a21 Update inlong audit usage documentation. The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference.
(inlong-website) 01/01: Update inlong audit usage documentation.
This is an automated email from the ASF dual-hosted git repository.
doleyzi pushed a commit to branch INLONG-945
in repository https://gitbox.apache.org/repos/asf/inlong-website.git
commit f3fac76a21595d9e16e7c07d12d23c88487fc179
Author: doleyzi
AuthorDate: Mon Apr 29 20:56:57 2024 +0800
Update inlong audit usage documentation.
---
docs/modules/audit/configure.md| 25 -
docs/modules/audit/img/audit_architecture.png | Bin 30983 -> 83639 bytes
docs/modules/audit/img/audit_openapi.png | Bin 0 -> 56423 bytes
docs/modules/audit/overview.md | 116 +++-
docs/modules/audit/quick_start.md | 32 +-
.../current/modules/audit/configure.md | 34 --
.../modules/audit/img/audit_architecture.png | Bin 33085 -> 83639 bytes
.../current/modules/audit/img/audit_openapi.png| Bin 0 -> 56423 bytes
.../current/modules/audit/overview.md | 119 +++--
.../current/modules/audit/quick_start.md | 38 ++-
10 files changed, 148 insertions(+), 216 deletions(-)
diff --git a/docs/modules/audit/configure.md b/docs/modules/audit/configure.md
index b53ff068f7..7f7b234db0 100644
--- a/docs/modules/audit/configure.md
+++ b/docs/modules/audit/configure.md
@@ -5,8 +5,9 @@ sidebar_position: 3
## Overview
-Audit-proxy source-channel-sink pipeline configuration
(audit-proxy-{tube|pulsar|kafka}.conf).Audit-store storage service
-configuration (application.properties)
+* Audit-proxy source-channel-sink pipeline configuration
(audit-proxy-{tube|pulsar|kafka}.conf).
+* Audit-store storage service configuration (application.properties).
+* OpenAPI audit-service configuration audit-service.properties.
## Audit-proxy source-channel-sink pipeline configuration
(`audit-proxy-{tube|pulsar|kafka}.conf`)
@@ -86,4 +87,22 @@ configuration (application.properties)
| clickhouse.driver | Set the driver type |
ru.yandex.clickhouse.ClickHouseDriver |
|
| clickhouse.url| clickhouse URL | jdbc:clickhouse://127.0.0.1:8123/Default
value ||
| clickhouse.username | account name | Default value | |
-| clickhouse.password | password | Default value | |
\ No newline at end of file
+| clickhouse.password | password | Default value | |
+
+### StarRocks configuration
+
+| Parameter | Description | Default value | Notes
|
+|-|---|-|-|
+| jdbc.driver | Driver type | com.mysql.cj.jdbc.Driver |
|
+| jdbc.url | StarRocks URL | jdbc:mysql://127.0.0.1:8123/default |
|
+| jdbc.username | account name | default | |
+| jdbc.password | password| default | |
+
+## OpenAPI audit-service
+configuration `audit-service.properties`
+
+| Parameter| Description | Default value | Notes
|
+|---|--|---|-|
+| mysql.jdbc.url| mysql URL| jdbc:mysql://127.0.0.1:8123/default |
|
+| mysql.username| account name | default | |
+| mysql.password| password | default | |
\ No newline at end of file
diff --git a/docs/modules/audit/img/audit_architecture.png
b/docs/modules/audit/img/audit_architecture.png
index 609b9b210b..3539c489f9 100644
Binary files a/docs/modules/audit/img/audit_architecture.png and
b/docs/modules/audit/img/audit_architecture.png differ
diff --git a/docs/modules/audit/img/audit_openapi.png
b/docs/modules/audit/img/audit_openapi.png
new file mode 100644
index 00..c7bc0f9774
Binary files /dev/null and b/docs/modules/audit/img/audit_openapi.png differ
diff --git a/docs/modules/audit/overview.md b/docs/modules/audit/overview.md
index 46178d62c2..3f1c5bfc52 100644
--- a/docs/modules/audit/overview.md
+++ b/docs/modules/audit/overview.md
@@ -16,6 +16,8 @@ The transmission status of each module, and whether the data
stream is lost or r
3. The distribution service consumes the audit data of MQ, and writes the
audit data to MySQL, Elasticsearch and ClickHouse.
4. The interface layer encapsulates the data of MySQL, Elasticsearch and
ClickHouse.
5. Application scenarios mainly include report display, audit reconciliation,
etc.
+6. Support audit and reconciliation of data supplementary recording scenarios.
+7. Support audit reconciliation in Flink checkpoint scenarios.
## Audit Dimension
| | | || | | | | | |
@@ -101,119 +103,31 @@ message AuditReply {
***2. Data Uniqueness***
***3. Reduce data loss caused by abnormal restart***
-### Main Logic Diagram
- branch master updated: Update inlong audit usage documentation. (#946)
This is an automated email from the ASF dual-hosted git repository.
doleyzi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong-website.git
The following commit(s) were added to refs/heads/master by this push:
new e976059d45 Update inlong audit usage documentation. (#946)
e976059d45 is described below
commit e976059d45f78487e6dc086428431cba1184955a
Author: doleyzi <43397300+dole...@users.noreply.github.com>
AuthorDate: Mon Apr 29 21:09:58 2024 +0800
Update inlong audit usage documentation. (#946)
---
docs/modules/audit/configure.md| 25 -
docs/modules/audit/img/audit_architecture.png | Bin 30983 -> 83639 bytes
docs/modules/audit/img/audit_openapi.png | Bin 0 -> 56423 bytes
docs/modules/audit/overview.md | 116 +++-
docs/modules/audit/quick_start.md | 32 +-
.../current/modules/audit/configure.md | 34 --
.../modules/audit/img/audit_architecture.png | Bin 33085 -> 83639 bytes
.../current/modules/audit/img/audit_openapi.png| Bin 0 -> 56423 bytes
.../current/modules/audit/overview.md | 119 +++--
.../current/modules/audit/quick_start.md | 38 ++-
10 files changed, 148 insertions(+), 216 deletions(-)
diff --git a/docs/modules/audit/configure.md b/docs/modules/audit/configure.md
index b53ff068f7..7f7b234db0 100644
--- a/docs/modules/audit/configure.md
+++ b/docs/modules/audit/configure.md
@@ -5,8 +5,9 @@ sidebar_position: 3
## Overview
-Audit-proxy source-channel-sink pipeline configuration
(audit-proxy-{tube|pulsar|kafka}.conf).Audit-store storage service
-configuration (application.properties)
+* Audit-proxy source-channel-sink pipeline configuration
(audit-proxy-{tube|pulsar|kafka}.conf).
+* Audit-store storage service configuration (application.properties).
+* OpenAPI audit-service configuration audit-service.properties.
## Audit-proxy source-channel-sink pipeline configuration
(`audit-proxy-{tube|pulsar|kafka}.conf`)
@@ -86,4 +87,22 @@ configuration (application.properties)
| clickhouse.driver | Set the driver type |
ru.yandex.clickhouse.ClickHouseDriver |
|
| clickhouse.url| clickhouse URL | jdbc:clickhouse://127.0.0.1:8123/Default
value ||
| clickhouse.username | account name | Default value | |
-| clickhouse.password | password | Default value | |
\ No newline at end of file
+| clickhouse.password | password | Default value | |
+
+### StarRocks configuration
+
+| Parameter | Description | Default value | Notes
|
+|-|---|-|-|
+| jdbc.driver | Driver type | com.mysql.cj.jdbc.Driver |
|
+| jdbc.url | StarRocks URL | jdbc:mysql://127.0.0.1:8123/default |
|
+| jdbc.username | account name | default | |
+| jdbc.password | password| default | |
+
+## OpenAPI audit-service
+configuration `audit-service.properties`
+
+| Parameter| Description | Default value | Notes
|
+|---|--|---|-|
+| mysql.jdbc.url| mysql URL| jdbc:mysql://127.0.0.1:8123/default |
|
+| mysql.username| account name | default | |
+| mysql.password| password | default | |
\ No newline at end of file
diff --git a/docs/modules/audit/img/audit_architecture.png
b/docs/modules/audit/img/audit_architecture.png
index 609b9b210b..3539c489f9 100644
Binary files a/docs/modules/audit/img/audit_architecture.png and
b/docs/modules/audit/img/audit_architecture.png differ
diff --git a/docs/modules/audit/img/audit_openapi.png
b/docs/modules/audit/img/audit_openapi.png
new file mode 100644
index 00..c7bc0f9774
Binary files /dev/null and b/docs/modules/audit/img/audit_openapi.png differ
diff --git a/docs/modules/audit/overview.md b/docs/modules/audit/overview.md
index 46178d62c2..3f1c5bfc52 100644
--- a/docs/modules/audit/overview.md
+++ b/docs/modules/audit/overview.md
@@ -16,6 +16,8 @@ The transmission status of each module, and whether the data
stream is lost or r
3. The distribution service consumes the audit data of MQ, and writes the
audit data to MySQL, Elasticsearch and ClickHouse.
4. The interface layer encapsulates the data of MySQL, Elasticsearch and
ClickHouse.
5. Application scenarios mainly include report display, audit reconciliation,
etc.
+6. Support audit and reconciliation of data supplementary recording scenarios.
+7. Support audit reconciliation in Flink ch
(inlong-website) branch INLONG-949 created (now 487adbf9bb)
This is an automated email from the ASF dual-hosted git repository. doleyzi pushed a change to branch INLONG-949 in repository https://gitbox.apache.org/repos/asf/inlong-website.git at 487adbf9bb Remove audit documents related to clickhouse and elasticsearch This branch includes the following new commits: new 487adbf9bb Remove audit documents related to clickhouse and elasticsearch The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference.
(inlong-website) 01/01: Remove audit documents related to clickhouse and elasticsearch
This is an automated email from the ASF dual-hosted git repository. doleyzi pushed a commit to branch INLONG-949 in repository https://gitbox.apache.org/repos/asf/inlong-website.git commit 487adbf9bbf931bf8e1b5b5f516a4d8931ca6de8 Author: doleyzi AuthorDate: Wed May 8 18:14:37 2024 +0800 Remove audit documents related to clickhouse and elasticsearch --- docs/modules/audit/configure.md| 50 --- docs/modules/audit/img/audit_api.png | Bin 31395 -> 0 bytes docs/modules/audit/img/audit_architecture.png | Bin 83639 -> 51469 bytes docs/modules/audit/img/audit_mysql.png | Bin 19356 -> 0 bytes docs/modules/audit/img/audit_openapi.png | Bin 56423 -> 0 bytes docs/modules/audit/img/audit_proxy.png | Bin 29358 -> 0 bytes docs/modules/audit/img/audit_sdk.png | Bin 81979 -> 0 bytes .../audit/img/audit_sdk_disaster_recovery.png | Bin 35275 -> 0 bytes docs/modules/audit/img/audit_ui.png| Bin 76860 -> 0 bytes docs/modules/audit/img/elasticsearch_index.png | Bin 22117 -> 0 bytes docs/modules/audit/img/elasticsearch_overview.png | Bin 47297 -> 0 bytes docs/modules/audit/img/elasticsearch_write.png | Bin 26657 -> 0 bytes docs/modules/audit/overview.md | 47 -- docs/modules/audit/quick_start.md | 52 .../current/modules/audit/configure.md | 33 + .../current/modules/audit/img/audit_api.png| Bin 33174 -> 0 bytes .../modules/audit/img/audit_architecture.png | Bin 83639 -> 51469 bytes .../current/modules/audit/img/audit_mysql.png | Bin 19165 -> 0 bytes .../current/modules/audit/img/audit_openapi.png| Bin 56423 -> 0 bytes .../current/modules/audit/img/audit_proxy.png | Bin 31329 -> 0 bytes .../current/modules/audit/img/audit_sdk.png| Bin 1 -> 0 bytes .../audit/img/audit_sdk_disaster_recovery.png | Bin 29207 -> 0 bytes .../current/modules/audit/img/audit_ui.png | Bin 79285 -> 0 bytes .../modules/audit/img/elasticsearch_index.png | Bin 21801 -> 0 bytes .../modules/audit/img/elasticsearch_overview.png | Bin 46579 -> 0 bytes .../modules/audit/img/elasticsearch_write.png | Bin 27577 -> 0 bytes .../current/modules/audit/overview.md | 53 + .../current/modules/audit/quick_start.md | 51 28 files changed, 74 insertions(+), 212 deletions(-) diff --git a/docs/modules/audit/configure.md b/docs/modules/audit/configure.md index 7f7b234db0..ae5997925d 100644 --- a/docs/modules/audit/configure.md +++ b/docs/modules/audit/configure.md @@ -18,7 +18,6 @@ sidebar_position: 3 | agent1.sources | source typpe |tcp-source || | agent1.channels| used channel | ch-msg1 | | | agent1.sinks | used sink | pulsar-sink-msg1 || -| ### sources Configuration @@ -50,50 +49,19 @@ sidebar_position: 3 ### MQ configuration -| Parameter | Description | Default value| Notes | -||-|---|| -| audit.config.proxy.type | MQ type |pulsar || -| audit.pulsar.server.url | pulsar broker | pulsar://127.0.0.1:6650 | | -| audit.pulsar.topic | pulsar topic | persistent://public/Default value/inlong-audit| | -| audit.pulsar.consumer.sub.name | consumer | inlong-audit-subscription | | -| audit.pulsar.enable.auth | Whether security certification is required | false| | -| audit.pulsar.token | pulsar authentication token | | | - -### DB configuration - -| Parameter
(inlong-website) branch INLONG-949 updated (487adbf9bb -> b117bf06bc)
This is an automated email from the ASF dual-hosted git repository. doleyzi pushed a change to branch INLONG-949 in repository https://gitbox.apache.org/repos/asf/inlong-website.git from 487adbf9bb Remove audit documents related to clickhouse and elasticsearch add b117bf06bc Remove audit documents related to clickhouse and elasticsearch No new revisions were added by this update. Summary of changes: docs/modules/audit/overview.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
(inlong-website) branch INLONG-949 updated (b117bf06bc -> 45398bf5fc)
This is an automated email from the ASF dual-hosted git repository. doleyzi pushed a change to branch INLONG-949 in repository https://gitbox.apache.org/repos/asf/inlong-website.git from b117bf06bc Remove audit documents related to clickhouse and elasticsearch add 45398bf5fc Remove audit documents related to clickhouse and elasticsearch No new revisions were added by this update. Summary of changes: docs/modules/audit/overview.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
(inlong-website) branch INLONG-949 updated (45398bf5fc -> 571d4aa02b)
This is an automated email from the ASF dual-hosted git repository. doleyzi pushed a change to branch INLONG-949 in repository https://gitbox.apache.org/repos/asf/inlong-website.git from 45398bf5fc Remove audit documents related to clickhouse and elasticsearch add 571d4aa02b Remove extra commas No new revisions were added by this update. Summary of changes: .../docusaurus-plugin-content-docs/current/modules/audit/quick_start.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
(inlong-website) branch INLONG-949 updated (571d4aa02b -> 63fa28efca)
This is an automated email from the ASF dual-hosted git repository. doleyzi pushed a change to branch INLONG-949 in repository https://gitbox.apache.org/repos/asf/inlong-website.git from 571d4aa02b Remove extra commas add 63fa28efca Modify content specifications No new revisions were added by this update. Summary of changes: docs/modules/audit/configure.md| 16 ++-- docs/modules/audit/overview.md | 4 +- docs/modules/audit/quick_start.md | 4 +- .../current/modules/audit/configure.md | 96 +++--- .../current/modules/audit/overview.md | 90 ++-- .../current/modules/audit/quick_start.md | 12 +-- 6 files changed, 111 insertions(+), 111 deletions(-)
(inlong) branch master updated: Add HDFS Audit items in the Audit SDK (#10380)
This is an automated email from the ASF dual-hosted git repository.
doleyzi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new 6b4e028278 Add HDFS Audit items in the Audit SDK (#10380)
6b4e028278 is described below
commit 6b4e028278cedba497da264349d9c6b3d6a64101
Author: doleyzi <43397300+dole...@users.noreply.github.com>
AuthorDate: Tue Jun 11 11:19:11 2024 +0800
Add HDFS Audit items in the Audit SDK (#10380)
---
.../src/main/java/org/apache/inlong/audit/AuditIdEnum.java | 6 +-
.../src/main/java/org/apache/inlong/audit/entity/AuditType.java | 3 ++-
.../main/java/org/apache/inlong/audit/util/AuditManagerUtils.java | 5 +
3 files changed, 12 insertions(+), 2 deletions(-)
diff --git
a/inlong-audit/audit-sdk/src/main/java/org/apache/inlong/audit/AuditIdEnum.java
b/inlong-audit/audit-sdk/src/main/java/org/apache/inlong/audit/AuditIdEnum.java
index d63bc3a17a..ca437fb035 100644
---
a/inlong-audit/audit-sdk/src/main/java/org/apache/inlong/audit/AuditIdEnum.java
+++
b/inlong-audit/audit-sdk/src/main/java/org/apache/inlong/audit/AuditIdEnum.java
@@ -31,6 +31,7 @@ import static
org.apache.inlong.audit.entity.AuditType.DATAPROXY;
import static org.apache.inlong.audit.entity.AuditType.DORIS;
import static org.apache.inlong.audit.entity.AuditType.ELASTICSEARCH;
import static org.apache.inlong.audit.entity.AuditType.HBASE;
+import static org.apache.inlong.audit.entity.AuditType.HDFS;
import static org.apache.inlong.audit.entity.AuditType.HIVE;
import static org.apache.inlong.audit.entity.AuditType.HUDI;
import static org.apache.inlong.audit.entity.AuditType.ICEBERG;
@@ -94,7 +95,10 @@ public enum AuditIdEnum {
SORT_TUBE_OUTPUT(34, OUTPUT, TUBE, "Sent Audit Metrics for Sort Tube"),
SORT_MYSQL_INPUT(35, INPUT, MYSQL, "Received Audit Metrics for Sort
MySQL"),
-SORT_MYSQL_OUTPUT(36, OUTPUT, MYSQL, "Sent Audit Metrics for Sort MySQL");
+SORT_MYSQL_OUTPUT(36, OUTPUT, MYSQL, "Sent Audit Metrics for Sort MySQL"),
+
+SORT_HDFS_INPUT(37, INPUT, HDFS, "Received Audit Metrics for Sort HDFS"),
+SORT_HDFS_OUTPUT(38, OUTPUT, HDFS, "Sent Audit Metrics for Sort HDFS");
private static final Logger LOGGER =
LoggerFactory.getLogger(AuditIdEnum.class);
private final int auditId;
diff --git
a/inlong-audit/audit-sdk/src/main/java/org/apache/inlong/audit/entity/AuditType.java
b/inlong-audit/audit-sdk/src/main/java/org/apache/inlong/audit/entity/AuditType.java
index 6fb554b407..0af03844d6 100644
---
a/inlong-audit/audit-sdk/src/main/java/org/apache/inlong/audit/entity/AuditType.java
+++
b/inlong-audit/audit-sdk/src/main/java/org/apache/inlong/audit/entity/AuditType.java
@@ -34,7 +34,8 @@ public enum AuditType {
POSTGRES("Postgres"),
BINLOG("Binlog"),
TUBE("Tube"),
-MYSQL("MySQL");
+MYSQL("MySQL"),
+HDFS("HDFS");
private final String auditType;
diff --git
a/inlong-audit/audit-sdk/src/main/java/org/apache/inlong/audit/util/AuditManagerUtils.java
b/inlong-audit/audit-sdk/src/main/java/org/apache/inlong/audit/util/AuditManagerUtils.java
index 8bc2c8e7a3..35009d22df 100644
---
a/inlong-audit/audit-sdk/src/main/java/org/apache/inlong/audit/util/AuditManagerUtils.java
+++
b/inlong-audit/audit-sdk/src/main/java/org/apache/inlong/audit/util/AuditManagerUtils.java
@@ -18,6 +18,7 @@
package org.apache.inlong.audit.util;
import org.apache.inlong.audit.AuditIdEnum;
+import org.apache.inlong.audit.MetricIdEnum;
import org.apache.inlong.audit.entity.AuditInformation;
import org.apache.inlong.audit.entity.FlowType;
@@ -135,6 +136,10 @@ public class AuditManagerUtils {
auditInformationList.addAll(combineAuditInformation(auditIdEnum.getAuditType().value(),
auditIdEnum.getFlowType()));
}
+for (MetricIdEnum metricIdEnum : MetricIdEnum.values()) {
+auditInformationList.add(new
AuditInformation(metricIdEnum.getValue(), metricIdEnum.getEnglishDescription(),
+metricIdEnum.getChineseDescription()));
+}
return auditInformationList;
}
(inlong) branch master updated: [INLONG-10387][Audit] Audit SDK supports obtaining Audit-Proxy capabilities through InLong Manager (#10398)
This is an automated email from the ASF dual-hosted git repository.
doleyzi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new 5fa60262c7 [INLONG-10387][Audit] Audit SDK supports obtaining
Audit-Proxy capabilities through InLong Manager (#10398)
5fa60262c7 is described below
commit 5fa60262c79be84744619e5be261b8c869cc5139
Author: doleyzi <43397300+dole...@users.noreply.github.com>
AuthorDate: Fri Jun 14 15:29:51 2024 +0800
[INLONG-10387][Audit] Audit SDK supports obtaining Audit-Proxy
capabilities through InLong Manager (#10398)
[INLONG-10387][Audit] Audit SDK supports obtaining Audit-Proxy
capabilities through InLong Manager (#10398)
---
inlong-audit/audit-common/pom.xml | 4 +
.../audit/{entities => entity}/AuditComponent.java | 14 ++-
.../apache/inlong/audit/entity}/AuditProxy.java| 19 +++-
.../apache/inlong/audit/entity/CommonResponse.java | 91 ++
.../org/apache/inlong/audit/utils/HttpUtils.java | 106 +
.../apache/inlong/audit/heartbeat/Heartbeat.java | 2 +-
inlong-audit/audit-sdk/README.md | 48 --
.../java/org/apache/inlong/audit/AuditIdEnum.java | 10 +-
.../org/apache/inlong/audit/AuditReporterImpl.java | 44 ++---
.../org/apache/inlong/audit/entity/AuditType.java | 6 +-
.../org/apache/inlong/audit/send/ProxyManager.java | 88 -
.../apache/inlong/audit/cache/AuditProxyCache.java | 2 +-
.../apache/inlong/audit/service/ApiService.java| 4 +-
13 files changed, 405 insertions(+), 33 deletions(-)
diff --git a/inlong-audit/audit-common/pom.xml
b/inlong-audit/audit-common/pom.xml
index 70f0d1911a..ee0fbf81e1 100644
--- a/inlong-audit/audit-common/pom.xml
+++ b/inlong-audit/audit-common/pom.xml
@@ -42,6 +42,10 @@
io.netty
netty-buffer
+
+junit
+junit
+
diff --git
a/inlong-audit/audit-common/src/main/java/org/apache/inlong/audit/entities/AuditComponent.java
b/inlong-audit/audit-common/src/main/java/org/apache/inlong/audit/entity/AuditComponent.java
similarity index 81%
rename from
inlong-audit/audit-common/src/main/java/org/apache/inlong/audit/entities/AuditComponent.java
rename to
inlong-audit/audit-common/src/main/java/org/apache/inlong/audit/entity/AuditComponent.java
index d7e077e3fa..b0d2e17b54 100644
---
a/inlong-audit/audit-common/src/main/java/org/apache/inlong/audit/entities/AuditComponent.java
+++
b/inlong-audit/audit-common/src/main/java/org/apache/inlong/audit/entity/AuditComponent.java
@@ -15,17 +15,29 @@
* limitations under the License.
*/
-package org.apache.inlong.audit.entities;
+package org.apache.inlong.audit.entity;
public enum AuditComponent {
AGENT("Agent"), DATAPROXY("DataProxy"), SORT("Sort"),
COMMON_AUDIT("Common");
private final String component;
+/**
+ * Constructor for the enum.
+ *
+ * @param component the name of the component
+ */
+
AuditComponent(String component) {
this.component = component;
}
+/**
+ * Returns the name of the component.
+ *
+ * @return the name of the component
+ */
+
public String getComponent() {
return component;
}
diff --git
a/inlong-audit/audit-service/src/main/java/org/apache/inlong/audit/entities/AuditProxy.java
b/inlong-audit/audit-common/src/main/java/org/apache/inlong/audit/entity/AuditProxy.java
similarity index 71%
rename from
inlong-audit/audit-service/src/main/java/org/apache/inlong/audit/entities/AuditProxy.java
rename to
inlong-audit/audit-common/src/main/java/org/apache/inlong/audit/entity/AuditProxy.java
index c594ae2ffe..8dc2e6f7e8 100644
---
a/inlong-audit/audit-service/src/main/java/org/apache/inlong/audit/entities/AuditProxy.java
+++
b/inlong-audit/audit-common/src/main/java/org/apache/inlong/audit/entity/AuditProxy.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.inlong.audit.entities;
+package org.apache.inlong.audit.entity;
import lombok.AllArgsConstructor;
import lombok.Data;
@@ -24,6 +24,23 @@ import lombok.Data;
@AllArgsConstructor
public class AuditProxy {
+/**
+ * The host of the audit proxy.
+ */
private String host;
+
+/**
+ * The port of the audit proxy.
+ */
private int port;
+
+/**
+ * Returns a string representation of the audit proxy.
+ *
+ * @return a string representation of the audit proxy
+ */
+@Override
+public String toString() {
+return String.format("%s:%d", host, port);
+}
}
diff --git
a/inlong-audit/audit-common/src/main/java/org/apache/inlong/audit/entity/CommonResponse.java
b/inlong-audit/audit-common/src/main/java/org/apache/inlong/audit/entit
(inlong) branch master updated: [INLONG-10774][SDK] Optimize Cmake compilation script for CPP DataProxy SDK (#10777)
This is an automated email from the ASF dual-hosted git repository.
doleyzi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new 12c2c5db95 [INLONG-10774][SDK] Optimize Cmake compilation script for
CPP DataProxy SDK (#10777)
12c2c5db95 is described below
commit 12c2c5db95d9097a6acf6fccc05c2151d5716b78
Author: doleyzi <43397300+dole...@users.noreply.github.com>
AuthorDate: Tue Aug 13 11:03:59 2024 +0800
[INLONG-10774][SDK] Optimize Cmake compilation script for CPP DataProxy SDK
(#10777)
---
.../dataproxy-sdk-cpp/CMakeLists.txt| 3 +--
.../dataproxy-sdk-twins/dataproxy-sdk-cpp/README.md | 21 -
.../dataproxy-sdk-twins/dataproxy-sdk-cpp/build.sh | 11 +--
.../{build.sh => build_third_party.sh} | 16 +++-
4 files changed, 17 insertions(+), 34 deletions(-)
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/CMakeLists.txt
b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/CMakeLists.txt
index 236843494e..ccbab35336 100644
--- a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/CMakeLists.txt
+++ b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/CMakeLists.txt
@@ -37,7 +37,6 @@ include_directories(src/client)
link_directories(${PROJECT_SOURCE_DIR}/third_party/lib)
link_directories(${PROJECT_SOURCE_DIR}/third_party/lib64)
-add_subdirectory(third_party)
add_subdirectory(src/utils)
add_subdirectory(src/config)
add_subdirectory(src/core)
@@ -62,5 +61,5 @@ add_library(dataproxy_sdk STATIC ${UTILS} ${CONFIGS} ${CORE}
${MANAGER} ${GROUP}
set_target_properties(dataproxy_sdk PROPERTIES OUTPUT_NAME "dataproxy_sdk"
PREFIX "")
-target_link_libraries(dataproxy_sdk liblog4cplusS.a libsnappy.a libcurl.a)
+target_link_libraries(dataproxy_sdk liblog4cplusS.a libsnappy.a libcurl.a
libssl.a libcrypto.a)
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/README.md
b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/README.md
index c136444328..fb50cbadf9 100644
--- a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/README.md
+++ b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/README.md
@@ -30,12 +30,15 @@ dataproxy-sdk cpp version, used for sending data to
dataproxy
* curl
* rapidjson
* asio
+* log4cplus
## Build
Go to the dataproxy-sdk-cpp root, and run
```
+./build_third_party.sh
+
./build.sh
```
@@ -74,24 +77,24 @@ Refer to `release/conf/config_example.json`.
1. First, init dataproxy-sdk, there are two ways you can choose:
-- A) `int32_t tc_api_init(const char* config_file)`. Here, `config_file` is
the path of your config file, and absolute
+- A) `int32_t InitApi(const char* config_file)`. Here, `config_file` is the
path of your config file, and absolute
path is recommended. Note that only once called is needed in one process.
-- B) `int32_t tc_api_init(ClientConfig& client_config)`. Here, `client_config`
is the pointer of a `ClientConfig`
- object.
2. Then, send
- data: `int32_t tc_api_send(const char* inlong_group_id, const char*
inlong_stream_id, const char* msg, int32_t msg_len, UserCallBack call_back =
NULL)`.
+ data: `int32_t Send(const char* inlong_group_id, const char*
inlong_stream_id, const char* msg, int32_t msg_len, UserCallBack call_back =
NULL)`.
If you set `call_back`, it will be callbacked if your data failed to send.
See the signature of `UserCallBack`
- in `release/inc/user_msg.h`.
+ in `src/core/sdk_msg.h`.
-3. Finally, close sdk if no more data to be sent: `int32_t
tc_api_close(int32_t max_waitms)`. Here, `max_waitms` is the
+3. Finally, close sdk if no more data to be sent: `int32_t CloseApi(int32_t
max_waitms)`. Here, `max_waitms` is the
interval of waiting data in memory to be sent.
-4. Note, the above functions return 0 if success, otherwise it means failure.
As for other return results, please refer
- to `SDKInvalidResult` in `release/inc/tc_api.h`.
+4. Note, the above functions return 0 if success, otherwise it m
+
+5. eans failure. As for other return results, please refer
+ to `SDKInvalidResult` in `src/core/inlong_api.h`.
## Demo
1. Refer to `release/demo/send_demo.cc`.
-2. Static lib is in `release/lib`. Header file is in `release/inc`.
+2. Static lib is in `release/lib`. Header file is in `release/lib`.
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/build.sh
b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/build.sh
index b0d0e9e571..aa914e9c1e 100755
--- a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/build.sh
+++ b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/build.sh
@@ -19,17 +19,8 @@
#!/bin/bash
-
-# Install third-party components
-cd ./third_party
-cmake .
-make
-
-cd ../
rm -r build
mkdir build
-
-# Compile project code
cd build
cmake ../
-make
+make -j4
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp
(inlong) branch master updated: [INLONG-10799][SDK] Support fork child process for DataProxy CPP SDK (#10800)
This is an automated email from the ASF dual-hosted git repository.
doleyzi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new 29612cd81a [INLONG-10799][SDK] Support fork child process for
DataProxy CPP SDK (#10800)
29612cd81a is described below
commit 29612cd81a0d3ed0ac9633a0ede74d730db39bfc
Author: doleyzi <43397300+dole...@users.noreply.github.com>
AuthorDate: Fri Aug 16 13:18:34 2024 +0800
[INLONG-10799][SDK] Support fork child process for DataProxy CPP SDK
(#10800)
---
.../dataproxy-sdk-cpp/src/config/sdk_conf.cc | 6 --
.../dataproxy-sdk-cpp/src/config/sdk_conf.h| 4 +---
.../dataproxy-sdk-cpp/src/manager/proxy_manager.cc | 10 +-
.../dataproxy-sdk-cpp/src/manager/proxy_manager.h | 6 --
4 files changed, 14 insertions(+), 12 deletions(-)
diff --git
a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/config/sdk_conf.cc
b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/config/sdk_conf.cc
index 68cc122c4c..edbc075a67 100644
--- a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/config/sdk_conf.cc
+++ b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/config/sdk_conf.cc
@@ -28,8 +28,10 @@
#include "../utils/utils.h"
namespace inlong {
-SdkConfig *SdkConfig::instance_ = new SdkConfig();
-SdkConfig *SdkConfig::getInstance() { return SdkConfig::instance_; }
+SdkConfig *SdkConfig::getInstance() {
+ static SdkConfig instance;
+ return &instance;
+}
bool SdkConfig::ParseConfig(const std::string &config_path) {
// Ensure the data consistency of each sdk instance
diff --git
a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/config/sdk_conf.h
b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/config/sdk_conf.h
index f8581a1198..3f46b1a0f4 100644
--- a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/config/sdk_conf.h
+++ b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/config/sdk_conf.h
@@ -29,7 +29,6 @@
namespace inlong {
class SdkConfig {
private:
- static SdkConfig *instance_;
std::string config_path_;
std::mutex mutex_;
void InitThreadParam(const rapidjson::Value &doc);
@@ -41,6 +40,7 @@ private:
void InitAuthParm(const rapidjson::Value &doc);
void OthersParam(const rapidjson::Value &doc);
bool GetLocalIPV4Address(std::string& err_info, std::string& localhost);
+ SdkConfig() { defaultInit(); };
public:
// cache parameter
@@ -116,8 +116,6 @@ private:
void defaultInit();
static SdkConfig *getInstance();
- SdkConfig() { defaultInit(); }
-
bool ParseConfig(const std::string &config_path);
void ShowClientConfig();
diff --git
a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/manager/proxy_manager.cc
b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/manager/proxy_manager.cc
index e3771db99e..e4aef239e4 100644
---
a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/manager/proxy_manager.cc
+++
b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/manager/proxy_manager.cc
@@ -28,17 +28,17 @@
namespace inlong {
const uint64_t MINUTE = 6;
-ProxyManager *ProxyManager::instance_ = new ProxyManager();
ProxyManager::~ProxyManager() {
- if (update_conf_thread_.joinable()) {
-update_conf_thread_.join();
- }
-
+ LOG_INFO("~ProxyManager");
exit_flag_ = true;
std::unique_lock con_lck(cond_mutex_);
update_flag_ = true;
con_lck.unlock();
cond_.notify_one();
+
+ if (update_conf_thread_.joinable()) {
+update_conf_thread_.join();
+ }
}
void ProxyManager::Init() {
timeout_ = SdkConfig::getInstance()->manager_url_timeout_;
diff --git
a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/manager/proxy_manager.h
b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/manager/proxy_manager.h
index f07333c989..3cd3e5491f 100644
---
a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/manager/proxy_manager.h
+++
b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/manager/proxy_manager.h
@@ -29,7 +29,6 @@
namespace inlong {
class ProxyManager {
private:
- static ProxyManager *instance_;
uint32_t timeout_;
read_write_mutex groupid_2_cluster_id_rwmutex_;
read_write_mutex groupid_2_proxy_map_rwmutex_;
@@ -59,7 +58,10 @@ private:
public:
ProxyManager(){};
~ProxyManager();
- static ProxyManager *GetInstance() { return instance_; }
+ static ProxyManager *GetInstance() {
+static ProxyManager instance;
+return &instance;
+ }
int32_t CheckBidConf(const std::string &inlong_group_id, bool is_inited);
void Update();
void DoUpdate();
(inlong) branch master updated: [INLONG-10821][SDK] Optimize the ability to receive data for DataProxy C++ SDK (#10835)
This is an automated email from the ASF dual-hosted git repository.
doleyzi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new fb395fe0e0 [INLONG-10821][SDK] Optimize the ability to receive data
for DataProxy C++ SDK (#10835)
fb395fe0e0 is described below
commit fb395fe0e0a44c1ade30f084c138aeaf7232e2bf
Author: doleyzi <43397300+dole...@users.noreply.github.com>
AuthorDate: Wed Aug 21 14:01:34 2024 +0800
[INLONG-10821][SDK] Optimize the ability to receive data for DataProxy C++
SDK (#10835)
---
.../dataproxy-sdk-cpp/src/client/tcp_client.cc | 246 +++---
.../dataproxy-sdk-cpp/src/client/tcp_client.h | 71 ++--
.../dataproxy-sdk-cpp/src/config/sdk_conf.cc | 7 +
.../dataproxy-sdk-cpp/src/config/sdk_conf.h| 1 +
.../dataproxy-sdk-cpp/src/core/api_code.h | 3 +-
.../dataproxy-sdk-cpp/src/core/api_imp.cc | 104 +++---
.../dataproxy-sdk-cpp/src/core/api_imp.h | 38 +--
.../dataproxy-sdk-cpp/src/core/sdk_msg.h | 11 +-
.../dataproxy-sdk-cpp/src/group/recv_group.cc | 365 +++--
.../dataproxy-sdk-cpp/src/group/recv_group.h | 77 +++--
.../dataproxy-sdk-cpp/src/manager/proxy_manager.cc | 6 +-
.../dataproxy-sdk-cpp/src/manager/proxy_manager.h | 2 +-
.../dataproxy-sdk-cpp/src/utils/capi_constant.h| 5 +
.../dataproxy-sdk-cpp/src/utils/send_buffer.h | 4 +-
14 files changed, 511 insertions(+), 429 deletions(-)
diff --git
a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/client/tcp_client.cc
b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/client/tcp_client.cc
index e27910c4e9..eb3264ba6c 100644
--- a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/client/tcp_client.cc
+++ b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/client/tcp_client.cc
@@ -1,47 +1,55 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
*/
#include "tcp_client.h"
-#include "../utils/utils.h"
-#include "api_code.h"
+
#include
+#include "../manager/buffer_manager.h"
+#include "../manager/metric_manager.h"
+#include "../utils/utils.h"
+
namespace inlong {
#define CLIENT_INFO client_info_ << "[" << status_ << "]"
TcpClient::TcpClient(IOContext &io_context, std::string ip, uint32_t port)
: socket_(std::make_shared(io_context)),
wait_timer_(std::make_shared(io_context)),
keep_alive_timer_(std::make_shared(io_context)),
- ip_(ip), port_(port), endpoint_(asio::ip::address::from_string(ip),
port),
- status_(kUndefined), recv_buf_(new BlockMemory()), exit_(false),
- proxy_loads_(30), wait_heart_beat_(false), reset_client_(false),
- heart_beat_index_(0), only_heart_heat_(false) {
+ ip_(ip),
+ port_(port),
+ endpoint_(asio::ip::address::from_string(ip), port),
+ status_(kUndefined),
+ recv_buf_(new BlockMemory()),
+ exit_(false),
+ proxy_loads_(30),
+ wait_heart_beat_(false),
+ reset_client_(false),
+ heart_beat_index_(0),
+ only_heart_heat_(false),
+ need_retry_(false),
+ retry_times_(0) {
client_info_ = " [" + ip_ + ":" + std::to_string(port_) + "]&q
(inlong) branch master updated: [INLONG-10838][SDK] Optimize the ability to send data for DataProxy C++ SDK (#10850)
This is an automated email from the ASF dual-hosted git repository.
doleyzi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new 9ac4e5af08 [INLONG-10838][SDK] Optimize the ability to send data for
DataProxy C++ SDK (#10850)
9ac4e5af08 is described below
commit 9ac4e5af08812b6b0364a5296f9afc63433cee82
Author: doleyzi <43397300+dole...@users.noreply.github.com>
AuthorDate: Wed Aug 21 18:46:39 2024 +0800
[INLONG-10838][SDK] Optimize the ability to send data for DataProxy C++ SDK
(#10850)
---
.../dataproxy-sdk-cpp/include/inlong_api.h | 54 +
.../dataproxy-sdk-cpp/src/client/tcp_client.cc | 10 +-
.../dataproxy-sdk-cpp/src/config/sdk_conf.cc | 6 +
.../dataproxy-sdk-cpp/src/config/sdk_conf.h| 2 +-
.../dataproxy-sdk-cpp/src/core/api_imp.cc | 34 +--
.../dataproxy-sdk-cpp/src/core/api_imp.h | 10 +-
.../dataproxy-sdk-cpp/src/core/inlong_api.cc | 29 ++-
.../dataproxy-sdk-cpp/src/core/inlong_api.h| 44 ++--
.../dataproxy-sdk-cpp/src/core/sdk_msg.h | 37 +++-
.../dataproxy-sdk-cpp/src/group/recv_group.cc | 10 +-
.../dataproxy-sdk-cpp/src/group/send_group.cc | 236 +
.../dataproxy-sdk-cpp/src/group/send_group.h | 57 +++--
.../dataproxy-sdk-cpp/src/protocol/msg_protocol.cc | 4 +-
.../dataproxy-sdk-cpp/src/utils/capi_constant.h| 1 +
.../dataproxy-sdk-cpp/src/utils/send_buffer.h | 153 +++--
15 files changed, 362 insertions(+), 325 deletions(-)
diff --git
a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/include/inlong_api.h
b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/include/inlong_api.h
new file mode 100644
index 00..8f79c3b802
--- /dev/null
+++ b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/include/inlong_api.h
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef INLONG_SDK_API_H
+#define INLONG_SDK_API_H
+
+#include
+#include
+#include
+#include
+#include
+
+namespace inlong {
+
+typedef int (*UserCallBack)(const char *, const char *, const char *, int32_t,
+ const int64_t, const char *);
+
+class ApiImp;
+
+class InLongApi {
+ public:
+ InLongApi();
+ ~InLongApi();
+ int32_t InitApi(const char *config_path);
+
+ int32_t AddInLongGroupId(const std::vector &group_ids);
+
+ int32_t Send(const char *inlong_group_id, const char *inlong_stream_id,
const char *msg, int32_t msg_len,
+ UserCallBack call_back = nullptr);
+
+ int32_t Send(const char *inlong_group_id, const char *inlong_stream_id,
const char *msg, int32_t msg_len,
+ int64_t data_time, UserCallBack call_back = nullptr);
+
+ int32_t CloseApi(int32_t max_waitms);
+
+ private:
+ std::shared_ptr api_impl_;
+};
+} // namespace inlong
+#endif // INLONG_SDK_API_H
diff --git
a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/client/tcp_client.cc
b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/client/tcp_client.cc
index eb3264ba6c..c3b7692809 100644
--- a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/client/tcp_client.cc
+++ b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/client/tcp_client.cc
@@ -164,7 +164,7 @@ void TcpClient::BeginWrite() {
}
last_update_time_ = Utils::getCurrentMsTime();
status_ = kWriting;
- asio::async_write(*socket_, asio::buffer(sendBuffer_->content(),
sendBuffer_->len()),
+ asio::async_write(*socket_, asio::buffer(sendBuffer_->GetData(),
sendBuffer_->GetDataLen()),
std::bind(&TcpClient::OnWroten, this,
std::placeholders::_1, std::placeholders::_2));
}
void TcpClient::OnWroten(const asio::error_code error, std::size_t
bytes_transferred) {
@@ -390,8 +390,8 @@ void TcpClient::ParseHeartBeat(size_t total_length) {
void TcpClient::ParseGenericResponse() {
if (sendBuffer_ != nullptr) {
-std::string stat_key = sendBuffer_->getGroupId() + kStatJoiner +
sendBuffer_->getStreamId();
-stat_map_[stat_key].AddSendSuccessMsgNum(sendBuffer_->msgCnt());
+std::string s
(inlong) branch master updated: [INLONG-11315][Audit] Resolve the conflict between the Audit SDK and other components' Protobuf versions (#11316)
This is an automated email from the ASF dual-hosted git repository.
doleyzi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new c7f358f4bf [INLONG-11315][Audit] Resolve the conflict between the
Audit SDK and other components' Protobuf versions (#11316)
c7f358f4bf is described below
commit c7f358f4bffcf641c36cbbe235db0ade3e14f56c
Author: doleyzi <43397300+dole...@users.noreply.github.com>
AuthorDate: Thu Oct 10 09:56:07 2024 +0800
[INLONG-11315][Audit] Resolve the conflict between the Audit SDK and other
components' Protobuf versions (#11316)
---
inlong-audit/audit-common/pom.xml | 30 ++
1 file changed, 30 insertions(+)
diff --git a/inlong-audit/audit-common/pom.xml
b/inlong-audit/audit-common/pom.xml
index 3ecbd4b3d7..aaf5048b29 100644
--- a/inlong-audit/audit-common/pom.xml
+++ b/inlong-audit/audit-common/pom.xml
@@ -67,6 +67,36 @@
+
+org.apache.maven.plugins
+maven-shade-plugin
+${plugin.shade.version}
+
+
+shade-audit
+
+shade
+
+package
+
+
+
false
+
+
+org.apache.inlong:*
+com.google.protobuf:*
+
+
+
+
+com.google.protobuf
+
org.apache.inlong.audit.shaded.com.google.protobuf
+
+
+
+
+
+
(inlong) branch master updated: [INLONG-11404][Audit] Optimize the Audit item in dashboard display (#11405)
This is an automated email from the ASF dual-hosted git repository.
doleyzi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new 7a1c89769c [INLONG-11404][Audit] Optimize the Audit item in dashboard
display (#11405)
7a1c89769c is described below
commit 7a1c89769c5fcce9f7380d95b5c39c1e10829431
Author: doleyzi <43397300+dole...@users.noreply.github.com>
AuthorDate: Thu Oct 24 18:53:43 2024 +0800
[INLONG-11404][Audit] Optimize the Audit item in dashboard display (#11405)
---
.../inlong/audit/util/AuditManagerUtils.java | 57 --
1 file changed, 53 insertions(+), 4 deletions(-)
diff --git
a/inlong-audit/audit-sdk/src/main/java/org/apache/inlong/audit/util/AuditManagerUtils.java
b/inlong-audit/audit-sdk/src/main/java/org/apache/inlong/audit/util/AuditManagerUtils.java
index e8802b593d..f470251b0e 100644
---
a/inlong-audit/audit-sdk/src/main/java/org/apache/inlong/audit/util/AuditManagerUtils.java
+++
b/inlong-audit/audit-sdk/src/main/java/org/apache/inlong/audit/util/AuditManagerUtils.java
@@ -28,6 +28,8 @@ import org.slf4j.LoggerFactory;
import java.util.LinkedList;
import java.util.List;
+import static org.apache.inlong.audit.AuditIdEnum.*;
+
/**
* Audit item ID generation rules: composed of basic audit item ID + extension
bits.
* Each module is assigned two basic audit item IDs, namely reception and
transmission.
@@ -169,22 +171,69 @@ public class AuditManagerUtils {
private static List combineAuditInformation(String
auditType, FlowType flowType) {
List auditInformationList = new LinkedList<>();
boolean[] combinations = {true, false};
+
for (boolean success : combinations) {
for (boolean isRealtime : combinations) {
for (boolean discard : combinations) {
for (boolean retry : combinations) {
-if (discard && retry) {
-continue;
+if (shouldIncludeCombination(auditType, flowType,
success, isRealtime, discard, retry)) {
+auditInformationList.add(
+buildAuditInformation(auditType, flowType,
success, isRealtime, discard, retry));
}
-auditInformationList
-.add(buildAuditInformation(auditType,
flowType, success, isRealtime, discard, retry));
}
}
}
}
+
return auditInformationList;
}
+/**
+ * Exclude some uncommon audit scenarios
+ * @param auditType
+ * @param flowType
+ * @param success
+ * @param isRealtime
+ * @param discard
+ * @param retry
+ * @return
+ */
+private static boolean shouldIncludeCombination(String auditType, FlowType
flowType, boolean success,
+boolean isRealtime, boolean discard, boolean retry) {
+// Exclude the situation when retry and discard occur at the same time
+if (discard && retry) {
+return false;
+}
+
+AuditIdEnum baseAuditId = AuditIdEnum.getAuditId(auditType, flowType);
+// Exclude the situation when non-real-time and one of
SDK、Agent、DataProxy occur at the same time
+if (!isRealtime && isExcludedWhenNotRealtime(baseAuditId)) {
+return false;
+}
+
+// Exclude the situation when failed、input and one of discard and
retry occur at the same time
+if (!success && flowType == FlowType.INPUT && (discard || retry)) {
+return false;
+}
+
+// Exclude the situation when failed、output and discard occur at the
same time
+if (!success && flowType == FlowType.OUTPUT && discard) {
+return false;
+}
+
+// Exclude the situation when success、input and retry occur at the
same time
+if (success && flowType == FlowType.INPUT && retry) {
+return false;
+}
+
+return true;
+}
+
+private static boolean isExcludedWhenNotRealtime(AuditIdEnum baseAuditId) {
+return baseAuditId == SDK_INPUT || baseAuditId == SDK_OUTPUT ||
baseAuditId == AGENT_INPUT
+|| baseAuditId == AGENT_OUTPUT || baseAuditId ==
DATA_PROXY_INPUT || baseAuditId == DATA_PROXY_OUTPUT;
+}
+
/**
* Get max Audit ID.
*
(inlong) branch master updated: [INLONG-11489][Audit] Use Throwable instead of Exception to capture Java package conflicts (#11490)
This is an automated email from the ASF dual-hosted git repository.
doleyzi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new 28c3c6ba88 [INLONG-11489][Audit] Use Throwable instead of Exception to
capture Java package conflicts (#11490)
28c3c6ba88 is described below
commit 28c3c6ba88725d495ce58af7e2c4677d04ee70d9
Author: doleyzi <43397300+dole...@users.noreply.github.com>
AuthorDate: Tue Nov 12 20:45:32 2024 +0800
[INLONG-11489][Audit] Use Throwable instead of Exception to capture Java
package conflicts (#11490)
---
.../src/main/java/org/apache/inlong/audit/utils/HttpUtils.java | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git
a/inlong-audit/audit-common/src/main/java/org/apache/inlong/audit/utils/HttpUtils.java
b/inlong-audit/audit-common/src/main/java/org/apache/inlong/audit/utils/HttpUtils.java
index 34c60dc66b..694674dbf8 100644
---
a/inlong-audit/audit-common/src/main/java/org/apache/inlong/audit/utils/HttpUtils.java
+++
b/inlong-audit/audit-common/src/main/java/org/apache/inlong/audit/utils/HttpUtils.java
@@ -98,8 +98,9 @@ public class HttpUtils {
return responseStr;
}
}
-} catch (Exception e) {
-LOGGER.error("Send get request has exception", e);
+} catch (Throwable e) {
+LOGGER.error("Http request url = {}, secretId = {}, secretKey =
{}, component = {} has exception!", url,
+secretId, secretKey, component, e);
}
return null;
}
(inlong-website) branch INLONG-1041 created (now d7d23caa7ff)
This is an automated email from the ASF dual-hosted git repository. doleyzi pushed a change to branch INLONG-1041 in repository https://gitbox.apache.org/repos/asf/inlong-website.git at d7d23caa7ff [INLONG-1041][Audit] Update audit documentation instructions No new revisions were added by this update.
(inlong-website) branch INLONG-1041 updated (d7d23caa7ff -> 39fb0cddf3f)
This is an automated email from the ASF dual-hosted git repository. doleyzi pushed a change to branch INLONG-1041 in repository https://gitbox.apache.org/repos/asf/inlong-website.git from d7d23caa7ff [INLONG-1041][Audit] Update audit documentation instructions add 39fb0cddf3f [INLONG-1041][Audit] Update audit documentation instructions No new revisions were added by this update. Summary of changes: docs/modules/audit/overview.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
(inlong-website) branch INLONG-1041 updated (39fb0cddf3f -> c10877de08b)
This is an automated email from the ASF dual-hosted git repository. doleyzi pushed a change to branch INLONG-1041 in repository https://gitbox.apache.org/repos/asf/inlong-website.git from 39fb0cddf3f [INLONG-1041][Audit] Update audit documentation instructions add c10877de08b [INLONG-1041][Audit] Update paragraph numbers No new revisions were added by this update. Summary of changes: docs/modules/audit/overview.md | 16 .../current/modules/audit/overview.md| 14 +++--- 2 files changed, 15 insertions(+), 15 deletions(-)
(inlong) branch master updated: [INLONG-11620][Audit] Provide an open API for module reconciliation (#11640)
This is an automated email from the ASF dual-hosted git repository.
doleyzi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new 303d0c64cc [INLONG-11620][Audit] Provide an open API for module
reconciliation (#11640)
303d0c64cc is described below
commit 303d0c64ccd1a7288e7df621dcc44495dfa92f47
Author: doleyzi <43397300+dole...@users.noreply.github.com>
AuthorDate: Sun Jan 5 10:51:28 2025 +0800
[INLONG-11620][Audit] Provide an open API for module reconciliation (#11640)
Co-authored-by: doleyzi
---
.../audit/Exception/InvalidRequestException.java} | 12 +-
.../inlong/audit/consts/OpenApiConstants.java | 4 +-
.../apache/inlong/audit/service/auditor/Audit.java | 190 +
.../audit/service/auditor/ReconciliationData.java | 60 +++
.../ApiType.java => auditor/RequestInfo.java} | 21 ++-
.../inlong/audit/service/cache/AbstractCache.java | 45 ++---
.../inlong/audit/service/cache/RealTimeQuery.java | 70
.../inlong/audit/service/config/SqlConstants.java | 73
.../inlong/audit/service/entities/ApiType.java | 2 +-
.../inlong/audit/service/node/ApiService.java | 104 ---
.../inlong/audit/service/utils/AuditUtils.java | 93 ++
11 files changed, 615 insertions(+), 59 deletions(-)
diff --git
a/inlong-audit/audit-service/src/main/java/org/apache/inlong/audit/service/entities/ApiType.java
b/inlong-audit/audit-common/src/main/java/org/apache/inlong/audit/Exception/InvalidRequestException.java
similarity index 81%
copy from
inlong-audit/audit-service/src/main/java/org/apache/inlong/audit/service/entities/ApiType.java
copy to
inlong-audit/audit-common/src/main/java/org/apache/inlong/audit/Exception/InvalidRequestException.java
index 1703fdaf4f..5f9703d5e2 100644
---
a/inlong-audit/audit-service/src/main/java/org/apache/inlong/audit/service/entities/ApiType.java
+++
b/inlong-audit/audit-common/src/main/java/org/apache/inlong/audit/Exception/InvalidRequestException.java
@@ -15,11 +15,11 @@
* limitations under the License.
*/
-package org.apache.inlong.audit.service.entities;
+package org.apache.inlong.audit.Exception;
-/**
- * OpenAPI type
- */
-public enum ApiType {
-MINUTES, HOUR, DAY, GET_IPS, GET_IDS, GET_AUDIT_PROXY;
+public class InvalidRequestException extends Exception {
+
+public InvalidRequestException(String message) {
+super(message);
+}
}
diff --git
a/inlong-audit/audit-common/src/main/java/org/apache/inlong/audit/consts/OpenApiConstants.java
b/inlong-audit/audit-common/src/main/java/org/apache/inlong/audit/consts/OpenApiConstants.java
index cdb2a05fda..cefd789aaa 100644
---
a/inlong-audit/audit-common/src/main/java/org/apache/inlong/audit/consts/OpenApiConstants.java
+++
b/inlong-audit/audit-common/src/main/java/org/apache/inlong/audit/consts/OpenApiConstants.java
@@ -35,6 +35,8 @@ public class OpenApiConstants {
public static final String DEFAULT_API_GET_IDS_PATH =
"/audit/query/getIds";
public static final String KEY_API_GET_AUDIT_PROXY_PATH =
"api.get.audit.proxy";
public static final String DEFAULT_API_GET_AUDIT_PROXY_PATH =
"/audit/query/getAuditProxy";
+public static final String KEY_API_RECONCILIATION_PATH =
"api.reconciliation.path";
+public static final String DEFAULT_API_RECONCILIATION_PATH =
"/audit/query/reconciliation";
public static final String KEY_API_THREAD_POOL_SIZE =
"api.thread.pool.size";
public static final int DEFAULT_API_THREAD_POOL_SIZE = 10;
public static final String KEY_API_BACKLOG_SIZE = "api.backlog.size";
@@ -60,7 +62,7 @@ public class OpenApiConstants {
public static final String PARAMS_AUDIT_CYCLE = "auditCycle";
public static final String KEY_HTTP_BODY_SUCCESS = "success";
public static final String KEY_HTTP_BODY_ERR_MSG = "errMsg";
-public static final String KEY_HTTP_BODY_ERR_DATA = "data";
+public static final String KEY_HTTP_BODY_DATA = "data";
public static final String KEY_HTTP_HEADER_CONTENT_TYPE = "Content-Type";
public static final String VALUE_HTTP_HEADER_CONTENT_TYPE =
"application/json;charset=utf-8";
public static final String KEY_HTTP_SERVER_BIND_PORT =
"api.http.server.bind.port";
diff --git
a/inlong-audit/audit-service/src/main/java/org/apache/inlong/audit/service/auditor/Audit.java
b/inlong-audit/audit-service/src/main/java/org/apache/inlong/audit/service/auditor/Audit.java
new file mode 100644
index 00..8d1015a448
--- /dev/null
+++
b/inlong-audit/audit-service/src/main/java/org/apache/inlong/audit/service/auditor/Audit.java
@@ -0,0 +1,190 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor lic
