[GitHub] [inlong] healchow merged pull request #6790: [INLONG-6785][Manager] Support register and manage the resource of Apache Hudi
healchow merged PR #6790: URL: https://github.com/apache/inlong/pull/6790 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[inlong] branch master updated: [INLONG-6785][Manager] Support register and manage the resource of Apache Hudi (#6790)
This is an automated email from the ASF dual-hosted git repository.
healchow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new e493b79d8 [INLONG-6785][Manager] Support register and manage the
resource of Apache Hudi (#6790)
e493b79d8 is described below
commit e493b79d8a68d53f7fcf745b4394e14b25c65758
Author: averyzhang
AuthorDate: Tue Dec 13 16:18:48 2022 +0800
[INLONG-6785][Manager] Support register and manage the resource of Apache
Hudi (#6790)
---
.../inlong/manager/client/File2HudiExample.java| 160 ++
.../inlong/manager/common/consts/DataNodeType.java | 1 +
.../inlong/manager/common/consts/SinkType.java | 1 +
.../plugin/flink/enums/ConnectorJarType.java | 2 +
.../plugin/listener/RestartStreamListener.java | 10 +-
.../manager/pojo/node/hudi/HudiDataNodeDTO.java| 74 +++
.../manager/pojo/node/hudi/HudiDataNodeInfo.java | 54 +
.../pojo/node/hudi/HudiDataNodeRequest.java| 49 +
.../manager/pojo/sink/hudi/HudiColumnInfo.java | 78 +++
.../manager/pojo/sink/hudi/HudiPartition.java} | 40 +++-
.../pojo/sink/hudi/HudiPartitionField.java}| 27 ++-
.../inlong/manager/pojo/sink/hudi/HudiSink.java| 91
.../inlong/manager/pojo/sink/hudi/HudiSinkDTO.java | 132 +++
.../manager/pojo/sink/hudi/HudiSinkRequest.java| 70 ++
.../manager/pojo/sink/hudi/HudiTableInfo.java} | 24 +-
.../inlong/manager/pojo/sink/hudi/HudiType.java| 60 +
.../manager/pojo/sort/util/LoadNodeUtils.java | 38
inlong-manager/manager-service/pom.xml | 4 +
.../service/node/hudi/HudiDataNodeOperator.java| 86
.../resource/sink/hudi/HudiCatalogClient.java | 242 +
.../resource/sink/hudi/HudiResourceOperator.java | 177 +++
.../service/sink/hudi/HudiSinkOperator.java| 130 +++
.../manager/service/sink/HudiSinkServiceTest.java | 94
.../sort/protocol/constant/HudiConstant.java | 56 +
.../sort/protocol/node/load/HudiLoadNode.java | 33 ++-
.../sort/protocol/node/load/HudiLoadNodeTest.java | 2 +-
.../inlong/sort/parser/HudiNodeSqlParserTest.java | 2 +-
27 files changed, 1695 insertions(+), 42 deletions(-)
diff --git
a/inlong-manager/manager-client-examples/src/test/java/org/apache/inlong/manager/client/File2HudiExample.java
b/inlong-manager/manager-client-examples/src/test/java/org/apache/inlong/manager/client/File2HudiExample.java
new file mode 100644
index 0..3ac40649b
--- /dev/null
+++
b/inlong-manager/manager-client-examples/src/test/java/org/apache/inlong/manager/client/File2HudiExample.java
@@ -0,0 +1,160 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.manager.client;
+
+import com.google.common.collect.Lists;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.inlong.manager.client.api.ClientConfiguration;
+import org.apache.inlong.manager.client.api.InlongClient;
+import org.apache.inlong.manager.client.api.InlongGroup;
+import org.apache.inlong.manager.client.api.InlongGroupContext;
+import org.apache.inlong.manager.client.api.InlongStreamBuilder;
+import org.apache.inlong.manager.common.enums.FieldType;
+import org.apache.inlong.manager.common.util.JsonUtils;
+import org.apache.inlong.manager.pojo.group.InlongGroupInfo;
+import org.apache.inlong.manager.pojo.sink.SinkField;
+import org.apache.inlong.manager.pojo.sink.hudi.HudiColumnInfo;
+import org.apache.inlong.manager.pojo.sink.hudi.HudiPartition;
+import org.apache.inlong.manager.pojo.sink.hudi.HudiSink;
+import org.apache.inlong.manager.pojo.source.file.FileSource;
+import org.apache.inlong.manager.pojo.stream.StreamField;
+import org.apache.shiro.util.Assert;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Test class for file to hudi.
+ */
+@Slf4j
+@Disabled
+public class File2HudiExample extends BaseExample {
+
+@Test
+public void testCreateGroupForHudi() {
[inlong] branch master updated: [INLONG-6751][Sort] Add read phase metric and table level metric for Oracle (#6808)
This is an automated email from the ASF dual-hosted git repository.
yunqing pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new 3126faaf5 [INLONG-6751][Sort] Add read phase metric and table level
metric for Oracle (#6808)
3126faaf5 is described below
commit 3126faaf5e8851aa62a8772eeb4096bf47034613
Author: emhui <111486498+e-m...@users.noreply.github.com>
AuthorDate: Tue Dec 13 16:49:37 2022 +0800
[INLONG-6751][Sort] Add read phase metric and table level metric for Oracle
(#6808)
---
.../base/metric/sub/SourceTableMetricData.java | 56 ---
.../sort/cdc/base/util/CallbackCollector.java | 47
.../inlong/sort/cdc/oracle/OracleSource.java | 9 +++-
.../oracle/debezium/DebeziumSourceFunction.java| 63 --
.../sort/cdc/oracle/table/OracleTableSource.java | 3 +-
5 files changed, 152 insertions(+), 26 deletions(-)
diff --git
a/inlong-sort/sort-connectors/base/src/main/java/org/apache/inlong/sort/base/metric/sub/SourceTableMetricData.java
b/inlong-sort/sort-connectors/base/src/main/java/org/apache/inlong/sort/base/metric/sub/SourceTableMetricData.java
index 112000903..a11d73711 100644
---
a/inlong-sort/sort-connectors/base/src/main/java/org/apache/inlong/sort/base/metric/sub/SourceTableMetricData.java
+++
b/inlong-sort/sort-connectors/base/src/main/java/org/apache/inlong/sort/base/metric/sub/SourceTableMetricData.java
@@ -122,9 +122,14 @@ public class SourceTableMetricData extends
SourceMetricData implements SourceSub
String metricGroupLabels = labels.entrySet().stream().map(entry ->
entry.getKey() + "=" + entry.getValue())
.collect(Collectors.joining(DELIMITER));
StringBuilder labelBuilder = new StringBuilder(metricGroupLabels);
-
labelBuilder.append(DELIMITER).append(Constants.DATABASE_NAME).append("=").append(schemaInfoArray[0])
-
.append(DELIMITER).append(Constants.TABLE_NAME).append("=").append(schemaInfoArray[1]);
-
+if (schemaInfoArray.length == 2) {
+
labelBuilder.append(DELIMITER).append(Constants.DATABASE_NAME).append("=").append(schemaInfoArray[0])
+
.append(DELIMITER).append(Constants.TABLE_NAME).append("=").append(schemaInfoArray[1]);
+} else if (schemaInfoArray.length == 3) {
+
labelBuilder.append(DELIMITER).append(Constants.DATABASE_NAME).append("=").append(schemaInfoArray[0])
+
.append(DELIMITER).append(Constants.SCHEMA_NAME).append("=").append(schemaInfoArray[1])
+
.append(DELIMITER).append(Constants.TABLE_NAME).append("=").append(schemaInfoArray[2]);
+}
MetricOption metricOption = MetricOption.builder()
.withInitRecords(subMetricState != null ?
subMetricState.getMetricValue(NUM_RECORDS_IN) : 0L)
.withInitBytes(subMetricState != null ?
subMetricState.getMetricValue(NUM_BYTES_IN) : 0L)
@@ -135,14 +140,18 @@ public class SourceTableMetricData extends
SourceMetricData implements SourceSub
}
/**
- * build record schema identify,in the form of database.table
+ * build record schema identify,in the form of database.schema.table or
database.table
*
* @param database the database name of record
+ * @param schema the schema name of record
* @param table the table name of record
* @return the record schema identify
*/
-public String buildSchemaIdentify(String database, String table) {
-return database + Constants.SEMICOLON + table;
+public String buildSchemaIdentify(String database, String schema, String
table) {
+if (schema == null) {
+return database + Constants.SEMICOLON + table;
+}
+return database + Constants.SEMICOLON + schema + Constants.SEMICOLON +
table;
}
/**
@@ -168,7 +177,7 @@ public class SourceTableMetricData extends SourceMetricData
implements SourceSub
outputMetricsWithEstimate(data);
return;
}
-String identify = buildSchemaIdentify(database, table);
+String identify = buildSchemaIdentify(database, null, table);
SourceMetricData subSourceMetricData;
if (subSourceMetricMap.containsKey(identify)) {
subSourceMetricData = subSourceMetricMap.get(identify);
@@ -186,6 +195,39 @@ public class SourceTableMetricData extends
SourceMetricData implements SourceSub
outputReadPhaseMetrics((isSnapshotRecord) ? ReadPhase.SNAPSHOT_PHASE :
ReadPhase.INCREASE_PHASE);
}
+/**
+ * output metrics with estimate
+ *
+ * @param database the database name of record
+ * @param schema the schema name of record
+ * @param table the table name of record
+ * @param isSnapshotRecord is it snapshot record
+ * @param data the data of record
+ */
+pub
[GitHub] [inlong] yunqingmoswu merged pull request #6808: [INLONG-6751][Sort] Add read phase metric and table level metric for Oracle
yunqingmoswu merged PR #6808: URL: https://github.com/apache/inlong/pull/6808 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [inlong] dockerzhang merged pull request #6846: [INLONG-6771][Manager] Add login failure limit
dockerzhang merged PR #6846: URL: https://github.com/apache/inlong/pull/6846 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[inlong] branch master updated: [INLONG-6771][Manager] Add login failure limit (#6846)
This is an automated email from the ASF dual-hosted git repository.
dockerzhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new 48b40feeb [INLONG-6771][Manager] Add login failure limit (#6846)
48b40feeb is described below
commit 48b40feeb351cd80858139deafc50d397f42863b
Author: leosanqing
AuthorDate: Tue Dec 13 17:13:40 2022 +0800
[INLONG-6771][Manager] Add login failure limit (#6846)
Co-authored-by: healchow
---
.../manager/pojo/user/UserLoginLockStatus.java | 33
.../inlong/manager/service/user/UserService.java | 6 +++
.../manager/service/user/UserServiceImpl.java | 62 ++
.../manager/web/controller/AnnoController.java | 9 +---
.../manager/web/controller/AnnoControllerTest.java | 59
5 files changed, 161 insertions(+), 8 deletions(-)
diff --git
a/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/user/UserLoginLockStatus.java
b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/user/UserLoginLockStatus.java
new file mode 100644
index 0..2aedd6f36
--- /dev/null
+++
b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/user/UserLoginLockStatus.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.manager.pojo.user;
+
+import lombok.Data;
+
+import java.time.LocalDateTime;
+
+/**
+ * Record whether the account is locked and the
+ */
+@Data
+public class UserLoginLockStatus {
+
+private int loginErrorCount;
+
+private LocalDateTime lockoutTime;
+}
diff --git
a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/user/UserService.java
b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/user/UserService.java
index a4baeb95e..9e2de4bfe 100644
---
a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/user/UserService.java
+++
b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/user/UserService.java
@@ -19,6 +19,7 @@ package org.apache.inlong.manager.service.user;
import org.apache.inlong.manager.pojo.common.PageResult;
import org.apache.inlong.manager.pojo.user.UserInfo;
+import org.apache.inlong.manager.pojo.user.UserLoginRequest;
import org.apache.inlong.manager.pojo.user.UserRequest;
/**
@@ -76,4 +77,9 @@ public interface UserService {
*/
Boolean delete(Integer userId, String currentUser);
+/**
+ * Account password login
+ */
+void login(UserLoginRequest req);
+
}
diff --git
a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/user/UserServiceImpl.java
b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/user/UserServiceImpl.java
index 212aadae5..963a21ef5 100644
---
a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/user/UserServiceImpl.java
+++
b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/user/UserServiceImpl.java
@@ -35,17 +35,26 @@ import org.apache.inlong.manager.dao.entity.UserEntity;
import org.apache.inlong.manager.dao.mapper.UserEntityMapper;
import org.apache.inlong.manager.pojo.common.PageResult;
import org.apache.inlong.manager.pojo.user.UserInfo;
+import org.apache.inlong.manager.pojo.user.UserLoginLockStatus;
+import org.apache.inlong.manager.pojo.user.UserLoginRequest;
import org.apache.inlong.manager.pojo.user.UserRequest;
+import org.apache.shiro.SecurityUtils;
+import org.apache.shiro.authc.AuthenticationException;
+import org.apache.shiro.authc.UsernamePasswordToken;
+import org.apache.shiro.subject.Subject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.nio.charset.StandardCharsets;
+import java.time.Duration;
+import java.time.LocalDateTime;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.Objects;
+import java.util.concurrent.ConcurrentHashM
[GitHub] [inlong] dockerzhang opened a new pull request, #6866: [INLONG-6865][Dashboard] Support npm ci command for stable package-lock
dockerzhang opened a new pull request, #6866: URL: https://github.com/apache/inlong/pull/6866 ### Prepare a Pull Request *(Change the title refer to the following example)* - Title Example: [INLONG-XYZ][Component] Title of the pull request *(The following *XYZ* should be replaced by the actual [GitHub Issue](https://github.com/apache/inlong/issues) number)* - Fixes #6865 ### Motivation refer to https://github.com/eirslett/frontend-maven-plugin/issues/855, to keep a stable package-lock. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [inlong] dockerzhang opened a new pull request, #6868: [INLONG-6867][CVE] Bump protobuf to 3.19.6 to fix Java vulnerable
dockerzhang opened a new pull request, #6868: URL: https://github.com/apache/inlong/pull/6868 ### Prepare a Pull Request *(Change the title refer to the following example)* - Title Example: [INLONG-XYZ][Component] Title of the pull request *(The following *XYZ* should be replaced by the actual [GitHub Issue](https://github.com/apache/inlong/issues) number)* - Fixes #6867 ### Motivation Protobuf Java vulnerable to Uncontrolled Resource Consumption -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [inlong] dockerzhang merged pull request #6866: [INLONG-6865][Dashboard] Support npm ci command for stable package-lock
dockerzhang merged PR #6866: URL: https://github.com/apache/inlong/pull/6866 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[inlong] branch master updated: [INLONG-6865][Dashboard] Support npm ci command for stable package-lock (#6866)
This is an automated email from the ASF dual-hosted git repository. dockerzhang pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/inlong.git The following commit(s) were added to refs/heads/master by this push: new 92f36200c [INLONG-6865][Dashboard] Support npm ci command for stable package-lock (#6866) 92f36200c is described below commit 92f36200cb1d48b6d2e424ec23a81f6ba029895a Author: Charles Zhang AuthorDate: Tue Dec 13 19:29:09 2022 +0800 [INLONG-6865][Dashboard] Support npm ci command for stable package-lock (#6866) --- inlong-dashboard/pom.xml | 5 - 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/inlong-dashboard/pom.xml b/inlong-dashboard/pom.xml index 66363ecc3..1bc865053 100644 --- a/inlong-dashboard/pom.xml +++ b/inlong-dashboard/pom.xml @@ -49,10 +49,13 @@ -npm-install +npm ci npm + +ci + npm run build
[GitHub] [inlong] yunqingmoswu opened a new pull request, #6870: [INLONG-6869][Sort] Supports dirty data side-output for elasticsearch sink
yunqingmoswu opened a new pull request, #6870: URL: https://github.com/apache/inlong/pull/6870 ### Prepare a Pull Request *(Change the title refer to the following example)* - Title: [INLONG-6869][Sort] Supports dirty data side-output for elasticsearch sink *(The following *XYZ* should be replaced by the actual [GitHub Issue](https://github.com/apache/inlong/issues) number)* - Fixes #6869 ### Motivation Supports dirty data side-output for elasticsearch sink. In this part: 1. Load 'DirtySinkFactory' and create 'DirtySink' by the config 2. It needs to determine whether it is dirty data in the connector. 3. Side output dirty data by the 'DirtySink' dependents on the configured, the built-in side-out of dirty data has 'LogDirtySink'(#6618) and 'S3DirtySink'(#6655). ### Modifications 1. Create a dirty sink and inject it into elasticsearch sink 2. Add dirty handle for 'ElasticsearchSinkBase' and 'RowElasticsearchSinkFunction' 3. Add a unit test for this ### Verifying this change *(Please pick either of the following options)* - [x] This change is a trivial rework/code cleanup without any test coverage. - [ ] This change is already covered by existing tests, such as: *(please describe tests)* - [ ] This change added tests and can be verified as follows: *(example:)* - *Added integration tests for end-to-end deployment with large payloads (10MB)* - *Extended integration test for recovery after broker failure* ### Documentation - Does this pull request introduce a new feature? (yes / no) - If yes, how is the feature documented? (not applicable / docs / JavaDocs / not documented) - If a feature is not applicable for documentation, explain why? - If a feature is not documented yet in this PR, please create a follow-up issue for adding the documentation -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [inlong] gosonzhang opened a new pull request, #6871: [INLONG-6860][Manager] Add open APIs related to inlong basic operations
gosonzhang opened a new pull request, #6871: URL: https://github.com/apache/inlong/pull/6871 - Fixes #6860 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [inlong] gong opened a new pull request, #6872: [INLONG-6864][Sort] Modify hive metric computing to ensure metric data accuracy
gong opened a new pull request, #6872: URL: https://github.com/apache/inlong/pull/6872 ### Prepare a Pull Request - [INLONG-6864][Sort] Modify hive metric computing to ensure metric data accuracy - Fixes #6864 ### Motivation * Modify hive metric computing to ensure metric data accuracy ### Modifications * computing metric after checkpoint -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [inlong] kinfuy commented on issue #6669: [Feature][Dashboard] Add status management for group
kinfuy commented on issue #6669: URL: https://github.com/apache/inlong/issues/6669#issuecomment-1350215975 Can you provide more descriptions for this task, and what specific needs to be done -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [inlong] ZuoFengZhang opened a new pull request, #6873: [INLONG-6861][Dashboard] Support extract node of Apache Hudi
ZuoFengZhang opened a new pull request, #6873: URL: https://github.com/apache/inlong/pull/6873 ### Prepare a Pull Request - [INLONG-6861][Dashboard] Support extract node of Apache Hudi - Fixes #6861 ### Motivation https://user-images.githubusercontent.com/5709212/207484171-3ec1386d-800d-414d-aaac-cc96480761c9.png";> https://user-images.githubusercontent.com/5709212/207484197-2914b097-a4a3-4eb8-9e88-6e87e5452b79.png";> ### Modifications *Describe the modifications you've done.* ### Verifying this change *(Please pick either of the following options)* - [x] This change is a trivial rework/code cleanup without any test coverage. - [ ] This change is already covered by existing tests, such as: *(please describe tests)* - [ ] This change added tests and can be verified as follows: *(example:)* - *Added integration tests for end-to-end deployment with large payloads (10MB)* - *Extended integration test for recovery after broker failure* ### Documentation - Does this pull request introduce a new feature? (yes / no) - If yes, how is the feature documented? (not applicable / docs / JavaDocs / not documented) - If a feature is not applicable for documentation, explain why? - If a feature is not documented yet in this PR, please create a follow-up issue for adding the documentation -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [inlong] woofyzhao commented on a diff in pull request #6871: [INLONG-6860][Manager] Add open APIs related to inlong basic operations
woofyzhao commented on code in PR #6871:
URL: https://github.com/apache/inlong/pull/6871#discussion_r1047937604
##
inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/group/InlongGroupServiceImpl.java:
##
@@ -170,6 +173,29 @@ public String save(InlongGroupRequest request, String
operator) {
return groupId;
}
+@Override
+@Transactional(rollbackFor = Throwable.class)
+public String save(InlongGroupRequest request, UserInfo opInfo) {
Review Comment:
Can we reuse the old method since most of the codes are identical?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [inlong] gong commented on a diff in pull request #6870: [INLONG-6869][Sort] Supports dirty data side-output for elasticsearch sink
gong commented on code in PR #6870:
URL: https://github.com/apache/inlong/pull/6870#discussion_r1047944224
##
inlong-sort/sort-connectors/elasticsearch-base/src/main/java/org/apache/inlong/sort/elasticsearch/table/RowElasticsearchSinkFunction.java:
##
@@ -146,55 +154,79 @@ public void snapshotState(FunctionSnapshotContext
context) throws Exception {
@Override
public void process(RowData element, RuntimeContext ctx, RequestIndexer
indexer) {
+final String key;
+try {
+key = createKey.apply(element);
+} catch (Exception e) {
+LOGGER.error(String.format("Generate index id error, raw data:
%s", element), e);
+dirtySinkHelper.invoke(element, DirtyType.INDEX_ID_GENERATE_ERROR,
e);
+return;
+}
+final byte[] document;
+try {
+document = serializationSchema.serialize(element);
+} catch (Exception e) {
+LOGGER.error(String.format("Serialize error, raw data: %s",
element), e);
+dirtySinkHelper.invoke(element, DirtyType.SERIALIZE_ERROR, e);
+return;
Review Comment:
Here need compute dirty metric when upload dirty data
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [inlong] leezng commented on a diff in pull request #6873: [INLONG-6861][Dashboard] Support extract node of Apache Hudi
leezng commented on code in PR #6873: URL: https://github.com/apache/inlong/pull/6873#discussion_r1047942090 ## inlong-dashboard/src/locales/cn.json: ## @@ -157,6 +164,10 @@ "meta.Sinks.ES.FlushRecordUnit": "条", "meta.Sinks.ES.RetryTimes": "重试次数", "meta.Sinks.ES.RetryTimesUnit": "次", + "meta.Sinks.ES.EsVersion": "Elasticsearch 版本", Review Comment: Is the irrelevant code here because there is no rebase? You need to remove this part of the code. ## inlong-dashboard/src/locales/cn.json: ## @@ -92,6 +92,13 @@ "meta.Sources.MQTT.Username": "用户", "meta.Sources.MQTT.Password": "密码", "meta.Sources.MQTT.MqttVersion": "MQTT 版本", + "meta.Sources.Hudi.DbName": "库名", + "meta.Sources.Hudi.TableName": "表名", + "meta.Sources.Hudi.Warehouse": "仓库路径", + "meta.Sources.Hudi.ReadStreamingSkipCompaction": "跳过合并中的提交", + "meta.Sources.Hudi.ReadStartCommit": "起始的Commit", + "meta.Sources.ReadStreamingSkipCompactionHelp": "流读时是否跳过 compaction 的 commits，跳过 compaction 有两个用途：1）避免 upsert 语义 下重复消费(compaction 的 instant 为重复数据，如果不跳过，有小概率会重复消费）; 2) changelog 模式下保证语义正确性。 0.11 开始，以上两个问题已经通过保留 compaction 的 instant time 修复", Review Comment: The prefix here needs to be added with `Hudi`, for example "meta.Sources.Hudi.X". -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [inlong] dockerzhang merged pull request #6859: [INLONG-6842][Sort] Improve mysql-cdc2.0 to support tables without primary key
dockerzhang merged PR #6859: URL: https://github.com/apache/inlong/pull/6859 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[inlong] branch master updated: [INLONG-6842][Sort] Improve mysql-cdc2.0 to support tables without primary key (#6859)
This is an automated email from the ASF dual-hosted git repository.
dockerzhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new cc6fcb652 [INLONG-6842][Sort] Improve mysql-cdc2.0 to support tables
without primary key (#6859)
cc6fcb652 is described below
commit cc6fcb6528aedb5be3f0f4d7d3d18d1e9f62b095
Author: Schnapps
AuthorDate: Wed Dec 14 10:32:37 2022 +0800
[INLONG-6842][Sort] Improve mysql-cdc2.0 to support tables without primary
key (#6859)
Co-authored-by: stingpeng
---
.../debezium/task/MySqlSnapshotSplitReadTask.java | 18 -
.../cdc/mysql/source/assigners/ChunkSplitter.java | 47 ++
.../cdc/mysql/source/split/MySqlSnapshotSplit.java | 8
.../mysql/table/MySqlTableInlongSourceFactory.java | 1 -
.../apache/inlong/sort/parser/AllMigrateTest.java | 2 +-
5 files changed, 65 insertions(+), 11 deletions(-)
diff --git
a/inlong-sort/sort-connectors/mysql-cdc/src/main/java/org/apache/inlong/sort/cdc/mysql/debezium/task/MySqlSnapshotSplitReadTask.java
b/inlong-sort/sort-connectors/mysql-cdc/src/main/java/org/apache/inlong/sort/cdc/mysql/debezium/task/MySqlSnapshotSplitReadTask.java
index 28fb660d9..6a37a0f18 100644
---
a/inlong-sort/sort-connectors/mysql-cdc/src/main/java/org/apache/inlong/sort/cdc/mysql/debezium/task/MySqlSnapshotSplitReadTask.java
+++
b/inlong-sort/sort-connectors/mysql-cdc/src/main/java/org/apache/inlong/sort/cdc/mysql/debezium/task/MySqlSnapshotSplitReadTask.java
@@ -149,7 +149,8 @@ public class MySqlSnapshotSplitReadTask extends
AbstractSnapshotChangeEventSourc
LOG.info("Snapshot step 2 - Snapshotting data");
createDataEvents(ctx, snapshotSplit.getTableId());
-final BinlogOffset highWatermark = currentBinlogOffset(jdbcConnection);
+BinlogOffset highWatermark = determineHighWatermark(lowWatermark);
+
LOG.info(
"Snapshot step 3 - Determining high watermark {} for split {}",
highWatermark,
@@ -162,6 +163,21 @@ public class MySqlSnapshotSplitReadTask extends
AbstractSnapshotChangeEventSourc
return SnapshotResult.completed(ctx.offset);
}
+/**
+ * for chunk that equals to the whole table we do not need to normalize
+ * the snapshot data and the binlog data, just set high watermark to low
watermark
+ * @return highWatermark
+ */
+private BinlogOffset determineHighWatermark(BinlogOffset lowWatermark) {
+if (snapshotSplit.isWholeSplit()) {
+LOG.info("for split {}, set highWatermark to lowWatermark {} since"
++ " it reads the whole table ", snapshotSplit,
lowWatermark);
+return lowWatermark;
+} else {
+return currentBinlogOffset(jdbcConnection);
+}
+}
+
@Override
protected SnapshottingTask getSnapshottingTask(OffsetContext
previousOffset) {
return new SnapshottingTask(false, true);
diff --git
a/inlong-sort/sort-connectors/mysql-cdc/src/main/java/org/apache/inlong/sort/cdc/mysql/source/assigners/ChunkSplitter.java
b/inlong-sort/sort-connectors/mysql-cdc/src/main/java/org/apache/inlong/sort/cdc/mysql/source/assigners/ChunkSplitter.java
index e4a6bd3e4..26494f3cf 100644
---
a/inlong-sort/sort-connectors/mysql-cdc/src/main/java/org/apache/inlong/sort/cdc/mysql/source/assigners/ChunkSplitter.java
+++
b/inlong-sort/sort-connectors/mysql-cdc/src/main/java/org/apache/inlong/sort/cdc/mysql/source/assigners/ChunkSplitter.java
@@ -114,17 +114,13 @@ class ChunkSplitter {
long start = System.currentTimeMillis();
Table table = mySqlSchema.getTableSchema(jdbc, tableId).getTable();
-Column splitColumn = ChunkUtils.getSplitColumn(table);
-final List chunks;
-try {
-chunks = splitTableIntoChunks(jdbc, tableId, splitColumn);
-} catch (SQLException e) {
-throw new FlinkRuntimeException("Failed to split chunks for
table " + tableId, e);
-}
+
+List chunks = getChunks(tableId, jdbc, table);
// convert chunks into splits
List splits = new ArrayList<>();
-RowType splitType = ChunkUtils.getSplitType(splitColumn);
+
+RowType splitType = ChunkUtils.getSplitType(getSplitColumn(table));
for (int i = 0; i < chunks.size(); i++) {
ChunkRange chunk = chunks.get(i);
MySqlSnapshotSplit split =
@@ -151,6 +147,41 @@ class ChunkSplitter {
}
}
+/**
+ * get the split column using primary key
+ * for those don't have primary key, return the first column
+ * @return chunks
+ */
+private Column getSplitColumn(Table table) {
+if (table.primaryKeyColumns().isEmpty()) {
+// since we do not need a split column when there is no primary key
+
[GitHub] [inlong] bluewang opened a new pull request, #6875: [INLONG-6874][Dashboard] Stream removes the Raw-CSV data format
bluewang opened a new pull request, #6875: URL: https://github.com/apache/inlong/pull/6875 ### Prepare a Pull Request *(Change the title refer to the following example)* - Title Example: [INLONG-XYZ][Component] Title of the pull request *(The following *XYZ* should be replaced by the actual [GitHub Issue](https://github.com/apache/inlong/issues) number)* - Fixes https://github.com/apache/inlong/issues/6874 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [inlong] gosonzhang commented on a diff in pull request #6871: [INLONG-6860][Manager] Add open APIs related to inlong basic operations
gosonzhang commented on code in PR #6871:
URL: https://github.com/apache/inlong/pull/6871#discussion_r1047972856
##
inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/group/InlongGroupServiceImpl.java:
##
@@ -170,6 +173,29 @@ public String save(InlongGroupRequest request, String
operator) {
return groupId;
}
+@Override
+@Transactional(rollbackFor = Throwable.class)
+public String save(InlongGroupRequest request, UserInfo opInfo) {
Review Comment:
Yes, business logic and page are currently consistent
This idea is to separate the page from the open api first, and implement the
call logic separately. After completing the subsequent implementation, we will
see if they are the same in the end: if they are consistent, we will merge them
into one; if they are inconsistent, we will merge them into one. deal with
separately
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [inlong] dockerzhang commented on pull request #6875: [INLONG-6874][Dashboard] Stream removes the Raw-CSV data format
dockerzhang commented on PR #6875: URL: https://github.com/apache/inlong/pull/6875#issuecomment-1350333594 All data types used chosen will be packaged by InLongMsg default, so we remove the raw type. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [inlong] healchow commented on issue #6771: [Improve][Manager] Add login failure limit
healchow commented on issue #6771: URL: https://github.com/apache/inlong/issues/6771#issuecomment-1350339937 > Hey did you fix it? Hey, @leosanqing already fix this issue. You can see #6846 for more detail. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [inlong] gong commented on a diff in pull request #6820: [INLONG-6819][Sort] Add multi table sink for PostgresSQL
gong commented on code in PR #6820:
URL: https://github.com/apache/inlong/pull/6820#discussion_r1047980457
##
inlong-sort/sort-connectors/jdbc/src/main/java/org/apache/inlong/sort/jdbc/internal/JdbcMultiBatchingOutputFormat.java:
##
@@ -506,13 +508,18 @@ protected void attemptFlush() throws IOException {
}
}
}
+if (!flushFlag && null != tableException) {
+LOG.info("Put tableIdentifier:{} exception:{}",
+tableIdentifier, tableException.getMessage());
+tableExceptionMap.put(tableIdentifier, tableException);
+if (isIgnoreTableException) {
+LOG.info("Stop write table:{} because occur
exception",
+tableIdentifier);
+break;
Review Comment:
`break` is right?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [inlong] gong commented on a diff in pull request #6825: [INLONG-6822][Sort] Add multi table source for MongoDb
gong commented on code in PR #6825:
URL: https://github.com/apache/inlong/pull/6825#discussion_r1047982648
##
inlong-sort/sort-connectors/mongodb-cdc/pom.xml:
##
@@ -44,6 +50,31 @@
audit-sdk
${project.version}
+
+org.apache.flink
+flink-table-common
+1.13.5
+
Review Comment:
no need it
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [inlong] gong commented on a diff in pull request #6825: [INLONG-6822][Sort] Add multi table source for MongoDb
gong commented on code in PR #6825:
URL: https://github.com/apache/inlong/pull/6825#discussion_r1047983174
##
inlong-sort/sort-connectors/mongodb-cdc/pom.xml:
##
@@ -44,6 +50,31 @@
audit-sdk
${project.version}
+
+org.apache.flink
+flink-table-common
+1.13.5
+
+
+io.debezium
+debezium-connector-oracle
+${debezium.version}
+
Review Comment:
remove it, error dependency
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [inlong] dockerzhang merged pull request #6875: [INLONG-6874][Dashboard] Stream removes the Raw-CSV data format
dockerzhang merged PR #6875: URL: https://github.com/apache/inlong/pull/6875 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[inlong] branch master updated: [INLONG-6874][Dashboard] Stream removes the Raw-CSV data format (#6875)
This is an automated email from the ASF dual-hosted git repository.
dockerzhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new 41cda6f62 [INLONG-6874][Dashboard] Stream removes the Raw-CSV data
format (#6875)
41cda6f62 is described below
commit 41cda6f62603c72c18834fab3d871692ebab6be4
Author: Lizhen <88174078+bluew...@users.noreply.github.com>
AuthorDate: Wed Dec 14 11:49:14 2022 +0800
[INLONG-6874][Dashboard] Stream removes the Raw-CSV data format (#6875)
---
inlong-dashboard/src/metas/streams/common/StreamDefaultInfo.ts | 4
1 file changed, 4 deletions(-)
diff --git a/inlong-dashboard/src/metas/streams/common/StreamDefaultInfo.ts
b/inlong-dashboard/src/metas/streams/common/StreamDefaultInfo.ts
index fd39a8cc9..83562675c 100644
--- a/inlong-dashboard/src/metas/streams/common/StreamDefaultInfo.ts
+++ b/inlong-dashboard/src/metas/streams/common/StreamDefaultInfo.ts
@@ -105,10 +105,6 @@ export class StreamDefaultInfo implements DataWithBackend,
RenderRow, RenderList
label: 'CSV',
value: 'CSV',
},
-{
- label: 'Raw-CSV',
- value: 'RAW_CSV',
-},
],
}),
rules: [{ required: true }],
[GitHub] [inlong] gong commented on a diff in pull request #6825: [INLONG-6822][Sort] Add multi table source for MongoDb
gong commented on code in PR #6825:
URL: https://github.com/apache/inlong/pull/6825#discussion_r1047989844
##
inlong-sort/sort-connectors/mongodb-cdc/src/main/java/org/apache/inlong/sort/cdc/mongodb/DebeziumSourceFunction.java:
##
@@ -493,15 +492,6 @@ public void deserialize(SourceRecord record, Collector
out) throws Exception
deserializer.deserialize(record, out);
}
-@Override
-public void deserialize(SourceRecord record,
Collector out,
-TableChanges.TableChange tableSchema)
throws Exception {
-if (sourceMetricData != null) {
-
sourceMetricData.outputMetricsWithEstimate(record.value());
-}
-deserializer.deserialize(record, out,
tableSchema);
-}
-
Review Comment:
why remove it
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [inlong] gong commented on a diff in pull request #6825: [INLONG-6822][Sort] Add multi table source for MongoDb
gong commented on code in PR #6825: URL: https://github.com/apache/inlong/pull/6825#discussion_r1047990473 ## inlong-sort/sort-connectors/mongodb-cdc/pom.xml: ## @@ -74,6 +105,7 @@ org.apache.avro:avro org.apache.kafka:* com.fasterxml.*:* +com.google.code.gson:* Review Comment: no need it -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [inlong] gong commented on pull request #6750: [INLONG-6657][Sort] Add dirty metric for Kafka
gong commented on PR #6750: URL: https://github.com/apache/inlong/pull/6750#issuecomment-1350376703 you need compute metric data after inlong-sort/sort-connectors/kafka/src/main/java/org/apache/inlong/sort/kafka/FlinkKafkaProducer#commit. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [inlong] gosonzhang merged pull request #6871: [INLONG-6860][Manager] Add open APIs related to inlong basic operations
gosonzhang merged PR #6871: URL: https://github.com/apache/inlong/pull/6871 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[inlong] branch master updated (41cda6f62 -> 690d9c0e8)
This is an automated email from the ASF dual-hosted git repository.
gosonzhang pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
from 41cda6f62 [INLONG-6874][Dashboard] Stream removes the Raw-CSV data
format (#6875)
add 690d9c0e8 [INLONG-6860][Manager] Add open APIs related to inlong basic
operations (#6871)
No new revisions were added by this update.
Summary of changes:
.../inlong/manager/common/enums/ErrorCodeEnum.java | 1 +
.../service/group/InlongGroupProcessService.java | 42 +
.../manager/service/group/InlongGroupService.java | 42 -
.../service/group/InlongGroupServiceImpl.java | 133 +++
.../manager/service/sink/StreamSinkService.java| 47 ++
.../service/sink/StreamSinkServiceImpl.java| 174 +++
.../service/stream/InlongStreamService.java| 51 ++
.../service/stream/InlongStreamServiceImpl.java| 186 +
.../openapi/OpenInLongGroupController.java | 94 +++
.../openapi/OpenInLongStreamController.java| 97 +++
.../OpenStreamSinkController.java} | 65 +++
11 files changed, 887 insertions(+), 45 deletions(-)
create mode 100644
inlong-manager/manager-web/src/main/java/org/apache/inlong/manager/web/controller/openapi/OpenInLongGroupController.java
create mode 100644
inlong-manager/manager-web/src/main/java/org/apache/inlong/manager/web/controller/openapi/OpenInLongStreamController.java
copy
inlong-manager/manager-web/src/main/java/org/apache/inlong/manager/web/controller/{StreamSinkController.java
=> openapi/OpenStreamSinkController.java} (69%)
[GitHub] [inlong] dockerzhang merged pull request #6828: [INLONG-6827][Manager] Optimize the config managerment of SortStandalone
dockerzhang merged PR #6828: URL: https://github.com/apache/inlong/pull/6828 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[inlong] branch master updated: [INLONG-6827][Manager] Optimize the config managerment of SortStandalone (#6828)
This is an automated email from the ASF dual-hosted git repository.
dockerzhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new de894ed9d [INLONG-6827][Manager] Optimize the config managerment of
SortStandalone (#6828)
de894ed9d is described below
commit de894ed9d56720052b66721bbe421e1bc0b263ae
Author: vernedeng
AuthorDate: Wed Dec 14 12:39:05 2022 +0800
[INLONG-6827][Manager] Optimize the config managerment of SortStandalone
(#6828)
---
.../service/core/impl/SortClusterServiceImpl.java | 69 ++
1 file changed, 43 insertions(+), 26 deletions(-)
diff --git
a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/core/impl/SortClusterServiceImpl.java
b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/core/impl/SortClusterServiceImpl.java
index e55aa3ea8..f3541316f 100644
---
a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/core/impl/SortClusterServiceImpl.java
+++
b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/core/impl/SortClusterServiceImpl.java
@@ -47,6 +47,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
+import java.util.Optional;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
@@ -107,9 +108,9 @@ public class SortClusterServiceImpl implements
SortClusterService {
public void reload() {
LOGGER.debug("start to reload sort config");
try {
-reloadAllClusterConfigV2();
+reloadAllClusterConfig();
} catch (Throwable t) {
-LOGGER.error(t.getMessage(), t);
+LOGGER.error("fail to reload cluster config", t);
}
LOGGER.debug("end to reload config");
}
@@ -161,7 +162,7 @@ public class SortClusterServiceImpl implements
SortClusterService {
.build();
}
-private void reloadAllClusterConfigV2() {
+private void reloadAllClusterConfig() {
// load all fields info
List fieldInfos = sortConfigLoader.loadAllFields();
fieldMap = new HashMap<>();
@@ -174,13 +175,16 @@ public class SortClusterServiceImpl implements
SortClusterService {
// get all task under a given cluster, has been reduced into cluster
and task.
List tasks = sortConfigLoader.loadAllTask();
Map> clusterTaskMap = tasks.stream()
-.filter(dto -> dto.getSortClusterName() != null)
+.filter(dto -> StringUtils.isNotBlank(dto.getSortClusterName())
+&& StringUtils.isNotBlank(dto.getSortTaskName())
+&& StringUtils.isNotBlank(dto.getDataNodeName())
+&& StringUtils.isNotBlank(dto.getSinkType()))
.collect(Collectors.groupingBy(SortTaskInfo::getSortClusterName));
// get all stream sinks
Map> task2AllStreams =
sinkEntities.stream()
.filter(entity ->
StringUtils.isNotBlank(entity.getInlongClusterName()))
-.collect(Collectors.groupingBy(StreamSinkEntity::getSinkName));
+
.collect(Collectors.groupingBy(StreamSinkEntity::getSortTaskName));
// get all data nodes and group by node name
List dataNodeEntities =
sortConfigLoader.loadAllDataNodeEntity();
@@ -199,7 +203,7 @@ public class SortClusterServiceImpl implements
SortClusterService {
clusterTaskMap.forEach((clusterName, taskList) -> {
try {
-SortClusterConfig config =
this.getConfigByClusterNameV2(clusterName,
+SortClusterConfig config =
this.getConfigByClusterName(clusterName,
taskList, task2AllStreams, task2DataNodeMap);
String jsonStr = GSON.toJson(config);
String md5 = DigestUtils.md5Hex(jsonStr);
@@ -207,8 +211,9 @@ public class SortClusterServiceImpl implements
SortClusterService {
newMd5Map.put(clusterName, md5);
} catch (Throwable e) {
// if get config failed, update the err log.
-newErrorLogMap.put(clusterName, e.getMessage());
-LOGGER.error("Failed to update cluster config={}, error={}",
clusterName, e.getMessage());
+String errMsg =
Optional.ofNullable(e.getMessage()).orElse("Unknown error, please check logs");
+newErrorLogMap.put(clusterName, errMsg);
+LOGGER.error("Failed to update cluster config={}",
clusterName, e);
}
});
@@ -217,7 +222,7 @@ public class SortClusterServiceImpl implements
SortClusterService {
sortClusterMd5Map = newMd5Map;
}
-private SortClusterConfig getConfigByClusterNameV2(
+pr
[GitHub] [inlong] EMsnap commented on a diff in pull request #6872: [INLONG-6864][Sort] Modify hive metric computing to ensure metric data accuracy
EMsnap commented on code in PR #6872:
URL: https://github.com/apache/inlong/pull/6872#discussion_r1048009387
##
inlong-sort/sort-connectors/hive/src/main/java/org/apache/inlong/sort/hive/filesystem/AbstractStreamingWriter.java:
##
@@ -134,7 +138,17 @@ public AbstractStreamingWriter(
* Commit up to this checkpoint id.
*/
protected void commitUpToCheckpoint(long checkpointId) throws Exception {
-helper.commitUpToCheckpoint(checkpointId);
+try {
+helper.commitUpToCheckpoint(checkpointId);
+if (metricData != null) {
+metricData.invoke(rowSize, dataSize);
+}
+rowSize = 0L;
+dataSize = 0L;
+} catch (Exception e) {
+LOG.error("hive sink commitUpToCheckpoint.", e);
Review Comment:
hive sink commitUpToCheckpoint occurs error
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [inlong] healchow commented on pull request #6863: [INLONG-6862][Doc] Use spotless to apply the asf header
healchow commented on PR #6863: URL: https://github.com/apache/inlong/pull/6863#issuecomment-1350472217 How to use this plugin? `mvn spotless:apply`? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [inlong] healchow commented on a diff in pull request #6863: [INLONG-6862][Doc] Use spotless to apply the asf header
healchow commented on code in PR #6863: URL: https://github.com/apache/inlong/pull/6863#discussion_r1048065038 ## pom.xml: ## @@ -1597,7 +1600,7 @@ -check +apply Review Comment: How to use this plugin? `mvn spotless:apply`? Is `mvn spotless:apply` command still in effect? ## pom.xml: ## @@ -1597,7 +1600,7 @@ -check +apply Review Comment: How to use this plugin? `mvn spotless:apply`? Is `mvn spotless:check` command still in effect? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [inlong-website] dockerzhang opened a new issue, #647: [Improve][Doc] Update the community guide
dockerzhang opened a new issue, #647: URL: https://github.com/apache/inlong-website/issues/647 ### Description Update the community guide ### Are you willing to submit PR? - [X] Yes, I am willing to submit a PR! ### Code of Conduct - [X] I agree to follow this project's [Code of Conduct](https://www.apache.org/foundation/policies/conduct) -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [inlong-website] dockerzhang opened a new pull request, #648: [INLONG-647][Doc] Update the community guide
dockerzhang opened a new pull request, #648: URL: https://github.com/apache/inlong-website/pull/648 ### Prepare a Pull Request *(Change the title refer to the following example)* - Title Example: [INLONG-XYZ][Component] Title of the pull request *(The following *XYZ* should be replaced by the actual [GitHub Issue](https://github.com/apache/inlong/issues) number)* - Fixes #647 ### Motivation *Explain here the context, and why you're making that change. What is the problem you're trying to solve.* ### Modifications *Describe the modifications you've done.* ### Verifying this change - [ ] Make sure that the change passes the CI checks. *(Please pick either of the following options)* This change is a trivial rework / code cleanup without any test coverage. *(or)* This change is already covered by existing tests, such as *(please describe tests)*. *(or)* This change added tests and can be verified as follows: *(example:)* - *Added integration tests for end-to-end deployment with large payloads (10MB)* - *Extended integration test for recovery after broker failure* ### Documentation - Does this pull request introduce a new feature? (yes / no) - If yes, how is the feature documented? (not applicable / docs / JavaDocs / not documented) - If a feature is not applicable for documentation, explain why? - If a feature is not documented yet in this PR, please create a followup issue for adding the documentation -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [inlong] dockerzhang commented on a diff in pull request #6863: [INLONG-6862][Doc] Use spotless to apply the asf header
dockerzhang commented on code in PR #6863: URL: https://github.com/apache/inlong/pull/6863#discussion_r1048085986 ## pom.xml: ## @@ -1597,7 +1600,7 @@ -check +apply Review Comment: When compiling, it will be executed `apply` by default, avoid checking first and then applying. `mvn spotless:check` and `mvn spotless:apply` are still available. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [inlong-website] dockerzhang merged pull request #648: [INLONG-647][Doc] Update the community guide
dockerzhang merged PR #648: URL: https://github.com/apache/inlong-website/pull/648 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [inlong-website] dockerzhang closed issue #647: [Improve][Doc] Update the community guide
dockerzhang closed issue #647: [Improve][Doc] Update the community guide URL: https://github.com/apache/inlong-website/issues/647 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[inlong-website] branch master updated: [INLONG-647][Doc] Update the community guide (#648)
This is an automated email from the ASF dual-hosted git repository. dockerzhang pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/inlong-website.git The following commit(s) were added to refs/heads/master by this push: new 088b848f09 [INLONG-647][Doc] Update the community guide (#648) 088b848f09 is described below commit 088b848f099e9d48002b5d1e6a4c32b6e754af7a Author: Charles Zhang AuthorDate: Wed Dec 14 15:20:26 2022 +0800 [INLONG-647][Doc] Update the community guide (#648) --- community/how-to-become-a-committer.md | 12 +- .../how-to-maintain-3rd-party-dependencies.md | 2 +- community/how-to-report-issues.md | 24 +- community/how-to-subscribe.md | 62 +++-- community/how-to-verify.md | 2 +- community/how-to-vote-a-committer-pmc.md | 253 +++-- .../current/how-to-become-a-committer.md | 36 +-- .../current/how-to-contribute.md | 77 --- .../current/how-to-report-issues.md| 20 +- .../current/how-to-subscribe.md| 65 +++--- .../current/how-to-vote-a-committer-pmc.md | 248 ++-- 11 files changed, 409 insertions(+), 392 deletions(-) diff --git a/community/how-to-become-a-committer.md b/community/how-to-become-a-committer.md index bcf82bc66e..7b38e2a7ac 100644 --- a/community/how-to-become-a-committer.md +++ b/community/how-to-become-a-committer.md @@ -3,10 +3,12 @@ title: How to Become a Committer or PMC sidebar_position: 4 --- -> [Apache InLong](https://inlong.apache.org) is to build a community in full accordance with Apache’s rules. -> Apache Committer is a term used in ASF (Apache Software Foundation) to refer to people who submit specific projects. -> Apache InLong Committer has the permission to write the InLong code base and can merge PR. -> Anyone who has made enough contributions to the community and gained enough trust can become an Apache InLong Committer. +:::note +[Apache InLong](https://inlong.apache.org) is to build a community in full accordance with Apache’s rules. +Apache Committer is a term used in ASF (Apache Software Foundation) to refer to people who submit specific projects. +Apache InLong Committer has the permission to write the InLong code base and can merge PR. +Anyone who has made enough contributions to the community and gained enough trust can become an Apache InLong Committer. +::: As long as anyone contributes to the InLong project, you are the officially recognized Contributor of the InLong project. There is no exact standard for growing from Contributor to Committer. There is also no expected timetable, but Committer candidates are generally long-term active contributors. @@ -18,7 +20,7 @@ List of potential contributions (in no particular order): - Update the official documentation so that the project documentation is the most recent, the best practices for writing InLong, and various valuable documents for feature analysis. - Perform test and report test results, performance test and performance comparison test of any modules, etc. - Actively participate in voting when the version is released. -- Participate in the discussion on the mailing list. Usually, there will be mails starting with [DISCUSS]. +- Participate in the discussion on the mailing list. Usually, there will be mails starting with `[DISCUSS]`. - Answer questions from users or developers on the mailing list. - Review the work of others (both code and non-code) and publish your suggestions. - Review issues on GitHub and maintain the latest status of issues, such as outdated closing issues, changing issue error information, etc. diff --git a/community/how-to-maintain-3rd-party-dependencies.md b/community/how-to-maintain-3rd-party-dependencies.md index 1e58118e6a..68c8ab564b 100644 --- a/community/how-to-maintain-3rd-party-dependencies.md +++ b/community/how-to-maintain-3rd-party-dependencies.md @@ -1,5 +1,5 @@ --- -title: How to maintain third-party dependencies in a project +title: How to Maintain Third-party Dependencies sidebar_position: 9 --- diff --git a/community/how-to-report-issues.md b/community/how-to-report-issues.md index abf8c4eccf..3fb64c9e59 100644 --- a/community/how-to-report-issues.md +++ b/community/how-to-report-issues.md @@ -29,18 +29,18 @@ Make sure the Project is set to InLong. Set the issue type field appropriately b - Improvement - Umbrella -For Summary, please provide a detailed title e.g. `[Bug][Dataproxy] Repeated registration jmx metric bean` instead of `Dataproxy registration error`. - -|Component | Description | -|::|:
[GitHub] [inlong] yunqingmoswu merged pull request #6870: [INLONG-6869][Sort] Supports dirty data side-output for elasticsearch sink
yunqingmoswu merged PR #6870: URL: https://github.com/apache/inlong/pull/6870 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[inlong] branch master updated: [INLONG-6869][Sort] Supports dirty data side-output for elasticsearch sink (#6870)
This is an automated email from the ASF dual-hosted git repository.
yunqing pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new 99947c98d [INLONG-6869][Sort] Supports dirty data side-output for
elasticsearch sink (#6870)
99947c98d is described below
commit 99947c98d6437a694ccd3ffd1177f88add12dd31
Author: yunqingmoswu <44659300+yunqingmo...@users.noreply.github.com>
AuthorDate: Wed Dec 14 15:23:38 2022 +0800
[INLONG-6869][Sort] Supports dirty data side-output for elasticsearch sink
(#6870)
---
.../inlong/sort/base/dirty/DirtySinkHelper.java| 108 ++
.../apache/inlong/sort/base/dirty/DirtyType.java | 16 +++
.../sort/elasticsearch6/ElasticsearchSink.java | 20 +++-
.../table/Elasticsearch6DynamicSink.java | 16 ++-
.../table/Elasticsearch6DynamicSinkFactory.java| 14 ++-
.../sort/elasticsearch7/ElasticsearchSink.java | 19 +++-
.../table/Elasticsearch7DynamicSink.java | 16 ++-
.../table/Elasticsearch7DynamicSinkFactory.java| 14 ++-
.../sort/elasticsearch/ElasticsearchSinkBase.java | 122 +++-
.../table/RowElasticsearchSinkFunction.java| 124 ++---
.../sort/parser/ElasticsearchSqlParseTest.java | 10 +-
11 files changed, 369 insertions(+), 110 deletions(-)
diff --git
a/inlong-sort/sort-connectors/base/src/main/java/org/apache/inlong/sort/base/dirty/DirtySinkHelper.java
b/inlong-sort/sort-connectors/base/src/main/java/org/apache/inlong/sort/base/dirty/DirtySinkHelper.java
new file mode 100644
index 0..a962b974e
--- /dev/null
+++
b/inlong-sort/sort-connectors/base/src/main/java/org/apache/inlong/sort/base/dirty/DirtySinkHelper.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.sort.base.dirty;
+
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.util.Preconditions;
+import org.apache.inlong.sort.base.dirty.sink.DirtySink;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.Nullable;
+import java.io.Serializable;
+
+/**
+ * Dirty sink helper, it helps dirty data sink for {@link DirtySink}
+ * @param
+ */
+public class DirtySinkHelper implements Serializable {
+
+private static final long serialVersionUID = 1L;
+private static final Logger LOGGER =
LoggerFactory.getLogger(DirtySinkHelper.class);
+
+private DirtyOptions dirtyOptions;
+private final @Nullable DirtySink dirtySink;
+
+public DirtySinkHelper(DirtyOptions dirtyOptions, @Nullable DirtySink
dirtySink) {
+this.dirtyOptions = Preconditions.checkNotNull(dirtyOptions,
"dirtyOptions is null");
+this.dirtySink = dirtySink;
+}
+
+/**
+ * Open for dirty sink
+ *
+ * @param configuration The configuration that is used for dirty sink
+ */
+public void open(Configuration configuration) {
+if (dirtySink != null) {
+try {
+dirtySink.open(configuration);
+} catch (Exception e) {
+throw new RuntimeException(e);
+}
+}
+}
+
+/**
+ * Dirty data sink
+ * @param dirtyData The dirty data
+ * @param dirtyType The dirty type {@link DirtyType}
+ * @param e The cause of dirty data
+ */
+public void invoke(T dirtyData, DirtyType dirtyType, Throwable e) {
+if (!dirtyOptions.ignoreDirty()) {
+RuntimeException ex;
+if (e instanceof RuntimeException) {
+ex = (RuntimeException) e;
+} else {
+ex = new RuntimeException(e);
+}
+throw ex;
+}
+if (dirtySink != null) {
+DirtyData.Builder builder = DirtyData.builder();
+try {
+builder.setData(dirtyData)
+.setDirtyType(dirtyType)
+.setLabels(dirtyOptions.getLabels())
+.setLogTag(dirtyOptions.getLogTag())
+.setDirtyMessage(e.getMessage())
+.setIdentifier(dirtyOptions.getIdentifier
[GitHub] [inlong] EMsnap opened a new pull request, #6877: [INLONG-6876][Manager] enable scan.incremental.snapshot for mysql extract node
EMsnap opened a new pull request, #6877: URL: https://github.com/apache/inlong/pull/6877 ### Prepare a Pull Request - Fixes #6876 ### Motivation see https://github.com/apache/inlong/pull/6870 ### Modifications see https://github.com/apache/inlong/pull/6870 ### Verifying this change ### Documentation - Does this pull request introduce a new feature? (yes / no) - If yes, how is the feature documented? (not applicable / docs / JavaDocs / not documented) - If a feature is not applicable for documentation, explain why? - If a feature is not documented yet in this PR, please create a follow-up issue for adding the documentation -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
