[GitHub] [inlong] yunqingmoswu commented on a diff in pull request #6750: [INLONG-6657][Sort] Add dirty metric for Kafka
yunqingmoswu commented on code in PR #6750:
URL: https://github.com/apache/inlong/pull/6750#discussion_r1053020028
##
inlong-sort/sort-connectors/kafka/src/main/java/org/apache/inlong/sort/kafka/DynamicKafkaSerializationSchema.java:
##
@@ -164,6 +170,9 @@ public ProducerRecord serialize(RowData
consumedRow, @Nullable L
final byte[] valueSerialized =
serializeWithDirtyHandle(consumedRow,
DirtyType.VALUE_SERIALIZE_ERROR, valueSerialization);
if (valueSerialized != null) {
+if (metricData != null) {
Review Comment:
Why report dirty data metric here?
##
inlong-sort/sort-connectors/kafka/src/main/java/org/apache/inlong/sort/kafka/FlinkKafkaProducer.java:
##
@@ -875,7 +902,12 @@ public void onCompletion(RecordMetadata metadata,
Exception e) {
public void onCompletion(RecordMetadata metadata,
Exception exception) {
if (exception != null && asyncException == null) {
asyncException = exception;
-sendDirtyMetrics(rowSize, dataSize);
+} else if (metadata != null) {
Review Comment:
What should do if the metadata is null?
##
inlong-sort/sort-connectors/kafka/src/main/java/org/apache/inlong/sort/kafka/KafkaDynamicSink.java:
##
@@ -214,6 +217,7 @@ public KafkaDynamicSink(
this.topicPattern = topicPattern;
this.dirtyOptions = dirtyOptions;
this.dirtySink = dirtySink;
+this.migrateAll = migrateAll;
Review Comment:
migrateall -> multipleSink
##
inlong-sort/sort-connectors/kafka/src/main/java/org/apache/inlong/sort/kafka/DynamicKafkaSerializationSchema.java:
##
@@ -199,6 +211,9 @@ public ProducerRecord serialize(RowData
consumedRow, @Nullable L
} else {
valueSerialized = serializeWithDirtyHandle(valueRow,
DirtyType.VALUE_SERIALIZE_ERROR, valueSerialization);
mayDirtyData = mayDirtyData || valueSerialized == null;
+if (metricData != null) {
Review Comment:
Why report dirty data metric here?
##
inlong-sort/sort-connectors/kafka/src/main/java/org/apache/inlong/sort/kafka/table/KafkaDynamicTableFactory.java:
##
@@ -525,7 +527,8 @@ protected KafkaDynamicSink createKafkaTableSink(
@Nullable String sinkMultipleFormat,
@Nullable String topicPattern,
DirtyOptions dirtyOptions,
-@Nullable DirtySink dirtySink) {
+@Nullable DirtySink dirtySink,
+boolean migrateAll) {
Review Comment:
migrateall -> multipleSink
##
inlong-sort/sort-connectors/kafka/src/main/java/org/apache/inlong/sort/kafka/FlinkKafkaProducer.java:
##
@@ -859,10 +883,13 @@ public void open(Configuration configuration) throws
Exception {
@Override
public void onCompletion(RecordMetadata metadata,
Exception e) {
if (e != null) {
-sendDirtyMetrics(rowSize, dataSize);
-LOG.error(
-"Error while sending record to Kafka:
" + e.getMessage(),
-e);
+LOG.error("Error while sending record to
Kafka: " + e.getMessage(), e);
+} else if (metadata != null) {
Review Comment:
What should do if the metadata is null?
##
inlong-sort/sort-connectors/kafka/src/main/java/org/apache/inlong/sort/kafka/DynamicKafkaSerializationSchema.java:
##
@@ -179,6 +188,9 @@ public ProducerRecord serialize(RowData
consumedRow, @Nullable L
} else {
final RowData keyRow = createProjectedRow(consumedRow,
RowKind.INSERT, keyFieldGetters);
keySerialized = serializeWithDirtyHandle(keyRow,
DirtyType.KEY_SERIALIZE_ERROR, keySerialization);
+if (metricData != null) {
Review Comment:
Why report dirty data metric here?
##
inlong-sort/sort-connectors/kafka/src/main/java/org/apache/inlong/sort/kafka/KafkaDynamicSink.java:
##
@@ -181,7 +183,8 @@ public KafkaDynamicSink(
@Nullable String sinkMultipleFormat,
@Nullable String topicPattern,
DirtyOptions dirtyOptions,
-@Nullable DirtySink dirtySink) {
+@Nullable DirtySink dirtySink,
+boolean migrateAll) {
Review Comment:
migrateall -> multipleSink
##
inlong-sort/sort-connectors/kafka/src/main/java/org/apache/inlong/sort/kafka/table/KafkaDynamicTableFactory.java:
##
@@ -440,7 +441,8 @@ public DynamicTableSink createDynamicTableSink(Context
context) {
sinkMultipleFormat,
tableOptions.getOptional(TOPIC_PATTERN).orElse(null),
dirtyOptions,
-dirtySin
[GitHub] [inlong] bluewang opened a new pull request, #6986: [INLONG-6982][Dashboard] Data source and data storage bullet box title optimization
bluewang opened a new pull request, #6986: URL: https://github.com/apache/inlong/pull/6986 ### Prepare a Pull Request *(Change the title refer to the following example)* - Title Example: [INLONG-XYZ][Component] Title of the pull request *(The following *XYZ* should be replaced by the actual [GitHub Issue](https://github.com/apache/inlong/issues) number)* - Fixes https://github.com/apache/inlong/issues/6982 ### Modifications     -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [inlong] kuansix commented on a diff in pull request #6979: [INLONG-6978][Sort] Get primary keys for PostgreSQL
kuansix commented on code in PR #6979:
URL: https://github.com/apache/inlong/pull/6979#discussion_r1053042933
##
inlong-sort/sort-connectors/jdbc/src/main/java/org/apache/inlong/sort/jdbc/dialect/PostgresDialect.java:
##
@@ -137,4 +146,71 @@ public List unsupportedTypes() {
LogicalTypeRoot.SYMBOL,
LogicalTypeRoot.UNRESOLVED);
}
+
+@Override
+public List getAndSetPkNamesFromDb(String tableIdentifier,
JdbcOptions jdbcOptions) {
+PreparedStatement st = null;
+try {
+JdbcOptions jdbcExecOptions =
+JdbcOptions.builder()
+.setDBUrl(jdbcOptions.getDbURL() + "/"
++
JdbcMultiBatchingComm.getTDbNameFromIdentifier(tableIdentifier))
+
.setTableName(JdbcMultiBatchingComm.getTbNameFromIdentifier(tableIdentifier))
+.setDialect(jdbcOptions.getDialect())
+.setParallelism(jdbcOptions.getParallelism())
+
.setConnectionCheckTimeoutSeconds(jdbcOptions.getConnectionCheckTimeoutSeconds())
+.setDriverName(jdbcOptions.getDriverName())
+.setUsername(jdbcOptions.getUsername().orElse(""))
+.setPassword(jdbcOptions.getPassword().orElse(""))
+.build();
+SimpleJdbcConnectionProvider tableConnectionProvider = new
SimpleJdbcConnectionProvider(jdbcExecOptions);
+Connection conn =
tableConnectionProvider.getOrEstablishConnection();
Review Comment:
close when task exit, implement in JdbcMultiBatchingOutputFormat.close
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [inlong] fuweng11 opened a new pull request, #6989: [INLONG-6988][Manager] Use the data node info for StarRocks
fuweng11 opened a new pull request, #6989: URL: https://github.com/apache/inlong/pull/6989 ### Prepare a Pull Request - Fixes #6988 ### Motivation Use the data node info for StarRocks ### Modifications - read from data node when starRocks url is not supplied -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [inlong] yunqingmoswu commented on a diff in pull request #6979: [INLONG-6978][Sort] Get primary keys for PostgreSQL
yunqingmoswu commented on code in PR #6979:
URL: https://github.com/apache/inlong/pull/6979#discussion_r1053061727
##
inlong-sort/sort-connectors/jdbc/src/main/java/org/apache/inlong/sort/jdbc/table/AbstractJdbcDialect.java:
##
@@ -98,4 +99,12 @@ public void validate(TableSchema schema) throws
ValidationException {
* @return a list of logical type roots.
*/
public abstract List unsupportedTypes();
+
+/**
+ * get getPkNames from query db.tb
+ *
+ * @return a list of PkNames.
+ */
+public abstract List getPkNamesFromDb(String tableIdentifier,
Review Comment:
maybe it is better to remove abstract, subclass can override it if needed.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [inlong] leezng merged pull request #6981: [INLONG-6980][Dashboard] inlong group detail optimization
leezng merged PR #6981: URL: https://github.com/apache/inlong/pull/6981 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[inlong] branch master updated: [INLONG-6980][Dashboard] inlong group detail optimization (#6981)
This is an automated email from the ASF dual-hosted git repository.
leezng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new 24b68d076 [INLONG-6980][Dashboard] inlong group detail optimization
(#6981)
24b68d076 is described below
commit 24b68d076b17467db7748ed630c35b9085cbe793
Author: Lizhen <88174078+bluew...@users.noreply.github.com>
AuthorDate: Tue Dec 20 17:28:19 2022 +0800
[INLONG-6980][Dashboard] inlong group detail optimization (#6981)
---
inlong-dashboard/src/locales/cn.json | 3 +--
inlong-dashboard/src/locales/en.json | 3 +--
inlong-dashboard/src/pages/GroupDetail/index.tsx | 4 +---
3 files changed, 3 insertions(+), 7 deletions(-)
diff --git a/inlong-dashboard/src/locales/cn.json
b/inlong-dashboard/src/locales/cn.json
index 50a542205..aca18ba74 100644
--- a/inlong-dashboard/src/locales/cn.json
+++ b/inlong-dashboard/src/locales/cn.json
@@ -361,7 +361,7 @@
"meta.Stream.FieldName": "字段名",
"meta.Stream.FieldNameRule": "以英文字母开头，只能包含英文字母、数字、下划线",
"meta.Stream.DataType": "数据格式",
- "meta.Stream.DataTypeHelp": "CSV：消息体为原生 CSV 的 InLong 消息类型；Raw-CSV：原生
CSV，消息按特定分隔符分隔",
+ "meta.Stream.DataTypeHelp": "CSV：消息体为原生 CSV 的 InLong 消息类型",
"meta.Stream.FieldType": "字段类型",
"meta.Stream.FieldComment": "字段描述",
"meta.Stream.DataEncoding": "数据编码",
@@ -491,7 +491,6 @@
"pages.GroupDetail.Sink.SaveAndRefresh": "保存并刷新配置",
"pages.GroupDetail.Stream.StreamConfigTitle": "数据流配置",
"pages.GroupDetail.Stream.CreateDataStream": "新建数据流",
- "pages.GroupDetail.PageTitle": "详情",
"pages.GroupDetail.Info": "数据流组",
"pages.GroupDetail.Streams": "数据流",
"pages.GroupDetail.Sources": "数据源",
diff --git a/inlong-dashboard/src/locales/en.json
b/inlong-dashboard/src/locales/en.json
index f31dcfd86..0bd7ec404 100644
--- a/inlong-dashboard/src/locales/en.json
+++ b/inlong-dashboard/src/locales/en.json
@@ -361,7 +361,7 @@
"meta.Stream.FieldName": "Field name",
"meta.Stream.FieldNameRule": "At the beginning of English letters, only
English letters, numbers, and underscores",
"meta.Stream.DataType": "DataType",
- "meta.Stream.DataTypeHelp": "CSV: InLong message type whose message body is
raw CSV; Raw-CSV: messages are separated by a specific separator",
+ "meta.Stream.DataTypeHelp": "CSV: InLong message type whose message body is
raw CSV",
"meta.Stream.FieldType": "FieldType",
"meta.Stream.FieldComment": "Field comment",
"meta.Stream.DataEncoding": "Data encoding",
@@ -488,7 +488,6 @@
"pages.GroupDetail.Sink.SaveAndRefresh": "Save and submit the process",
"pages.GroupDetail.Stream.StreamConfigTitle": "Data stream configuration",
"pages.GroupDetail.Stream.CreateDataStream": "Create",
- "pages.GroupDetail.PageTitle": "Detail",
"pages.GroupDetail.Info": "Group Info",
"pages.GroupDetail.Streams": "Streams",
"pages.GroupDetail.Sources": "Sources",
diff --git a/inlong-dashboard/src/pages/GroupDetail/index.tsx
b/inlong-dashboard/src/pages/GroupDetail/index.tsx
index a855963c8..326c8d2cc 100644
--- a/inlong-dashboard/src/pages/GroupDetail/index.tsx
+++ b/inlong-dashboard/src/pages/GroupDetail/index.tsx
@@ -162,9 +162,7 @@ const Comp: React.FC = () => {
[GitHub] [inlong] gong commented on a diff in pull request #6979: [INLONG-6978][Sort] Get primary keys for PostgreSQL
gong commented on code in PR #6979:
URL: https://github.com/apache/inlong/pull/6979#discussion_r1053094557
##
inlong-sort/sort-connectors/jdbc/src/main/java/org/apache/inlong/sort/jdbc/internal/JdbcMultiBatchingComm.java:
##
@@ -173,4 +173,26 @@ private static RowData getPrimaryKey(RowData row,
RowData.FieldGetter[] fieldGet
return pkRow;
}
+/**
+ * Get table name From tableIdentifier
+ * tableIdentifier maybe: ${dbName}.${tbName} or
${dbName}.${schemaName}.${tbName}
+ *
+ * @param tableIdentifier The table identifier for which to get table name.
+ */
+public static String getTbNameFromIdentifier(String tableIdentifier) {
+String[] fileArray = tableIdentifier.split("\\.");
+if (2 == fileArray.length) {
+return fileArray[1];
+}
+if (3 == fileArray.length) {
+return fileArray[1] + "." + fileArray[2];
+}
+return null;
+}
+
+public static String getTDbNameFromIdentifier(String tableIdentifier) {
Review Comment:
method name is similar, I suggest use full name.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [inlong] gosonzhang opened a new pull request, #6990: [INLONG-6985][DataProxy] Make maxMonitorCnt setting configurable
gosonzhang opened a new pull request, #6990: URL: https://github.com/apache/inlong/pull/6990 - Fixes #6985 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [inlong] bluewang opened a new pull request, #6991: [INLONG-6987][Dashboard] Writing standard when splicing Chinese and English words
bluewang opened a new pull request, #6991: URL: https://github.com/apache/inlong/pull/6991 ### Prepare a Pull Request *(Change the title refer to the following example)* - Title Example: [INLONG-XYZ][Component] Title of the pull request *(The following *XYZ* should be replaced by the actual [GitHub Issue](https://github.com/apache/inlong/issues) number)* - Fixes https://github.com/apache/inlong/issues/6987 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [inlong] fuweng11 opened a new pull request, #6993: [INLONG-6992][Manager] Fix the problem of streamSource is always in the status of TO_BE_ISSUED_DELETE
fuweng11 opened a new pull request, #6993: URL: https://github.com/apache/inlong/pull/6993 ### Prepare a Pull Request - Fixes #6992 ### Motivation Fix the problem of streamSource is always in the status of TO_BE_ISSUED_DELETE ### Modifications StreamSource without clusterName are also selected to issue. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [inlong] dockerzhang merged pull request #6993: [INLONG-6992][Manager] Fix the problem of streamSource is always in the status of TO_BE_ISSUED_DELETE
dockerzhang merged PR #6993: URL: https://github.com/apache/inlong/pull/6993 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[inlong] branch master updated: [INLONG-6992][Manager] Fix the problem of agent is always in the status of TO_BE_ISSUED_DELETE (#6993)
This is an automated email from the ASF dual-hosted git repository.
dockerzhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new 532e3a9a8 [INLONG-6992][Manager] Fix the problem of agent is always in
the status of TO_BE_ISSUED_DELETE (#6993)
532e3a9a8 is described below
commit 532e3a9a8473cbdb1c879de3336df58a1ec9abdc
Author: fuweng11 <76141879+fuwen...@users.noreply.github.com>
AuthorDate: Tue Dec 20 20:05:45 2022 +0800
[INLONG-6992][Manager] Fix the problem of agent is always in the status of
TO_BE_ISSUED_DELETE (#6993)
---
.../manager-dao/src/main/resources/mappers/StreamSourceEntityMapper.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git
a/inlong-manager/manager-dao/src/main/resources/mappers/StreamSourceEntityMapper.xml
b/inlong-manager/manager-dao/src/main/resources/mappers/StreamSourceEntityMapper.xml
index e19a05edc..804d47c03 100644
---
a/inlong-manager/manager-dao/src/main/resources/mappers/StreamSourceEntityMapper.xml
+++
b/inlong-manager/manager-dao/src/main/resources/mappers/StreamSourceEntityMapper.xml
@@ -229,7 +229,7 @@
from stream_source
-inlong_cluster_name = #{clusterName, jdbcType=VARCHAR}
+(inlong_cluster_name = #{clusterName, jdbcType=VARCHAR} or
inlong_cluster_name is NULL)
and agent_ip = #{agentIp, jdbcType=VARCHAR}
and `status` in
[GitHub] [inlong] dockerzhang merged pull request #6984: [INLONG-6983][Dashboard] Group page supports multiple columns
dockerzhang merged PR #6984: URL: https://github.com/apache/inlong/pull/6984 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[inlong] branch master updated: [INLONG-6983][Dashboard] Group page supports multiple columns (#6984)
This is an automated email from the ASF dual-hosted git repository.
dockerzhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new 483f2503b [INLONG-6983][Dashboard] Group page supports multiple
columns (#6984)
483f2503b is described below
commit 483f2503bbb76fdccad6281013e26e4186928ce0
Author: Daniel
AuthorDate: Tue Dec 20 20:13:14 2022 +0800
[INLONG-6983][Dashboard] Group page supports multiple columns (#6984)
---
inlong-dashboard/src/components/FormGenerator/FormGenerator.tsx | 3 +++
inlong-dashboard/src/pages/GroupDetail/Info/index.tsx | 3 ++-
2 files changed, 5 insertions(+), 1 deletion(-)
diff --git a/inlong-dashboard/src/components/FormGenerator/FormGenerator.tsx
b/inlong-dashboard/src/components/FormGenerator/FormGenerator.tsx
index 9821670d8..86bd73fd8 100644
--- a/inlong-dashboard/src/components/FormGenerator/FormGenerator.tsx
+++ b/inlong-dashboard/src/components/FormGenerator/FormGenerator.tsx
@@ -44,6 +44,8 @@ export interface FormGeneratorProps extends FormProps {
onFilter?: Function;
// default: true
viewOnly?: boolean;
+ // Define col uniformly for all FormItems
+ col?: number;
}
export interface ContentsItemProps {
@@ -123,6 +125,7 @@ const FormGenerator: React.FC = props
=> {
return {
...v,
+ col: v.col || props.col,
name,
type: viewOnly ? 'text' : v.type,
suffix:
diff --git a/inlong-dashboard/src/pages/GroupDetail/Info/index.tsx
b/inlong-dashboard/src/pages/GroupDetail/Info/index.tsx
index 3947acb2a..f80b12e41 100644
--- a/inlong-dashboard/src/pages/GroupDetail/Info/index.tsx
+++ b/inlong-dashboard/src/pages/GroupDetail/Info/index.tsx
@@ -122,7 +122,8 @@ const Comp = ({ inlongGroupId, readonly, isCreate }: Props,
ref) => {
content={formContent}
initialValues={data}
onValuesChange={(c, values) => setMqType(values.mqType)}
-useMaxWidth={600}
+useMaxWidth={1400}
+col={12}
/>
{!isCreate && !readonly && (
[GitHub] [inlong] kinfuy opened a new pull request, #6995: [INLONG-6926][Dashboard] Adding MQ type filter box for data flow group, data consumption list
kinfuy opened a new pull request, #6995: URL: https://github.com/apache/inlong/pull/6995 ### Prepare a Pull Request *(Change the title refer to the following example)* - Title Example: [INLONG-XYZ][Component] Title of the pull request *(The following *XYZ* should be replaced by the actual [GitHub Issue](https://github.com/apache/inlong/issues) number)* - Fixes #6994 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [inlong] dockerzhang merged pull request #6991: [INLONG-6987][Dashboard] Writing standard when splicing Chinese and English words
dockerzhang merged PR #6991: URL: https://github.com/apache/inlong/pull/6991 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[inlong] branch master updated: [INLONG-6987][Dashboard] Writing standard when splicing Chinese and English words (#6991)
This is an automated email from the ASF dual-hosted git repository. dockerzhang pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/inlong.git The following commit(s) were added to refs/heads/master by this push: new d32318599 [INLONG-6987][Dashboard] Writing standard when splicing Chinese and English words (#6991) d32318599 is described below commit d32318599f974c6b6217299f2462825061420fb4 Author: Lizhen <88174078+bluew...@users.noreply.github.com> AuthorDate: Tue Dec 20 22:31:47 2022 +0800 [INLONG-6987][Dashboard] Writing standard when splicing Chinese and English words (#6991) --- inlong-dashboard/src/locales/cn.json | 54 ++-- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/inlong-dashboard/src/locales/cn.json b/inlong-dashboard/src/locales/cn.json index aca18ba74..81ab02f8f 100644 --- a/inlong-dashboard/src/locales/cn.json +++ b/inlong-dashboard/src/locales/cn.json @@ -17,7 +17,7 @@ "meta.Sources.Name": "数据源名称", "meta.Sources.Type": "类型", "meta.Sources.File.SerializationType": "文件类型", - "meta.Sources.File.DataSourceIP": "数据源IP", + "meta.Sources.File.DataSourceIP": "数据源 IP", "meta.Sources.File.FilePath": "⽂件路径", "meta.Sources.File.FilePathHelp": "必须是绝对路径，支持正则表达式，多个时以逗号分隔，如：/data/.*.log", "meta.Sources.File.IpRule": "请输入正确的IP地址", @@ -100,7 +100,7 @@ "meta.Sources.Hudi.ReadStartCommit": "起始的Commit", "meta.Sources.Hudi.ReadStreamingSkipCompactionHelp": "流读时是否跳过 compaction 的 commits，跳过 compaction 有两个用途：1）避免 upsert 语义 下重复消费(compaction 的 instant 为重复数据，如果不跳过，有小概率会重复消费）; 2) changelog 模式下保证语义正确性。 0.11 开始，以上两个问题已经通过保留 compaction 的 instant time 修复", "meta.Sources.Hudi.ReadStartCommitHelp": "指定 'MMddHHmmss'格式的起始commit（闭区间）", - "meta.Sources.Hudi.ExtListHelper": "hudi表的DDL属性需带前缀'ddl.'", + "meta.Sources.Hudi.ExtListHelper": "hudi 表的 DDL 属性需带前缀'ddl.'", "meta.Sources.Hudi.ExtList": "高级属性", "meta.Sinks.SinkName": "名称", "meta.Sinks.SinkNameRule": "以英文字母开头，只能包含英文字母、数字、中划线、下划线", @@ -124,9 +124,9 @@ "meta.Sinks.Hive.DataSeparator.VerticalLine": "竖线(|)", "meta.Sinks.Hive.DataSeparator.DoubleQuotes": "双引号(\")", "meta.Sinks.Hive.PartitionFieldList": "分区字段", - "meta.Sinks.Hive.PartitionFieldListHelp": "字段类型若为timestamp，则必须设置此字段值的格式，支持 MICROSECONDS，MILLISECONDS，SECONDS，SQL，ISO_8601，以及自定义，比如：-MM-dd HH:mm:ss 等", + "meta.Sinks.Hive.PartitionFieldListHelp": "字段类型若为 timestamp，则必须设置此字段值的格式，支持 MICROSECONDS，MILLISECONDS，SECONDS，SQL，ISO_8601，以及自定义，比如：-MM-dd HH:mm:ss 等", "meta.Sinks.Hive.DbName": "DB名称", - "meta.Sinks.Hive.TableName": "表名称", + "meta.Sinks.Hive.TableName": "表名称", "meta.Sinks.Hive.FieldNameRule": "以小写英文字母开头，只能包含小写英文字母、数字、下划线", "meta.Sinks.Hive.ConnectionTest": "测试连接", "meta.Sinks.Hive.ConnectionSucceeded": "连接成功", @@ -137,7 +137,7 @@ "meta.Sinks.Hive.IsMetaField": "是否为元字段", "meta.Sinks.Hive.FieldFormat": "字段格式", "meta.Sinks.Hive.DataNodeName": "数据节点", - "meta.Sinks.ClickHouse.DbName": "DB名称", + "meta.Sinks.ClickHouse.DbName": "DB 名称", "meta.Sinks.ClickHouse.TableName": "表名称", "meta.Sinks.ClickHouse.FlushInterval": "刷新的间隔", "meta.Sinks.ClickHouse.FlushIntervalUnit": "秒", @@ -178,9 +178,9 @@ "meta.Sinks.ES.DateFormat": "日期格式", "meta.Sinks.Kafka.Server": "服务器地址", "meta.Sinks.Kafka.SerializationType": "序列化类型", - "meta.Sinks.Kafka.PartitionNum": "Topic分区数", + "meta.Sinks.Kafka.PartitionNum": "Topic 分区数", "meta.Sinks.Kafka.AutoOffsetReset": "自动偏移量重置", - "meta.Sinks.Iceberg.DbName": "DB名称", + "meta.Sinks.Iceberg.DbName": "DB 名称", "meta.Sinks.Iceberg.TableName": "表名称", "meta.Sinks.Iceberg.FileFormat": "⽂件格式", "meta.Sinks.Iceberg.Description": "表描述", @@ -192,7 +192,7 @@ "meta.Sinks.Iceberg.FieldDescription": "字段描述", "meta.Sinks.Iceberg.PartitionStrategy": "分区策略", "meta.Sinks.Iceberg.DataNodeName": "数据节点", - "meta.Sinks.Hudi.DbName": "DB名称", + "meta.Sinks.Hudi.DbName": "DB 名称", "meta.Sinks.Hudi.TableName": "表名称", "meta.Sinks.Hudi.Warehouse": "仓库路径", "meta.Sinks.Hudi.FileFormat": "⽂件格式", @@ -234,7 +234,7 @@ "meta.Sinks.Oracle.IsMetaField": "是否为元字段", "meta.Sinks.Oracle.FieldFormat": "字段格式", "meta.Sinks.Oracle.FieldDescription": "字段描述", - "meta.Sinks.PostgreSQL.DbName": "DB名称", + "meta.Sinks.PostgreSQL.DbName": "DB 名称", "meta.Sinks.PostgreSQL.TableName": "表名称", "meta.Sinks.PostgreSQL.PrimaryKey": "主键", "meta.Sinks.PostgreSQL.FieldName": "字段名", @@ -305,30 +305,30 @@ "meta.Sinks.StarRocks.IsMetaField": "是否为元字段", "meta.Sinks.StarRocks.FieldFormat": "字段格式", "meta.Sinks.StarRocks.FieldDescription": "字段描述", - "meta.Group.InlongGroupId": "数据流组ID", + "meta.Group.InlongGroupId": "数据流组 ID", "meta.Group.InlongGroupIdRules": "只能包含小写字母、数字、中划线、下划线", "meta.Group.InlongGroupName": "数据流组名称", "meta.Group.InlongGroupOwners": "责任人", "meta.Group.InlongGroupOwnersExtra": "责任人，可查看、修改数据流组信息", "meta.Group.InlongGroupIntroduction": "描述", "meta.Group.DataReportType": "数据上报"
[GitHub] [inlong] dockerzhang commented on pull request #6990: [INLONG-6985][DataProxy] Make maxMonitorCnt setting configurable
dockerzhang commented on PR #6990: URL: https://github.com/apache/inlong/pull/6990#issuecomment-1359456055 @woofyzhao PTAL, thanks. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [inlong] ZuoFengZhang opened a new pull request, #6999: [INLONG-6996][Dashboard][Manager] Support JSON format
ZuoFengZhang opened a new pull request, #6999: URL: https://github.com/apache/inlong/pull/6999 ### Prepare a Pull Request *[INLONG-6996][Dashboard][Manager] Support JSON format* - Fixes #6996 ### Motivation *Explain here the context, and why you're making that change. What is the problem you're trying to solve?* ### Modifications *Describe the modifications you've done.* ### Verifying this change *(Please pick either of the following options)* - [x] This change is a trivial rework/code cleanup without any test coverage. - [ ] This change is already covered by existing tests, such as: *(please describe tests)* - [ ] This change added tests and can be verified as follows: *(example:)* - *Added integration tests for end-to-end deployment with large payloads (10MB)* - *Extended integration test for recovery after broker failure* ### Documentation - Does this pull request introduce a new feature? (yes / no) - If yes, how is the feature documented? (not applicable / docs / JavaDocs / not documented) - If a feature is not applicable for documentation, explain why? - If a feature is not documented yet in this PR, please create a follow-up issue for adding the documentation -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [inlong] ZuoFengZhang opened a new pull request, #7001: [INLONG-6998][Dashboard][Manager] Support ignore deserialization erro…
ZuoFengZhang opened a new pull request, #7001: URL: https://github.com/apache/inlong/pull/7001 ### Prepare a Pull Request *[INLONG-6998][Dashboard][Manager] Support ignore deserialization erro…* - Fixes #6998 ### Motivation *Explain here the context, and why you're making that change. What is the problem you're trying to solve?* ### Modifications *Describe the modifications you've done.* ### Verifying this change *(Please pick either of the following options)* - [ ] This change is a trivial rework/code cleanup without any test coverage. - [ ] This change is already covered by existing tests, such as: *(please describe tests)* - [ ] This change added tests and can be verified as follows: *(example:)* - *Added integration tests for end-to-end deployment with large payloads (10MB)* - *Extended integration test for recovery after broker failure* ### Documentation - Does this pull request introduce a new feature? (yes / no) - If yes, how is the feature documented? (not applicable / docs / JavaDocs / not documented) - If a feature is not applicable for documentation, explain why? - If a feature is not documented yet in this PR, please create a follow-up issue for adding the documentation -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [inlong] ZuoFengZhang opened a new pull request, #7002: [INLONG-7000][Dashboard][Manager] Add AVRO format for InlongStream
ZuoFengZhang opened a new pull request, #7002: URL: https://github.com/apache/inlong/pull/7002 ### Prepare a Pull Request *[INLONG-7000][Dashboard][Manager] Add AVRO format for InlongStream* - Fixes #7000 ### Motivation *Explain here the context, and why you're making that change. What is the problem you're trying to solve?* ### Modifications *Describe the modifications you've done.* ### Verifying this change *(Please pick either of the following options)* - [x] This change is a trivial rework/code cleanup without any test coverage. - [x] This change is already covered by existing tests, such as: *(please describe tests)* - [ ] This change added tests and can be verified as follows: *(example:)* - *Added integration tests for end-to-end deployment with large payloads (10MB)* - *Extended integration test for recovery after broker failure* ### Documentation - Does this pull request introduce a new feature? (yes / no) - If yes, how is the feature documented? (not applicable / docs / JavaDocs / not documented) - If a feature is not applicable for documentation, explain why? - If a feature is not documented yet in this PR, please create a follow-up issue for adding the documentation -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [inlong] healchow commented on pull request #6927: [INLONG-6926][Dashboard] Change the cluster type to lower case
healchow commented on PR #6927: URL: https://github.com/apache/inlong/pull/6927#issuecomment-1360649198 @kinfuy Hi, I see you force-pushed a new commit. It is not recommended to use the `force` action, because it will overwrite the previous commit and review info. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [inlong] healchow commented on pull request #6973: [INLONG-6972][Dashboard] All "consumption" is changed to "subscription" for data consumption
healchow commented on PR #6973: URL: https://github.com/apache/inlong/pull/6973#issuecomment-1360650321 @kinfuy Hi, I see you force-pushed a new commit. It is not recommended to use the force action, because it will overwrite the previous commit and review info. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [inlong] healchow merged pull request #6973: [INLONG-6972][Dashboard] All "consumption" is changed to "subscription" for data consumption
healchow merged PR #6973: URL: https://github.com/apache/inlong/pull/6973 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[inlong] branch master updated: [INLONG-6972][Dashboard] Change all "consumption" to "subscription" for data consumption (#6973)
This is an automated email from the ASF dual-hosted git repository.
healchow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new 486ebdb9b [INLONG-6972][Dashboard] Change all "consumption" to
"subscription" for data consumption (#6973)
486ebdb9b is described below
commit 486ebdb9bffaeee351e6ca8038534d3a36804064
Author: kinfuy <37766068+kin...@users.noreply.github.com>
AuthorDate: Wed Dec 21 09:38:05 2022 +0800
[INLONG-6972][Dashboard] Change all "consumption" to "subscription" for
data consumption (#6973)
---
inlong-dashboard/src/configs/menus/index.ts| 2 +-
inlong-dashboard/src/i18n.ts | 8
inlong-dashboard/src/locales/cn.json | 14 +++---
inlong-dashboard/src/locales/en.json | 12 ++--
inlong-dashboard/src/pages/ConsumeDetail/index.tsx | 6 +++---
5 files changed, 21 insertions(+), 21 deletions(-)
diff --git a/inlong-dashboard/src/configs/menus/index.ts
b/inlong-dashboard/src/configs/menus/index.ts
index 26199c40c..ed6c28c86 100644
--- a/inlong-dashboard/src/configs/menus/index.ts
+++ b/inlong-dashboard/src/configs/menus/index.ts
@@ -33,7 +33,7 @@ const menus: MenuItemType[] = [
},
{
path: '/consume',
-name: i18n.t('configs.menus.Consumes'),
+name: i18n.t('configs.menus.Subscribe'),
},
{
name: i18n.t('configs.menus.Clusters'),
diff --git a/inlong-dashboard/src/i18n.ts b/inlong-dashboard/src/i18n.ts
index 4a3e035f7..f71bfb580 100644
--- a/inlong-dashboard/src/i18n.ts
+++ b/inlong-dashboard/src/i18n.ts
@@ -26,8 +26,8 @@ const resources = {
en: {
translation: {
'configs.menus.Process': 'Approval',
- 'configs.menus.Groups': 'Inlong Groups',
- 'configs.menus.Consumes': 'Inlong Consumes',
+ 'configs.menus.Groups': 'Integration',
+ 'configs.menus.Subscribe': 'Subscription',
'configs.menus.Clusters': 'Clusters',
'configs.menus.ClusterTags': 'ClusterTags',
'configs.menus.SystemManagement': 'System',
@@ -39,8 +39,8 @@ const resources = {
cn: {
translation: {
'configs.menus.Process': '审批管理',
- 'configs.menus.Groups': '数据流组',
- 'configs.menus.Consumes': '数据消费',
+ 'configs.menus.Groups': '数据集成',
+ 'configs.menus.Subscribe': '数据订阅',
'configs.menus.Clusters': '集群管理',
'configs.menus.ClusterTags': '标签管理',
'configs.menus.SystemManagement': '系统管理',
diff --git a/inlong-dashboard/src/locales/cn.json
b/inlong-dashboard/src/locales/cn.json
index 81ab02f8f..ebe11ada6 100644
--- a/inlong-dashboard/src/locales/cn.json
+++ b/inlong-dashboard/src/locales/cn.json
@@ -384,8 +384,8 @@
"meta.Consume.FilterEnabled": "是否过滤消费",
"meta.Consume.Yes": "是",
"meta.Consume.No": "否",
- "meta.Consume.Owner": "消费责任人",
- "meta.Consume.OwnersExtra": "可查看、修改消费信息",
+ "meta.Consume.Owner": "订阅责任人",
+ "meta.Consume.OwnersExtra": "可查看、修改订阅信息",
"meta.Nodes.Name": "节点名称",
"meta.Nodes.Type": "类型",
"meta.Nodes.Owners": "责任人",
@@ -546,17 +546,17 @@
"pages.Approvals.GroupId": "数据流组 ID",
"pages.Approvals.MyApplication": "我的申请",
"pages.Approvals.MyApproval": "我的审批",
- "pages.ConsumeCreate.ConsumerInformation": "消费信息",
+ "pages.ConsumeCreate.SubscriptionInformation": "订阅信息",
"pages.ConsumeCreate.Prev": "上一步",
"pages.ConsumeCreate.Next": "下一步",
"pages.ConsumeCreate.Submit": "提交审批",
"pages.ConsumeCreate.Back": "返回",
- "pages.ConsumeCreate.NewConsume": "新建消费",
+ "pages.ConsumeCreate.NewSubscribe": "新建订阅",
"pages.ConsumeDashboard.config.BeAllocated": "待分配",
"pages.ConsumeDashboard.config.ApplicationStatus": "申请状态",
"pages.ConsumeDashboard.config.Reject": "已驳回",
"pages.ConsumeDashboard.config.OperatingStatus": "运行状态",
- "pages.ConsumeDashboard.config.Total": "消费总数",
+ "pages.ConsumeDashboard.config.Total": "订阅总数",
"pages.ConsumeDashboard.config.Pending": "待审批",
"pages.ConsumeDashboard.config.ConsumerGroup": "消费组",
"pages.ConsumeDashboard.config.ConsumeInlongGroupId": "数据流组ID",
@@ -568,8 +568,8 @@
"pages.ConsumeDashboard.status.NoStatus": "暂无",
"pages.ConsumeDashboard.status.Passed": "已通过",
"pages.ConsumeDashboard.status.Cancelled": "已取消",
- "pages.ConsumeDashboard.NewConsume": "新建消费",
- "pages.ConsumeDetail.ConsumeDetails": "消费详情",
+ "pages.ConsumeDashboard.NewSubscribe": "新建订阅",
+ "pages.ConsumeDetail.SubscribeDetails": "订阅详情",
"pages.Clusters.Type": "类型",
"pages.Clusters.TypeAll": "全部",
"pages.Clusters.Create": "新建集群",
diff --git a/inlong-dashboard/src/locales/en.json
b/inlong-dashboard/src/locales/en.json
index 0bd7ec404..06020ce3e 100644
--- a/inlong-dashboard/src/locales/en.json
+++ b/inlong-dashboard/src/locales/en.json
@@ -384,8 +384,8 @@
"meta.Consume.FilterEnabled": "Filter Enabled",
"meta.Consume.Yes": "Yes",
"meta.Consume.No": "No",
- "meta.Consume.Owner": "Consume Owners",
- "meta.Consume.OwnersEx
[GitHub] [inlong] kinfuy commented on pull request #6973: [INLONG-6972][Dashboard] All "consumption" is changed to "subscription" for data consumption
kinfuy commented on PR #6973: URL: https://github.com/apache/inlong/pull/6973#issuecomment-1360653394 > @kinfuy Hi, I see you force-pushed a new commit. It is not recommended to use the force action, because it will overwrite the previous commit and review info. OK, thank you for your advice -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [inlong] ZuoFengZhang commented on pull request #6936: [INLONG-6935][Dashboard] Add Key-Value format for InlongStream
ZuoFengZhang commented on PR #6936: URL: https://github.com/apache/inlong/pull/6936#issuecomment-1360662381 The value of `datatype` configured on the front end is not passed to StreamSource, the default is csv format: https://github.com/apache/inlong/blob/master/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/source/pulsar/PulsarSourceOperator.java#L123 https://github.com/apache/inlong/blob/master/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/source/kafka/KafkaSourceOperator.java#L109 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [inlong] dockerzhang commented on pull request #7002: [INLONG-7000][Dashboard][Manager] Add AVRO format for InlongStream
dockerzhang commented on PR #7002: URL: https://github.com/apache/inlong/pull/7002#issuecomment-1360664072 @ZuoFengZhang I think we have to make sure the Sort could know this data type. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [inlong] dockerzhang commented on pull request #6999: [INLONG-6996][Dashboard][Manager] Add JSON format for InlongStream
dockerzhang commented on PR #6999: URL: https://github.com/apache/inlong/pull/6999#issuecomment-1360666794 @ZuoFengZhang I think we have to make sure the Sort could know this data type. Otherwise, do you miss the TubeMQ Operator? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[inlong] branch master updated: [INLONG-6985][DataProxy] Make maxMonitorCnt setting configurable (#6990)
This is an automated email from the ASF dual-hosted git repository. dockerzhang pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/inlong.git The following commit(s) were added to refs/heads/master by this push: new d9dc65e83 [INLONG-6985][DataProxy] Make maxMonitorCnt setting configurable (#6990) d9dc65e83 is described below commit d9dc65e836104f47e009f9eb11d380f1d1169e1f Author: Goson Zhang <4675...@qq.com> AuthorDate: Wed Dec 21 09:50:12 2022 +0800 [INLONG-6985][DataProxy] Make maxMonitorCnt setting configurable (#6990) --- .../conf/dataproxy-mulit-pulsar-http-example.conf | 7 ++- .../conf/dataproxy-mulit-pulsar-udp-example.conf | 5 + inlong-dataproxy/conf/dataproxy-tubemq.conf| 12 inlong-dataproxy/conf/dataproxy.conf | 7 +++ .../apache/inlong/dataproxy/consts/ConfigConstants.java| 1 + .../org/apache/inlong/dataproxy/http/HttpBaseSource.java | 13 + .../java/org/apache/inlong/dataproxy/sink/PulsarSink.java | 14 ++ .../java/org/apache/inlong/dataproxy/sink/TubeSink.java| 14 ++ .../org/apache/inlong/dataproxy/source/BaseSource.java | 11 ++- 9 files changed, 70 insertions(+), 14 deletions(-) diff --git a/inlong-dataproxy/conf/dataproxy-mulit-pulsar-http-example.conf b/inlong-dataproxy/conf/dataproxy-mulit-pulsar-http-example.conf index b744ecefb..56d2d7a85 100644 --- a/inlong-dataproxy/conf/dataproxy-mulit-pulsar-http-example.conf +++ b/inlong-dataproxy/conf/dataproxy-mulit-pulsar-http-example.conf @@ -46,6 +46,7 @@ agent1.sources.http-source.metric-recovery-path = ./data/file/recovery agent1.sources.http-source.metric-agent-port=8003 agent1.sources.http-source.metric-cache-size=100 agent1.sources.http-source.set=10 +agent1.sources.http-source.max-monitor-cnt=50 agent1.channels.ch-msg1.type = memory agent1.channels.ch-msg1.capacity = 1 @@ -77,12 +78,16 @@ agent1.channels.ch-msg6.fsyncInterval = 10 agent1.sinks.pulsar-sink-msg1.channel = ch-msg1 agent1.sinks.pulsar-sink-msg1.type = org.apache.inlong.dataproxy.sink.PulsarSink +agent1.sinks.pulsar-sink-msg1.max-monitor-cnt=50 agent1.sinks.pulsar-sink-msg2.channel = ch-msg2 agent1.sinks.pulsar-sink-msg2.type = org.apache.inlong.dataproxy.sink.PulsarSink +agent1.sinks.pulsar-sink-msg2.max-monitor-cnt=50 agent1.sinks.pulsar-sink-msg5.channel = ch-msg5 agent1.sinks.pulsar-sink-msg5.type = org.apache.inlong.dataproxy.sink.PulsarSink +agent1.sinks.pulsar-sink-msg5.max-monitor-cnt=50 agent1.sinks.pulsar-sink-msg6.channel = ch-msg6 -agent1.sinks.pulsar-sink-msg6.type = org.apache.inlong.dataproxy.sink.PulsarSink \ No newline at end of file +agent1.sinks.pulsar-sink-msg6.type = org.apache.inlong.dataproxy.sink.PulsarSink +agent1.sinks.pulsar-sink-msg6.max-monitor-cnt=50 \ No newline at end of file diff --git a/inlong-dataproxy/conf/dataproxy-mulit-pulsar-udp-example.conf b/inlong-dataproxy/conf/dataproxy-mulit-pulsar-udp-example.conf index 018b02bc1..827db35a1 100644 --- a/inlong-dataproxy/conf/dataproxy-mulit-pulsar-udp-example.conf +++ b/inlong-dataproxy/conf/dataproxy-mulit-pulsar-udp-example.conf @@ -46,6 +46,7 @@ agent1.sources.upd-source.metric-recovery-path = ./data/file/recovery agent1.sources.upd-source.metric-agent-port = 8003 agent1.sources.upd-source.metric-cache-size = 100 agent1.sources.upd-source.set = 10 +agent1.sources.upd-source.max-monitor-cnt=50 agent1.channels.ch-msg1.type = memory agent1.channels.ch-msg1.capacity = 1 @@ -77,12 +78,16 @@ agent1.channels.ch-msg6.fsyncInterval = 10 agent1.sinks.pulsar-sink-msg1.channel = ch-msg1 agent1.sinks.pulsar-sink-msg1.type = org.apache.inlong.dataproxy.sink.PulsarSink +agent1.sinks.pulsar-sink-msg1.max-monitor-cnt=50 agent1.sinks.pulsar-sink-msg2.channel = ch-msg2 agent1.sinks.pulsar-sink-msg2.type = org.apache.inlong.dataproxy.sink.PulsarSink +agent1.sinks.pulsar-sink-msg2.max-monitor-cnt=50 agent1.sinks.pulsar-sink-msg5.channel = ch-msg5 agent1.sinks.pulsar-sink-msg5.type = org.apache.inlong.dataproxy.sink.PulsarSink +agent1.sinks.pulsar-sink-msg5.max-monitor-cnt=50 agent1.sinks.pulsar-sink-msg6.channel = ch-msg6 agent1.sinks.pulsar-sink-msg6.type = org.apache.inlong.dataproxy.sink.PulsarSink +agent1.sinks.pulsar-sink-msg6.max-monitor-cnt=50 diff --git a/inlong-dataproxy/conf/dataproxy-tubemq.conf b/inlong-dataproxy/conf/dataproxy-tubemq.conf index 67f2e7fcf..9e4180165 100644 --- a/inlong-dataproxy/conf/dataproxy-tubemq.conf +++ b/inlong-dataproxy/conf/dataproxy-tubemq.conf @@ -51,6 +51,7 @@ agent1.sources.tcp-source.set=10 agent1.sources.tcp-source.old-metric-on=true agent1.sources.tcp-source.new-metric-on=true agent1.sources.tcp-source.metric_topic_prefix=manager_tmertic +agent1.sources.tcp-source.max-monitor-cnt=50 # http-source agent1.sources.http-source.channels = ch-msg1 ch-msg2
[GitHub] [inlong] dockerzhang merged pull request #6990: [INLONG-6985][DataProxy] Make maxMonitorCnt setting configurable
dockerzhang merged PR #6990: URL: https://github.com/apache/inlong/pull/6990 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [inlong] dockerzhang commented on pull request #6936: [INLONG-6935][Dashboard] Add Key-Value format for InlongStream
dockerzhang commented on PR #6936: URL: https://github.com/apache/inlong/pull/6936#issuecomment-1360675077 > The value of `datatype` configured on the front end is not passed to StreamSource, the default is csv format: > > https://github.com/apache/inlong/blob/master/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/source/pulsar/PulsarSourceOperator.java#L123 > > https://github.com/apache/inlong/blob/master/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/source/kafka/KafkaSourceOperator.java#L109 Could you add another PR to make sure the manager supports the KV type? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [inlong] leezng merged pull request #6927: [INLONG-6926][Dashboard] Change the cluster type to lower case
leezng merged PR #6927: URL: https://github.com/apache/inlong/pull/6927 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [inlong] leezng merged pull request #6986: [INLONG-6982][Dashboard] Data source and data storage bullet box title optimization
leezng merged PR #6986: URL: https://github.com/apache/inlong/pull/6986 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[inlong] branch master updated: [INLONG-6982][Dashboard] Data source and data storage bullet box title optimization (#6986)
This is an automated email from the ASF dual-hosted git repository.
leezng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new 337213ac9 [INLONG-6982][Dashboard] Data source and data storage bullet
box title optimization (#6986)
337213ac9 is described below
commit 337213ac93bb42f597f9be945df540e2325be0e9
Author: Lizhen <88174078+bluew...@users.noreply.github.com>
AuthorDate: Wed Dec 21 10:10:53 2022 +0800
[INLONG-6982][Dashboard] Data source and data storage bullet box title
optimization (#6986)
---
inlong-dashboard/src/locales/cn.json | 2 ++
inlong-dashboard/src/locales/en.json | 2 ++
inlong-dashboard/src/pages/GroupDetail/DataSources/DetailModal.tsx | 7 ++-
inlong-dashboard/src/pages/GroupDetail/DataStorage/DetailModal.tsx | 2 +-
4 files changed, 11 insertions(+), 2 deletions(-)
diff --git a/inlong-dashboard/src/locales/cn.json
b/inlong-dashboard/src/locales/cn.json
index ebe11ada6..89c6256c9 100644
--- a/inlong-dashboard/src/locales/cn.json
+++ b/inlong-dashboard/src/locales/cn.json
@@ -464,6 +464,7 @@
"pages.GroupDetail.Sources.DeleteSuccessfully": "删除成功",
"pages.GroupDetail.Sources.DataStreams": "数据流",
"pages.GroupDetail.Sources.Create": "新建数据源",
+ "pages.GroupDetail.Sources.Edit": "编辑数据源",
"pages.GroupDetail.Sources.status.Disable": "失效",
"pages.GroupDetail.Sources.status.Normal": "正常",
"pages.GroupDetail.Sources.status.Failure": "失败",
@@ -480,6 +481,7 @@
"pages.GroupDetail.Sources.status.BeenFrozen": "已下发停止",
"pages.GroupDetail.Sources.status.BeenActive": "已下发重启",
"pages.GroupDetail.Sink.New": "新建流向配置",
+ "pages.GroupDetail.Sink.Edit": "编辑流向配置",
"pages.GroupDetail.Sink.Type": "类型",
"pages.GroupDetail.Sink.DataStreams": "数据流",
"pages.GroupDetail.Sink.Status.New": "新建",
diff --git a/inlong-dashboard/src/locales/en.json
b/inlong-dashboard/src/locales/en.json
index 06020ce3e..73594640e 100644
--- a/inlong-dashboard/src/locales/en.json
+++ b/inlong-dashboard/src/locales/en.json
@@ -461,6 +461,7 @@
"pages.GroupDetail.Sources.DeleteSuccessfully": "Delete successfully",
"pages.GroupDetail.Sources.DataStreams": "DataStreams",
"pages.GroupDetail.Sources.Create": "Create",
+ "pages.GroupDetail.Sources.Edit": "Edit",
"pages.GroupDetail.Sources.status.Disable": "Disable",
"pages.GroupDetail.Sources.status.Normal": "Normal",
"pages.GroupDetail.Sources.status.Failure": "Failure",
@@ -477,6 +478,7 @@
"pages.GroupDetail.Sources.status.BeenFrozen": "BeenStop",
"pages.GroupDetail.Sources.status.BeenActive": "BeenRestart",
"pages.GroupDetail.Sink.New": "Create",
+ "pages.GroupDetail.Sink.Edit": "Edit",
"pages.GroupDetail.Sink.Type": "Type",
"pages.GroupDetail.Sink.DataStreams": "DataStreams",
"pages.GroupDetail.Sink.Status.New": "New",
diff --git a/inlong-dashboard/src/pages/GroupDetail/DataSources/DetailModal.tsx
b/inlong-dashboard/src/pages/GroupDetail/DataSources/DetailModal.tsx
index 1ba545076..6d346883b 100644
--- a/inlong-dashboard/src/pages/GroupDetail/DataSources/DetailModal.tsx
+++ b/inlong-dashboard/src/pages/GroupDetail/DataSources/DetailModal.tsx
@@ -97,7 +97,12 @@ const Comp: React.FC = ({ id, inlongGroupId,
defaultType, ...modalProps }
return (
<>
-
+
setType(values.sourceType)}
diff --git a/inlong-dashboard/src/pages/GroupDetail/DataStorage/DetailModal.tsx
b/inlong-dashboard/src/pages/GroupDetail/DataStorage/DetailModal.tsx
index 4da3893a9..540889a87 100644
--- a/inlong-dashboard/src/pages/GroupDetail/DataStorage/DetailModal.tsx
+++ b/inlong-dashboard/src/pages/GroupDetail/DataStorage/DetailModal.tsx
@@ -155,7 +155,7 @@ const Comp: React.FC = ({ inlongGroupId,
defaultType, id, ...m
return (
[GitHub] [inlong] EMsnap commented on a diff in pull request #6979: [INLONG-6978][Sort] Get primary keys for PostgreSQL
EMsnap commented on code in PR #6979:
URL: https://github.com/apache/inlong/pull/6979#discussion_r1053911832
##
inlong-sort/sort-connectors/jdbc/src/main/java/org/apache/inlong/sort/jdbc/internal/JdbcMultiBatchingComm.java:
##
@@ -173,4 +174,47 @@ private static RowData getPrimaryKey(RowData row,
RowData.FieldGetter[] fieldGet
return pkRow;
}
+public static JdbcOptions getExecJdbcOptions(JdbcOptions jdbcOptions,
String tableIdentifier) {
+JdbcOptions jdbcExecOptions =
+JdbcOptions.builder()
+.setDBUrl(jdbcOptions.getDbURL() + "/" +
getDatabaseNameFromIdentifier(tableIdentifier))
+
.setTableName(getTableNameFromIdentifier(tableIdentifier))
+.setDialect(jdbcOptions.getDialect())
+.setParallelism(jdbcOptions.getParallelism())
+
.setConnectionCheckTimeoutSeconds(jdbcOptions.getConnectionCheckTimeoutSeconds())
+.setDriverName(jdbcOptions.getDriverName())
+.setUsername(jdbcOptions.getUsername().orElse(""))
+.setPassword(jdbcOptions.getPassword().orElse(""))
+.build();
+return jdbcExecOptions;
+}
+
+/**
+ * Get table name From tableIdentifier
+ * tableIdentifier maybe: ${dbName}.${tbName} or
${dbName}.${schemaName}.${tbName}
+ *
+ * @param tableIdentifier The table identifier for which to get table name.
+ */
+public static String getTableNameFromIdentifier(String tableIdentifier) {
+String[] fileArray = tableIdentifier.split("\\.");
Review Comment:
please use a name that has the right meaning
##
inlong-sort/sort-connectors/jdbc/src/main/java/org/apache/inlong/sort/jdbc/internal/JdbcMultiBatchingComm.java:
##
@@ -173,4 +174,47 @@ private static RowData getPrimaryKey(RowData row,
RowData.FieldGetter[] fieldGet
return pkRow;
}
+public static JdbcOptions getExecJdbcOptions(JdbcOptions jdbcOptions,
String tableIdentifier) {
+JdbcOptions jdbcExecOptions =
+JdbcOptions.builder()
+.setDBUrl(jdbcOptions.getDbURL() + "/" +
getDatabaseNameFromIdentifier(tableIdentifier))
+
.setTableName(getTableNameFromIdentifier(tableIdentifier))
+.setDialect(jdbcOptions.getDialect())
+.setParallelism(jdbcOptions.getParallelism())
+
.setConnectionCheckTimeoutSeconds(jdbcOptions.getConnectionCheckTimeoutSeconds())
+.setDriverName(jdbcOptions.getDriverName())
+.setUsername(jdbcOptions.getUsername().orElse(""))
+.setPassword(jdbcOptions.getPassword().orElse(""))
+.build();
+return jdbcExecOptions;
+}
+
+/**
+ * Get table name From tableIdentifier
+ * tableIdentifier maybe: ${dbName}.${tbName} or
${dbName}.${schemaName}.${tbName}
+ *
+ * @param tableIdentifier The table identifier for which to get table name.
+ */
+public static String getTableNameFromIdentifier(String tableIdentifier) {
+String[] fileArray = tableIdentifier.split("\\.");
Review Comment:
Maybe IdentifierArray?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [inlong] gong commented on a diff in pull request #6979: [INLONG-6978][Sort] Get primary keys for PostgreSQL
gong commented on code in PR #6979:
URL: https://github.com/apache/inlong/pull/6979#discussion_r1053912126
##
inlong-sort/sort-connectors/jdbc/src/main/java/org/apache/inlong/sort/jdbc/dialect/PostgresDialect.java:
##
@@ -169,7 +169,7 @@ public List unsupportedTypes() {
public PreparedStatement setQuerySql(Connection conn,
String tableIdentifier) throws SQLException {
Review Comment:
The method name is too generic. It can't express specific meaning.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [inlong] EMsnap merged pull request #6750: [INLONG-6657][Sort] Add dirty metric for Kafka
EMsnap merged PR #6750: URL: https://github.com/apache/inlong/pull/6750 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[inlong] branch master updated (337213ac9 -> e6a8d9c7d)
This is an automated email from the ASF dual-hosted git repository. zirui pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/inlong.git from 337213ac9 [INLONG-6982][Dashboard] Data source and data storage bullet box title optimization (#6986) add e6a8d9c7d [INLONG-6657][Sort] Add dirty metric for Kafka (#6750) No new revisions were added by this update. Summary of changes: .../sort/base/metric/sub/SinkSubMetricData.java| 15 ++- .../sort/base/metric/sub/SinkTopicMetricData.java | 102 .../inlong/sort/base/util/MetricStateUtils.java| 38 +- .../kafka/DynamicKafkaSerializationSchema.java | 8 ++ .../inlong/sort/kafka/FlinkKafkaProducer.java | 132 + .../apache/inlong/sort/kafka/KafkaDynamicSink.java | 12 +- .../sort/kafka/table/KafkaDynamicTableFactory.java | 10 +- .../table/UpsertKafkaDynamicTableFactory.java | 5 +- 8 files changed, 258 insertions(+), 64 deletions(-) copy inlong-dataproxy/dataproxy-source/src/main/java/org/apache/inlong/dataproxy/config/loader/CommonPropertiesLoader.java => inlong-sort/sort-connectors/base/src/main/java/org/apache/inlong/sort/base/metric/sub/SinkSubMetricData.java (75%) create mode 100644 inlong-sort/sort-connectors/base/src/main/java/org/apache/inlong/sort/base/metric/sub/SinkTopicMetricData.java
[GitHub] [inlong] dockerzhang merged pull request #6974: [INLONG-6968][Sort] Fix the SqlParseException: Encountered "\'"
dockerzhang merged PR #6974: URL: https://github.com/apache/inlong/pull/6974 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[inlong] branch master updated: [INLONG-6968][Sort] Fix the SqlParseException: Encountered "\'" (#6974)
This is an automated email from the ASF dual-hosted git repository.
dockerzhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new 2f7529146 [INLONG-6968][Sort] Fix the SqlParseException: Encountered
"\'" (#6974)
2f7529146 is described below
commit 2f75291467b9f1045e56ff15806fd0231284b812
Author: wangpeix <105137931+wangp...@users.noreply.github.com>
AuthorDate: Wed Dec 21 10:33:47 2022 +0800
[INLONG-6968][Sort] Fix the SqlParseException: Encountered "\'" (#6974)
Co-authored-by: wangpeix
---
.../java/org/apache/inlong/sort/parser/impl/NativeFlinkSqlParser.java | 2 +-
.../java/org/apache/inlong/sort/parser/NativeFlinkSqlParserTest.java| 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git
a/inlong-sort/sort-core/src/main/java/org/apache/inlong/sort/parser/impl/NativeFlinkSqlParser.java
b/inlong-sort/sort-core/src/main/java/org/apache/inlong/sort/parser/impl/NativeFlinkSqlParser.java
index dbbaa069a..3227c2ad1 100644
---
a/inlong-sort/sort-core/src/main/java/org/apache/inlong/sort/parser/impl/NativeFlinkSqlParser.java
+++
b/inlong-sort/sort-core/src/main/java/org/apache/inlong/sort/parser/impl/NativeFlinkSqlParser.java
@@ -82,7 +82,7 @@ public class NativeFlinkSqlParser implements Parser {
Preconditions.checkNotNull(tableEnv, "tableEnv is null");
List createTableSqls = new ArrayList<>();
List insertSqls = new ArrayList<>();
-String[] statementSet = statements.split(";");
+String[] statementSet = statements.split(";(\\r?\\n|\\r)");
for (String statement : statementSet) {
statement = statement.trim();
if (statement.toUpperCase(Locale.ROOT).startsWith("CREATE TABLE")
|| statement.toUpperCase(Locale.ROOT)
diff --git
a/inlong-sort/sort-core/src/test/java/org/apache/inlong/sort/parser/NativeFlinkSqlParserTest.java
b/inlong-sort/sort-core/src/test/java/org/apache/inlong/sort/parser/NativeFlinkSqlParserTest.java
index 07f65d21b..a8f60f0be 100644
---
a/inlong-sort/sort-core/src/test/java/org/apache/inlong/sort/parser/NativeFlinkSqlParserTest.java
+++
b/inlong-sort/sort-core/src/test/java/org/apache/inlong/sort/parser/NativeFlinkSqlParserTest.java
@@ -71,7 +71,7 @@ public class NativeFlinkSqlParserTest {
+ "SELECT \n"
+ "`name` AS `name`,\n"
+ "`age` AS `age`\n"
-+ "FROM `table_1`;";
++ "FROM `table_1`;\n";
NativeFlinkSqlParser parser =
NativeFlinkSqlParser.getInstance(tableEnv, data);
ParseResult result = parser.parse();
Assert.assertTrue(result.tryExecute());
[GitHub] [inlong] dockerzhang merged pull request #6989: [INLONG-6988][Manager] Use the data node info for StarRocks
dockerzhang merged PR #6989: URL: https://github.com/apache/inlong/pull/6989 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[inlong] branch master updated: [INLONG-6988][Manager] Use the data node info for StarRocks (#6989)
This is an automated email from the ASF dual-hosted git repository.
dockerzhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new 6bad73345 [INLONG-6988][Manager] Use the data node info for StarRocks
(#6989)
6bad73345 is described below
commit 6bad73345a13c9f8f8ad7f2b83e73ccfd7fc4653
Author: fuweng11 <76141879+fuwen...@users.noreply.github.com>
AuthorDate: Wed Dec 21 10:34:12 2022 +0800
[INLONG-6988][Manager] Use the data node info for StarRocks (#6989)
---
.../inlong/manager/common/consts/DataNodeType.java | 1 +
.../pojo/node/starrocks/StarRocksDataNodeDTO.java | 64
.../pojo/node/starrocks/StarRocksDataNodeInfo.java | 51 +
.../node/starrocks/StarRocksDataNodeRequest.java | 46
.../node/starrocks/StarRocksDataNodeOperator.java | 86 ++
.../sink/starrocks/StarRocksResourceOperator.java | 24 +-
.../sink/starrocks/StarRocksSinkOperator.java | 10 +++
7 files changed, 281 insertions(+), 1 deletion(-)
diff --git
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/DataNodeType.java
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/DataNodeType.java
index 16fa92c77..a32adac37 100644
---
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/DataNodeType.java
+++
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/DataNodeType.java
@@ -29,5 +29,6 @@ public class DataNodeType {
public static final String CLICKHOUSE = "CLICKHOUSE";
public static final String ELASTICSEARCH = "ELASTICSEARCH";
public static final String MYSQL = "MYSQL";
+public static final String STARROCKS = "STARROCKS";
}
diff --git
a/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/node/starrocks/StarRocksDataNodeDTO.java
b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/node/starrocks/StarRocksDataNodeDTO.java
new file mode 100644
index 0..c01d99c6b
--- /dev/null
+++
b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/node/starrocks/StarRocksDataNodeDTO.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.manager.pojo.node.starrocks;
+
+import io.swagger.annotations.ApiModel;
+import io.swagger.annotations.ApiModelProperty;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import org.apache.inlong.manager.common.enums.ErrorCodeEnum;
+import org.apache.inlong.manager.common.exceptions.BusinessException;
+import org.apache.inlong.manager.common.util.CommonBeanUtils;
+import org.apache.inlong.manager.common.util.JsonUtils;
+
+import javax.validation.constraints.NotNull;
+
+/**
+ * StarRocks data node info
+ */
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+@ApiModel("StarRocks data node info")
+public class StarRocksDataNodeDTO {
+
+@ApiModelProperty("StarRocks FE http address")
+private String loadUrl;
+
+/**
+ * Get the dto instance from the request
+ */
+public static StarRocksDataNodeDTO getFromRequest(StarRocksDataNodeRequest
request) throws Exception {
+return CommonBeanUtils.copyProperties(request,
StarRocksDataNodeDTO::new, true);
+}
+
+/**
+ * Get the dto instance from the JSON string.
+ */
+public static StarRocksDataNodeDTO getFromJson(@NotNull String extParams) {
+try {
+return JsonUtils.parseObject(extParams,
StarRocksDataNodeDTO.class);
+} catch (Exception e) {
+throw new
BusinessException(ErrorCodeEnum.GROUP_INFO_INCORRECT.getMessage());
+}
+}
+
+}
diff --git
a/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/node/starrocks/StarRocksDataNodeInfo.java
b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/node/starrocks/StarRocksDataNodeInfo.java
new file mode 100644
index 0..8595445e8
--- /dev/null
+++
b/inlong-manager/manager-pojo/src/main/java/org/apa
[inlong] branch master updated (d9dc65e83 -> 3f7ab8e54)
This is an automated email from the ASF dual-hosted git repository. leezng pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/inlong.git from d9dc65e83 [INLONG-6985][DataProxy] Make maxMonitorCnt setting configurable (#6990) add 3f7ab8e54 [INLONG-6926][Dashboard] Cluster management type label, lower case (#6927) No new revisions were added by this update. Summary of changes: inlong-dashboard/src/metas/clusters/common/ClusterDefaultInfo.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-)
[GitHub] [inlong] kuansix commented on a diff in pull request #6979: [INLONG-6978][Sort] Get primary keys for PostgreSQL
kuansix commented on code in PR #6979:
URL: https://github.com/apache/inlong/pull/6979#discussion_r1053920456
##
inlong-sort/sort-connectors/jdbc/src/main/java/org/apache/inlong/sort/jdbc/internal/JdbcMultiBatchingComm.java:
##
@@ -173,4 +174,47 @@ private static RowData getPrimaryKey(RowData row,
RowData.FieldGetter[] fieldGet
return pkRow;
}
+public static JdbcOptions getExecJdbcOptions(JdbcOptions jdbcOptions,
String tableIdentifier) {
+JdbcOptions jdbcExecOptions =
+JdbcOptions.builder()
+.setDBUrl(jdbcOptions.getDbURL() + "/" +
getDatabaseNameFromIdentifier(tableIdentifier))
+
.setTableName(getTableNameFromIdentifier(tableIdentifier))
+.setDialect(jdbcOptions.getDialect())
+.setParallelism(jdbcOptions.getParallelism())
+
.setConnectionCheckTimeoutSeconds(jdbcOptions.getConnectionCheckTimeoutSeconds())
+.setDriverName(jdbcOptions.getDriverName())
+.setUsername(jdbcOptions.getUsername().orElse(""))
+.setPassword(jdbcOptions.getPassword().orElse(""))
+.build();
+return jdbcExecOptions;
+}
+
+/**
+ * Get table name From tableIdentifier
+ * tableIdentifier maybe: ${dbName}.${tbName} or
${dbName}.${schemaName}.${tbName}
+ *
+ * @param tableIdentifier The table identifier for which to get table name.
+ */
+public static String getTableNameFromIdentifier(String tableIdentifier) {
+String[] fileArray = tableIdentifier.split("\\.");
Review Comment:
fieldArray done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [inlong] kuansix commented on a diff in pull request #6979: [INLONG-6978][Sort] Get primary keys for PostgreSQL
kuansix commented on code in PR #6979:
URL: https://github.com/apache/inlong/pull/6979#discussion_r1053920553
##
inlong-sort/sort-connectors/jdbc/src/main/java/org/apache/inlong/sort/jdbc/dialect/PostgresDialect.java:
##
@@ -169,7 +169,7 @@ public List unsupportedTypes() {
public PreparedStatement setQuerySql(Connection conn,
String tableIdentifier) throws SQLException {
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [inlong] leezng commented on a diff in pull request #6995: [INLONG-6994][Dashboard] Adding MQ type filter box for data flow group, data consumption list
leezng commented on code in PR #6995:
URL: https://github.com/apache/inlong/pull/6995#discussion_r1053920630
##
inlong-dashboard/src/pages/GroupDashboard/config.tsx:
##
@@ -68,6 +69,17 @@ export const getFilterFormContent = defaultValues => [
dropdownMatchSelectWidth: false,
},
},
+ {
+type: 'select',
+name: 'mqType',
+label: i18n.t('meta.Group.MQType'),
+initialValue: defaultValues.mqType,
+props: {
+ allowClear: true,
+ options: allDefaultGroups,
Review Comment:
This is consistent with the above.
##
inlong-dashboard/src/pages/ConsumeDashboard/config.tsx:
##
@@ -79,6 +80,17 @@ export const getFilterFormContent = defaultValues => [
dropdownMatchSelectWidth: false,
},
},
+ {
+type: 'select',
+name: 'mqType',
+label: i18n.t('meta.Consume.MQType'),
+initialValue: defaultValues.mqType,
+props: {
+ allowClear: true,
+ options: allDefaultConsumes,
Review Comment:
You can't use `allDefaultConsumes` directly, you need to use the exposed
`hooks`, as follows:
https://github.com/apache/inlong/blob/e6a8d9c7d42a8f8b8aae9ec88fa9bbae1c0a451c/inlong-dashboard/src/pages/Nodes/index.tsx#L33
https://github.com/apache/inlong/blob/e6a8d9c7d42a8f8b8aae9ec88fa9bbae1c0a451c/inlong-dashboard/src/pages/Nodes/index.tsx#L103
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [inlong] dockerzhang commented on issue #7003: [Feature][Dashboard] Refactor format configuration interface of InlongStream
dockerzhang commented on issue #7003: URL: https://github.com/apache/inlong/issues/7003#issuecomment-1360764635 @ZuoFengZhang, thanks for your suggestion; I couldn't agree more. but I think there are some points we should consider: - All optional format configurations must go through InLongMsg by default, and the user will not perceive it - If you do not pass InLongMsg, select No in the advanced options - Synchronous modification of dashboard/manager/sort -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [inlong] dockerzhang commented on a diff in pull request #6969: [INLONG-6961][Dashboard][Manager] Support for configing non-InlongMsg formats
dockerzhang commented on code in PR #6969: URL: https://github.com/apache/inlong/pull/6969#discussion_r1053929977 ## inlong-dashboard/src/locales/cn.json: ## @@ -374,6 +374,8 @@ "meta.Stream.ExecuteWorkflow": "执行工作流", "meta.Stream.ExecuteConfirm": "确认执行工作流吗?", "meta.Stream.ExecuteSuccess": "执行成功", + "meta.Stream.WrapWithInlongMsg": "InlongMsg打包", Review Comment: ```suggestion "meta.Stream.WrapWithInlongMsg": "InlongMsg 打包", ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [inlong] kuansix commented on a diff in pull request #6931: [INLONG-6925][Sort] Add read phase metric and table level metric for MongoDB-CDC
kuansix commented on code in PR #6931:
URL: https://github.com/apache/inlong/pull/6931#discussion_r1053930896
##
inlong-sort/sort-connectors/mongodb-cdc/src/main/java/org/apache/inlong/sort/cdc/mongodb/DebeziumSourceFunction.java:
##
@@ -485,7 +498,19 @@ public void run(SourceContext sourceContext) throws
Exception {
@Override
public void deserialize(SourceRecord record,
Collector out) throws Exception {
-if (sourceMetricData != null) {
+if (sourceMetricData != null && record != null
&& migrateAll) {
+Struct value = (Struct) record.value();
+Struct source =
value.getStruct(MongoDBEnvelope.NAMESPACE_FIELD);
+if (null == source) {
+source =
value.getStruct(RecordUtils.DOCUMENT_TO_FIELD);
+}
+String dbName =
source.getString(MongoDBEnvelope.NAMESPACE_DATABASE_FIELD);
+String tableName =
source.getString(MongoDBEnvelope.NAMESPACE_COLLECTION_FIELD);
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [inlong] ZuoFengZhang commented on pull request #6936: [INLONG-6935][Dashboard] Add Key-Value format for InlongStream
ZuoFengZhang commented on PR #6936: URL: https://github.com/apache/inlong/pull/6936#issuecomment-1360790749 LGTM -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [inlong] ZuoFengZhang commented on pull request #6936: [INLONG-6935][Dashboard] Add Key-Value format for InlongStream
ZuoFengZhang commented on PR #6936: URL: https://github.com/apache/inlong/pull/6936#issuecomment-1360796087 > > The value of `datatype` configured on the front end is not passed to StreamSource, the default is csv format: > > https://github.com/apache/inlong/blob/master/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/source/pulsar/PulsarSourceOperator.java#L123 > > https://github.com/apache/inlong/blob/master/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/source/kafka/KafkaSourceOperator.java#L109 > > Could you add another PR to make sure the manager supports the KV type? I have another pr, just fixed it. https://github.com/apache/inlong/pull/7001 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [inlong] gong commented on a diff in pull request #6931: [INLONG-6925][Sort] Add read phase metric and table level metric for MongoDB-CDC
gong commented on code in PR #6931: URL: https://github.com/apache/inlong/pull/6931#discussion_r1053936640 ## inlong-sort/sort-connectors/mongodb-cdc/src/main/java/org/apache/inlong/sort/cdc/mongodb/DebeziumSourceFunction.java: ## @@ -505,11 +505,11 @@ public void deserialize(SourceRecord record, Collector out) throws Exception source = value.getStruct(RecordUtils.DOCUMENT_TO_FIELD); } String dbName = source.getString(MongoDBEnvelope.NAMESPACE_DATABASE_FIELD); -String tableName = source.getString(MongoDBEnvelope.NAMESPACE_COLLECTION_FIELD); +String collectionName = source.getString(MongoDBEnvelope.NAMESPACE_COLLECTION_FIELD); SnapshotRecord snapshotRecord = SnapshotRecord.fromSource(source); boolean isSnapshotRecord = (SnapshotRecord.TRUE == snapshotRecord); sourceMetricData -.outputMetricsWithEstimate(dbName, tableName, isSnapshotRecord, value); +.outputMetricsWithEstimate(dbName, collectionName, isSnapshotRecord, value); Review Comment: mongo table level metric use collection. Not to use table. Full metric name is flink_taskmanager_job_task_operator_groupId_streamId_nodeId_database_collection_numRecordsIn. You need modify this method `outputMetricsWithEstimate` that produce metric group and label. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [inlong] gong commented on pull request #6931: [INLONG-6925][Sort] Add read phase metric and table level metric for MongoDB-CDC
gong commented on PR #6931: URL: https://github.com/apache/inlong/pull/6931#issuecomment-1360797992 > mongo table level metric use collection. Not to use table. Full metric name is `flink_taskmanager_job_task_operator_groupId_streamId_nodeId_database_collection_numRecordsIn` @kuansix here need modify -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [inlong] gong closed pull request #6931: [INLONG-6925][Sort] Add read phase metric and table level metric for MongoDB-CDC
gong closed pull request #6931: [INLONG-6925][Sort] Add read phase metric and table level metric for MongoDB-CDC URL: https://github.com/apache/inlong/pull/6931 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [inlong] gong commented on a diff in pull request #6931: [INLONG-6925][Sort] Add read phase metric and table level metric for MongoDB-CDC
gong commented on code in PR #6931: URL: https://github.com/apache/inlong/pull/6931#discussion_r1053937589 ## inlong-sort/sort-connectors/mongodb-cdc/src/main/java/org/apache/inlong/sort/cdc/mongodb/DebeziumSourceFunction.java: ## @@ -505,11 +505,11 @@ public void deserialize(SourceRecord record, Collector out) throws Exception source = value.getStruct(RecordUtils.DOCUMENT_TO_FIELD); } String dbName = source.getString(MongoDBEnvelope.NAMESPACE_DATABASE_FIELD); -String tableName = source.getString(MongoDBEnvelope.NAMESPACE_COLLECTION_FIELD); +String collectionName = source.getString(MongoDBEnvelope.NAMESPACE_COLLECTION_FIELD); SnapshotRecord snapshotRecord = SnapshotRecord.fromSource(source); boolean isSnapshotRecord = (SnapshotRecord.TRUE == snapshotRecord); sourceMetricData -.outputMetricsWithEstimate(dbName, tableName, isSnapshotRecord, value); +.outputMetricsWithEstimate(dbName, collectionName, isSnapshotRecord, value); Review Comment: @chestnut-c please review here -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [inlong] chestnut-c opened a new pull request, #7007: [INLONG-7006][Sort] Optimize reading metric at the table level, and the metric label is passed in by a specific connector.
chestnut-c opened a new pull request, #7007: URL: https://github.com/apache/inlong/pull/7007 ### Prepare a Pull Request [INLONG-7006][Sort] Optimize reading metric at the table level, and the metric label is passed in by a specific connector. - Fixes #7006 ### Motivation Optimize reading metric at the table level, and the metric label is passed in by a specific connector. ### Modifications The indicator label of the base class is passed in by the specific connector used to avoid introducing too many labels in the base class. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [inlong] gong merged pull request #6903: [INLONG-6899][Sort] StarRocks supports table level metric
gong merged PR #6903: URL: https://github.com/apache/inlong/pull/6903 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[inlong] branch master updated: [INLONG-6899][Sort] StarRocks supports table level metric (#6903)
This is an automated email from the ASF dual-hosted git repository.
pacinogong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new d0124270a [INLONG-6899][Sort] StarRocks supports table level metric
(#6903)
d0124270a is described below
commit d0124270a7b7778093c33e87288e0c4e06258cac
Author: Liao Rui
AuthorDate: Wed Dec 21 14:27:47 2022 +0800
[INLONG-6899][Sort] StarRocks supports table level metric (#6903)
---
.../sort/base/metric/sub/SinkSubMetricData.java| 11 +-
.../sort/base/metric/sub/SinkTableMetricData.java | 216 +
.../sort/base/metric/sub/SinkTopicMetricData.java | 2 +-
.../inlong/sort/base/util/MetricStateUtils.java| 2 +-
.../starrocks/manager/StarRocksSinkManager.java| 13 +-
.../table/sink/StarRocksDynamicSinkFunction.java | 10 +-
6 files changed, 240 insertions(+), 14 deletions(-)
diff --git
a/inlong-sort/sort-connectors/base/src/main/java/org/apache/inlong/sort/base/metric/sub/SinkSubMetricData.java
b/inlong-sort/sort-connectors/base/src/main/java/org/apache/inlong/sort/base/metric/sub/SinkSubMetricData.java
index 143a925ca..3f863a317 100644
---
a/inlong-sort/sort-connectors/base/src/main/java/org/apache/inlong/sort/base/metric/sub/SinkSubMetricData.java
+++
b/inlong-sort/sort-connectors/base/src/main/java/org/apache/inlong/sort/base/metric/sub/SinkSubMetricData.java
@@ -17,10 +17,12 @@
package org.apache.inlong.sort.base.metric.sub;
-import org.apache.inlong.sort.base.metric.SinkMetricData;
-
import java.util.Map;
+import org.apache.inlong.sort.base.metric.SinkMetricData;
+/**
+ * A collection class for handling sub metrics
+ */
public interface SinkSubMetricData {
/**
@@ -28,6 +30,5 @@ public interface SinkSubMetricData {
*
* @return The sub sink metric map
*/
-Map getSubSourceMetricMap();
-
-}
+Map getSubSinkMetricMap();
+}
\ No newline at end of file
diff --git
a/inlong-sort/sort-connectors/base/src/main/java/org/apache/inlong/sort/base/metric/sub/SinkTableMetricData.java
b/inlong-sort/sort-connectors/base/src/main/java/org/apache/inlong/sort/base/metric/sub/SinkTableMetricData.java
new file mode 100644
index 0..82dcf74f1
--- /dev/null
+++
b/inlong-sort/sort-connectors/base/src/main/java/org/apache/inlong/sort/base/metric/sub/SinkTableMetricData.java
@@ -0,0 +1,216 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.sort.base.metric.sub;
+
+import static org.apache.inlong.sort.base.Constants.DELIMITER;
+import static org.apache.inlong.sort.base.Constants.DIRTY_BYTES_OUT;
+import static org.apache.inlong.sort.base.Constants.DIRTY_RECORDS_OUT;
+import static org.apache.inlong.sort.base.Constants.NUM_BYTES_OUT;
+import static org.apache.inlong.sort.base.Constants.NUM_RECORDS_OUT;
+
+import com.google.common.collect.Maps;
+import java.nio.charset.StandardCharsets;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.stream.Collectors;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.flink.metrics.MetricGroup;
+import org.apache.inlong.sort.base.Constants;
+import org.apache.inlong.sort.base.metric.MetricOption;
+import org.apache.inlong.sort.base.metric.MetricOption.RegisteredMetric;
+import org.apache.inlong.sort.base.metric.MetricState;
+import org.apache.inlong.sort.base.metric.SinkMetricData;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * A collection class for handling sub metrics of table schema type
+ */
+public class SinkTableMetricData extends SinkMetricData implements
SinkSubMetricData {
+
+public static final Logger LOGGER =
LoggerFactory.getLogger(SinkTableMetricData.class);
+
+/**
+ * The sub sink metric data container of sink metric data
+ */
+private final Map subSinkMetricMap =
Maps.newHashMap();
+
+public SinkTableMetricData(MetricOption option, MetricGroup metricGroup) {
+super(option, metricGroup);
+}
+
+/**
+ * register sub sink metrics group from metric state
+ *
+ * @param metricState MetricState
+ */
+public void registerSubMetricsGroup(Metri
[GitHub] [inlong] gong commented on a diff in pull request #6931: [INLONG-6925][Sort] Add read phase metric and table level metric for MongoDB-CDC
gong commented on code in PR #6931: URL: https://github.com/apache/inlong/pull/6931#discussion_r1054025580 ## inlong-sort/sort-connectors/mongodb-cdc/src/main/java/org/apache/inlong/sort/cdc/mongodb/DebeziumSourceFunction.java: ## @@ -505,11 +505,11 @@ public void deserialize(SourceRecord record, Collector out) throws Exception source = value.getStruct(RecordUtils.DOCUMENT_TO_FIELD); } String dbName = source.getString(MongoDBEnvelope.NAMESPACE_DATABASE_FIELD); -String tableName = source.getString(MongoDBEnvelope.NAMESPACE_COLLECTION_FIELD); +String collectionName = source.getString(MongoDBEnvelope.NAMESPACE_COLLECTION_FIELD); SnapshotRecord snapshotRecord = SnapshotRecord.fromSource(source); boolean isSnapshotRecord = (SnapshotRecord.TRUE == snapshotRecord); sourceMetricData -.outputMetricsWithEstimate(dbName, tableName, isSnapshotRecord, value); +.outputMetricsWithEstimate(dbName, collectionName, isSnapshotRecord, value); Review Comment: @kuansix You can pull code and modfiy here after the [PR](https://github.com/apache/inlong/pull/7007) is merge -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [inlong] dockerzhang commented on pull request #7001: [INLONG-6998][Dashboard][Manager] Support ignore deserialization erro…
dockerzhang commented on PR #7001: URL: https://github.com/apache/inlong/pull/7001#issuecomment-1360924482 Please add more details about why we could ignore the deserialization error -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [inlong] dockerzhang merged pull request #6967: [INLONG-6960][Agent] Fix repeat uuid generation for different messages
dockerzhang merged PR #6967: URL: https://github.com/apache/inlong/pull/6967 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[inlong] branch master updated: [INLONG-6960][Agent] Fix repeat uuid generation for different messages (#6967)
This is an automated email from the ASF dual-hosted git repository.
dockerzhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new 2ea25400f [INLONG-6960][Agent] Fix repeat uuid generation for
different messages (#6967)
2ea25400f is described below
commit 2ea25400f881a42b343cbe48aaa5debfbccc17aa
Author: xueyingzhang <86780714+poc...@users.noreply.github.com>
AuthorDate: Wed Dec 21 14:57:28 2022 +0800
[INLONG-6960][Agent] Fix repeat uuid generation for different messages
(#6967)
---
.../inlong/agent/plugin/message/SequentialID.java | 78 +---
.../inlong/agent/plugin/sinks/SenderManager.java | 3 +-
.../agent/plugin/message/SequentialIDTest.java | 33 +
.../org/apache/inlong/common/util/SnowFlake.java | 84 ++
4 files changed, 153 insertions(+), 45 deletions(-)
diff --git
a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/message/SequentialID.java
b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/message/SequentialID.java
index 879e42e57..a551530ee 100755
---
a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/message/SequentialID.java
+++
b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/message/SequentialID.java
@@ -18,10 +18,14 @@
package org.apache.inlong.agent.plugin.message;
import org.apache.inlong.agent.utils.AgentUtils;
+import org.apache.inlong.common.util.SnowFlake;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.net.InetAddress;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.locks.ReentrantLock;
+import java.net.UnknownHostException;
+
+import static org.apache.inlong.common.util.SnowFlake.MAX_MACHINE_NUM;
/**
* Generate uniq sequential id, reset base id if max number
@@ -29,32 +33,35 @@ import java.util.concurrent.locks.ReentrantLock;
*/
public class SequentialID {
-private static final int MAX_ID = 20;
-private static final String IP_HEX = getHex();
+private static final Logger LOGGER =
LoggerFactory.getLogger(SequentialID.class);
+
private static SequentialID uniqueSequentialID = null;
-private final AtomicInteger id = new AtomicInteger(1);
-private final ReentrantLock idLock = new ReentrantLock();
-private final AtomicInteger uid = new AtomicInteger(1);
-private final ReentrantLock uidLock = new ReentrantLock();
+private SnowFlake snowFlake;
private SequentialID() {
-
+long machineId = ipStr2Int(AgentUtils.getLocalIp());
+snowFlake = new SnowFlake(machineId);
}
-private static String getHex() {
-String localIp = AgentUtils.getLocalIp();
+private long ipStr2Int(String ip) {
+long result = 0;
+InetAddress ipv;
try {
-InetAddress ia = InetAddress.getByName(localIp);
-byte[] hostBytes = ia.getAddress();
-StringBuilder sb = new StringBuilder();
-for (byte hostByte : hostBytes) {
-sb.append(String.format("%02x", hostByte));
-}
-return sb.toString();
-} catch (Exception e) {
-// ignore it
+ipv = InetAddress.getByName(ip);
+} catch (UnknownHostException e) {
+LOGGER.error("convert ip to int error", e);
+return AgentUtils.getRandomBySeed(10);
}
-return "";
+for (byte b : ipv.getAddress()) {
+result = result << 8 | (b & 0xFF);
+}
+if (result < 0) {
+result = AgentUtils.getRandomBySeed(10);
+}
+if (result > MAX_MACHINE_NUM) {
+result %= MAX_MACHINE_NUM;
+}
+return result;
}
/**
@@ -63,36 +70,19 @@ public class SequentialID {
public static synchronized SequentialID getInstance() {
if (uniqueSequentialID == null) {
-uniqueSequentialID = new SequentialID();
-}
-return uniqueSequentialID;
-}
-
-public int getNextId() {
-idLock.lock();
-try {
-if (id.get() > MAX_ID) {
-id.set(1);
+synchronized (SequentialID.class) {
+if (uniqueSequentialID == null) {
+uniqueSequentialID = new SequentialID();
+}
}
-return id.incrementAndGet();
-} finally {
-idLock.unlock();
}
+return uniqueSequentialID;
}
/**
* get next uuid
*/
public String getNextUuid() {
-uidLock.lock();
-try {
-if (uid.get() > MAX_ID) {
-uid.set(1);
-}
-return IP_HEX + "-" + String.format("%014x-%08x",
-System.currentTimeMillis(), uid.incrementAndGet());
-} finally {
-
[GitHub] [inlong] bluewang opened a new pull request, #7008: [INLONG-7005][Dashboard] Support starrocks node management
bluewang opened a new pull request, #7008: URL: https://github.com/apache/inlong/pull/7008 ### Prepare a Pull Request *(Change the title refer to the following example)* - Title Example: [INLONG-XYZ][Component] Title of the pull request *(The following *XYZ* should be replaced by the actual [GitHub Issue](https://github.com/apache/inlong/issues) number)* - Fixes #XYZ ### Modifications   -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [inlong] dockerzhang merged pull request #6936: [INLONG-6935][Dashboard] Add Key-Value format for InlongStream
dockerzhang merged PR #6936: URL: https://github.com/apache/inlong/pull/6936 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[inlong] branch master updated: [INLONG-6926][Dashboard] Add Key-Value format for InlongStream (#6926) (#6936)
This is an automated email from the ASF dual-hosted git repository.
dockerzhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new bf66c67f9 [INLONG-6926][Dashboard] Add Key-Value format for
InlongStream (#6926) (#6936)
bf66c67f9 is described below
commit bf66c67f94bb6b3892e024ee85e98122bc996b49
Author: kinfuy <37766068+kin...@users.noreply.github.com>
AuthorDate: Wed Dec 21 15:26:37 2022 +0800
[INLONG-6926][Dashboard] Add Key-Value format for InlongStream (#6926)
(#6936)
---
inlong-dashboard/src/metas/streams/common/StreamDefaultInfo.ts | 4
1 file changed, 4 insertions(+)
diff --git a/inlong-dashboard/src/metas/streams/common/StreamDefaultInfo.ts
b/inlong-dashboard/src/metas/streams/common/StreamDefaultInfo.ts
index 83562675c..843c15b9a 100644
--- a/inlong-dashboard/src/metas/streams/common/StreamDefaultInfo.ts
+++ b/inlong-dashboard/src/metas/streams/common/StreamDefaultInfo.ts
@@ -105,6 +105,10 @@ export class StreamDefaultInfo implements DataWithBackend,
RenderRow, RenderList
label: 'CSV',
value: 'CSV',
},
+{
+ label: 'KV',
+ value: 'KV',
+},
],
}),
rules: [{ required: true }],
[GitHub] [inlong] dockerzhang merged pull request #7007: [INLONG-7006][Sort] Optimize reading metric at the table level, and the metric label is passed in by a specific connector.
dockerzhang merged PR #7007: URL: https://github.com/apache/inlong/pull/7007 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[inlong] branch master updated: [INLONG-7006][Sort] Optimize reading metric at the table level, and the metric label is passed in by a specific connector. (#7007)
This is an automated email from the ASF dual-hosted git repository.
dockerzhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new af3437e3d [INLONG-7006][Sort] Optimize reading metric at the table
level, and the metric label is passed in by a specific connector. (#7007)
af3437e3d is described below
commit af3437e3dfdb4ff81fb6807ec2e7b99e66915fcb
Author: chestnufang <65438734+chestnu...@users.noreply.github.com>
AuthorDate: Wed Dec 21 15:50:43 2022 +0800
[INLONG-7006][Sort] Optimize reading metric at the table level, and the
metric label is passed in by a specific connector. (#7007)
Co-authored-by: chestnufang
---
.../base/metric/sub/SourceTableMetricData.java | 89 +-
.../sort/cdc/debezium/DebeziumSourceFunction.java | 5 +-
.../source/metrics/MySqlSourceReaderMetrics.java | 5 +-
.../oracle/debezium/DebeziumSourceFunction.java| 5 +-
.../sort/cdc/postgres/DebeziumSourceFunction.java | 5 +-
5 files changed, 51 insertions(+), 58 deletions(-)
diff --git
a/inlong-sort/sort-connectors/base/src/main/java/org/apache/inlong/sort/base/metric/sub/SourceTableMetricData.java
b/inlong-sort/sort-connectors/base/src/main/java/org/apache/inlong/sort/base/metric/sub/SourceTableMetricData.java
index 3ca762bea..eac1224f6 100644
---
a/inlong-sort/sort-connectors/base/src/main/java/org/apache/inlong/sort/base/metric/sub/SourceTableMetricData.java
+++
b/inlong-sort/sort-connectors/base/src/main/java/org/apache/inlong/sort/base/metric/sub/SourceTableMetricData.java
@@ -24,6 +24,7 @@ import static
org.apache.inlong.sort.base.Constants.READ_PHASE;
import com.google.common.collect.Maps;
import java.nio.charset.StandardCharsets;
+import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.stream.Collectors;
@@ -55,9 +56,15 @@ public class SourceTableMetricData extends SourceMetricData
implements SourceSub
* The sub source metric data container of source metric data
*/
private final Map subSourceMetricMap =
Maps.newHashMap();
+/**
+ * The sub source metric label of source sub metric group and this must be
consistent with the schema information
+ * recorded by the specific connector.
+ */
+private final List tableMetricLabelList;
-public SourceTableMetricData(MetricOption option, MetricGroup metricGroup)
{
+public SourceTableMetricData(MetricOption option, MetricGroup metricGroup,
List tableMetricLabelList) {
super(option, metricGroup);
+this.tableMetricLabelList = tableMetricLabelList;
}
/**
@@ -83,7 +90,7 @@ public class SourceTableMetricData extends SourceMetricData
implements SourceSub
}
Map subMetricStateMap =
metricState.getSubMetricStateMap();
for (Entry subMetricStateEntry :
subMetricStateMap.entrySet()) {
-String[] schemaInfoArray =
parseSchemaIdentify(subMetricStateEntry.getKey());
+String[] schemaInfoArray =
subMetricStateEntry.getKey().split(Constants.SPILT_SEMICOLON);
final MetricState subMetricState = subMetricStateEntry.getValue();
SourceMetricData subSourceMetricData =
buildSubSourceMetricData(schemaInfoArray,
subMetricState, this);
@@ -121,13 +128,8 @@ public class SourceTableMetricData extends
SourceMetricData implements SourceSub
String metricGroupLabels = labels.entrySet().stream().map(entry ->
entry.getKey() + "=" + entry.getValue())
.collect(Collectors.joining(DELIMITER));
StringBuilder labelBuilder = new StringBuilder(metricGroupLabels);
-if (schemaInfoArray.length == 2) {
-
labelBuilder.append(DELIMITER).append(Constants.DATABASE_NAME).append("=").append(schemaInfoArray[0])
-
.append(DELIMITER).append(Constants.TABLE_NAME).append("=").append(schemaInfoArray[1]);
-} else if (schemaInfoArray.length == 3) {
-
labelBuilder.append(DELIMITER).append(Constants.DATABASE_NAME).append("=").append(schemaInfoArray[0])
-
.append(DELIMITER).append(Constants.SCHEMA_NAME).append("=").append(schemaInfoArray[1])
-
.append(DELIMITER).append(Constants.TABLE_NAME).append("=").append(schemaInfoArray[2]);
+for (int i = 0; i < tableMetricLabelList.size(); i++) {
+
labelBuilder.append(DELIMITER).append(tableMetricLabelList.get(i)).append("=").append(schemaInfoArray[i]);
}
MetricOption metricOption = MetricOption.builder()
.withInitRecords(subMetricState != null ?
subMetricState.getMetricValue(NUM_RECORDS_IN) : 0L)
@@ -135,32 +137,7 @@ public class SourceTableMetricData extends
SourceMetricData implements SourceSub
.withInlongLabels(labelBuilder.toString())
.withRegisterMetric(RegisteredMetric.ALL)
[GitHub] [inlong] leezng merged pull request #7008: [INLONG-7005][Dashboard] Support starrocks node management
leezng merged PR #7008: URL: https://github.com/apache/inlong/pull/7008 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[inlong] branch master updated: [INLONG-7005][Dashboard] Support starrocks node management (#7008)
This is an automated email from the ASF dual-hosted git repository.
leezng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new 9767a7c8d [INLONG-7005][Dashboard] Support starrocks node management
(#7008)
9767a7c8d is described below
commit 9767a7c8d5511b6985c58f860786047690238ce0
Author: Lizhen <88174078+bluew...@users.noreply.github.com>
AuthorDate: Wed Dec 21 15:57:54 2022 +0800
[INLONG-7005][Dashboard] Support starrocks node management (#7008)
---
inlong-dashboard/src/locales/cn.json | 4 ++
inlong-dashboard/src/locales/en.json | 4 ++
.../src/metas/nodes/defaults/StarRocks.ts | 67 ++
inlong-dashboard/src/metas/nodes/defaults/index.ts | 5 ++
.../src/metas/sinks/defaults/StarRocks.ts | 63
5 files changed, 104 insertions(+), 39 deletions(-)
diff --git a/inlong-dashboard/src/locales/cn.json
b/inlong-dashboard/src/locales/cn.json
index 89c6256c9..7f3dceedb 100644
--- a/inlong-dashboard/src/locales/cn.json
+++ b/inlong-dashboard/src/locales/cn.json
@@ -292,6 +292,7 @@
"meta.Sinks.HBase.IsMetaField": "是否为元字段",
"meta.Sinks.HBase.FieldFormat": "字段格式",
"meta.Sinks.HBase.FieldDescription": "字段描述",
+ "meta.Sinks.StarRocks.DataNodeName": "数据节点",
"meta.Sinks.StarRocks.TableName": "表名称",
"meta.Sinks.StarRocks.PrimaryKey": "主键",
"meta.Sinks.StarRocks.DatabaseName": "数据库名",
@@ -418,6 +419,9 @@
"meta.Nodes.ClickHouse.Username": "用户名",
"meta.Nodes.ClickHouse.Password": "密码",
"meta.Nodes.ClickHouse.Url": "地址",
+ "meta.Nodes.StarRocks.Username": "用户名",
+ "meta.Nodes.StarRocks.Password": "密码",
+ "meta.Nodes.StarRocks.Url": "地址",
"components.EditableTable.NewLine": "新增一行",
"components.FormGenerator.plugins.PleaseChoose": "请选择",
"components.FormGenerator.plugins.PleaseInput": "请输入",
diff --git a/inlong-dashboard/src/locales/en.json
b/inlong-dashboard/src/locales/en.json
index 73594640e..12e448f9f 100644
--- a/inlong-dashboard/src/locales/en.json
+++ b/inlong-dashboard/src/locales/en.json
@@ -292,6 +292,7 @@
"meta.Sinks.HBase.IsMetaField": "IsMetaField",
"meta.Sinks.HBase.FieldFormat": "FieldFormat",
"meta.Sinks.HBase.FieldDescription": "FieldDescription",
+ "meta.Sinks.StarRocks.DataNodeName": "DataNode",
"meta.Sinks.StarRocks.TableName": "TableName",
"meta.Sinks.StarRocks.PrimaryKey": "PrimaryKey",
"meta.Sinks.StarRocks.DatabaseName": "Database Name",
@@ -415,6 +416,9 @@
"meta.Nodes.Iceberg.Url": "URL",
"meta.Nodes.Iceberg.Warehouse": "Warehouse",
"meta.Nodes.Iceberg.CatalogType": "Catalog Type",
+ "meta.Nodes.StarRocks.Username": "Username",
+ "meta.Nodes.StarRocks.Password": "Password",
+ "meta.Nodes.StarRocks.Url": "URL",
"components.EditableTable.NewLine": "New line",
"components.FormGenerator.plugins.PleaseChoose": "Please select",
"components.FormGenerator.plugins.PleaseInput": "Please input",
diff --git a/inlong-dashboard/src/metas/nodes/defaults/StarRocks.ts
b/inlong-dashboard/src/metas/nodes/defaults/StarRocks.ts
new file mode 100644
index 0..0a49b5a51
--- /dev/null
+++ b/inlong-dashboard/src/metas/nodes/defaults/StarRocks.ts
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { DataWithBackend } from '@/metas/DataWithBackend';
+import { RenderRow } from '@/metas/RenderRow';
+import { RenderList } from '@/metas/RenderList';
+import { NodeInfo } from '../common/NodeInfo';
+
+const { I18n } = DataWithBackend;
+const { FieldDecorator } = RenderRow;
+
+export default class StarRocksNode
+ extends NodeInfo
+ implements DataWithBackend, RenderRow, RenderList
+{
+ @FieldDecorator({
+type: 'input',
+rules: [{ required: true }],
+ })
+ @I18n('meta.Nodes.StarRocks.Username')
+ username: string;
+
+ @FieldDecorator({
+type: 'password',
+rules: [{ required: true }],
+ })
+ @I18n('meta.Nodes.StarRocks.Password')
+ token: string;
+
+ @FieldDecorator({
+type: 'input',
+rules: [{ required: true }],
+props: values => ({
+ disabled: [110, 130].includes(values?.status),
+ placeholder: 'jdbc
