This is an automated email from the ASF dual-hosted git repository.
dockerzhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong-website.git
The following commit(s) were added to refs/heads/master by this push:
new 823796d789 [INLONG-560][Doc] Update sort-standalone docs (#561)
823796d789 is described below
commit 823796d789d4e8a7183de62e9fc905e8fc938320
Author: vernedeng <deng...@pku.edu.cn>
AuthorDate: Mon Sep 19 11:18:04 2022 +0800
[INLONG-560][Doc] Update sort-standalone docs (#561)
---
.../sort-standalone/elasticsearch_example.md | 145 ++++--------
docs/modules/sort-standalone/hive_example.md | 262 +++++++++++++--------
docs/modules/sort-standalone/overview.md | 15 +-
docs/modules/sort-standalone/quick_start.md | 249 +++++++-------------
4 files changed, 301 insertions(+), 370 deletions(-)
diff --git a/docs/modules/sort-standalone/elasticsearch_example.md
b/docs/modules/sort-standalone/elasticsearch_example.md
index d48b78a4f5..2c47f4caf4 100644
--- a/docs/modules/sort-standalone/elasticsearch_example.md
+++ b/docs/modules/sort-standalone/elasticsearch_example.md
@@ -27,8 +27,8 @@
sortSource.type=org.apache.inlong.sort.standalone.source.sortsdk.SortSdkSource
sortClusterConfig.type=file
sortClusterConfig.file=SortClusterConfig.conf
sortSourceConfig.QueryConsumeConfigType=file
-#sortTaskId.conf
+# manager config example
#sortClusterConfig.type=manager
#sortSourceConfig.QueryConsumeConfigType=manager
#managerUrlLoaderType=org.apache.inlong.sort.standalone.config.loader.CommonPropertiesManagerUrlLoader
@@ -47,16 +47,15 @@ sortSourceConfig.QueryConsumeConfigType=file
"idParams": [{
"indexNamePattern": "inlong0fc00000046_{yyyyMMdd}",
"contentOffset": "0",
- "inlongGroupId": "atta",
- "fieldOffset": "2",
+ "inlongGroupId": "testgroup",
+ "fieldOffset": "0",
"fieldNames": "ftime extinfo t1 t2 t3 t4",
"inlongStreamId": "0fc00000046",
"separator": "|"
}],
"sinkParams": {
- "httpHosts": "11.187.135.221:9200",
- "password": "yingyan@ES",
- "auditSetName": "es-rmrv7g7a",
+ "httpHosts": "ip:port",
+ "password": "password",
"bulkSizeMb": "10",
"flushInterval": "60",
"keywordMaxLength": "32767",
@@ -78,11 +77,11 @@ sortSourceConfig.QueryConsumeConfigType=file
"sortTaskId": "sid_es_v3",
"cacheZones": {
"pc_atta6th_sz1": {
- "zoneName": "pc_atta6th_sz1",
- "serviceUrl": "http://9.139.53.86:8080";,
- "authentication":
"eyJrZXlJZCI6InB1bHNhci04MnhhN24zZWs1ZHciLCJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJwdWxzYXItODJ4YTduM2VrNWR3X2FkbWluIn0.D5H_j8UQk8KYWHw_mzq2HmR393SnbL5Gz7JYCANBPnI",
+ "zoneName": "${PULSAR_CLUSTER_NAME}",
+ "serviceUrl": "http://${PULSAR_IP}:${PULSAR_PORT}";,
+ "authentication": "${PULSAR_AUTH}",
"topics": [{
- "topic":
"pulsar-82xa7n3ek5dw/atta/atta_topic_1",
+ "topic": "${TENANT/NAMESPACE/TOPIC}",
"partitionCnt": 10,
"topicProperties": {}
}],
@@ -93,103 +92,59 @@ sortSourceConfig.QueryConsumeConfigType=file
}
```
-## Modify configuration file:conf/common.properties
-
-| Parameter | Required | DefaultValue |Remark |
-| ------------ | ------------ | ------------ | ------------ |
-|clusterId | Y | NA | inlong-sort-standalone cluster id |
-|nodeId | N | Local IP | Current node id |
-|metricDomains | N | Sort | domain name of metric |
-|metricDomains.Sort.domainListeners | N |
org.apache.inlong.sort.standalone.metrics.prometheus.PrometheusMetricListener |
class name list of metric listener, separated by space |
-|metricDomains.Sort.snapshotInterval | N | 60000 | interval snapshoting metric
data(millisecond) |
-|prometheusHttpPort | N | 8080 | HTTP server port of prometheus simple client |
-|sortChannel.type | N |
org.apache.inlong.sort.standalone.channel.BufferQueueChannel | Channel class
name |
-|sortSink.type | Y | NA | Sink class name |
-|sortSource.type | N |
org.apache.inlong.sort.standalone.source.sortsdk.SortSdkSource | Source class
name |
-|sortClusterConfig.type | N | manager | Loader source of cluster
configuration data: [file,manager,UserDefinedClassName]. |
-|sortClusterConfig.file | N | SortClusterConfig.conf | File name in class
resource when sortClusterConfig.type=file. |
-|sortClusterConfig.managerUrl | N | NA | The parameter is the cluster
configuration URL of InlongManager when sortClusterConfig.type=manager.
<br/>For example:http://${manager
ip:port}/api/inlong/manager/openapi/sort/standalone/getClusterConfig |
-|sortSourceConfig.QueryConsumeConfigType | N | manager | Loader source of
sort task configuration data: [file,manager,UserDefinedClassName]. <br/>Sort
task configuration file is ${sortTaskId}.conf in the class resource when
sortSourceConfig.QueryConsumeConfigType=file. |
-|sortSourceConfig.managerUrl | N | NA | The parameter is the sort task
configuration URL of InlongManager when sortClusterConfig.type=manager.
<br/>For example:http://${manager
ip:port}/api/inlong/manager/openapi/sort/standalone/getSortSource |
-
-## Modify configuration file: SortClusterConfig.conf
-
-- Get cluster configuration data from the file:SortClusterConfig.conf in
classpath, it can not support online updating.
-- Get cluster configuration data from InlongManager URL, it can support online
updating.
-
-| Parameter | Required |Type | DefaultValue |Remark |
-| ------------ | ------------ | ------------ | ------------ | ------------ |
-|clusterName | Y |String | NA | inlong-sort-standalone cluster id |
-|sortTasks | Y | JsonArray<SortTaskConfig> |NA | Sort task list |
-
-### Modify configuration: SortTaskConfig
-
-| Parameter | Required | DefaultValue |Remark |
-| ------------ | ------------ | ------------ | ------------ |
-|name | Y | NA | sort task name |
-|type | Y | NA | sort task type, for example:HIVE("hive"), TUBE("tube"),
KAFKA("kafka"), PULSAR("pulsar"), ELASTICSEARCH("elasticsearch"), UNKNOWN("n")
|
-|idParams | Y | NA | Inlong DataStream configuration |
-|sinkParams | Y | NA | Sort task parameters |
-
### Modify configuration: idParams of Elasticsearch sort task
-| Parameter | Required | DefaultValue |Remark |
-| ------------ | ------------ | ------------ | ------------ |
-|inlongGroupId | Y | NA | inlongGroupId |
-|inlongStreamId | Y | NA | inlongStreamId |
-|separator | Y | NA | separator of Inlong datastream in data source |
-|fieldNames | Y | NA | field name list of Elasticsearch index, separated
by space. |
-|indexNamePattern | Y | NA | index name pattern of Elasticsearch,date time
variable include {yyyyMMdd},{yyyyMMddHH},{yyyyMMddHHmm}. |
-|contentOffset | Y | NA | field index offset of source content |
-|fieldOffset | Y | NA | offset of Elasticsearch index field name list |
+| Parameter | Required | DefaultValue | Remark
|
+|------------------|-----------|----------------|--------------------------------------------------------------------------------------------------------|
+| inlongGroupId | Y | NA | inlongGroupId
|
+| inlongStreamId | Y | NA | inlongStreamId
|
+| separator | Y | NA | separator of Inlong
datastream in data source
|
+| fieldNames | Y | NA | field name list of
Elasticsearch index, separated by space.
|
+| indexNamePattern | Y | NA | index name pattern of
Elasticsearch,date time variable include
{yyyyMMdd},{yyyyMMddHH},{yyyyMMddHHmm}. |
+| contentOffset | Y | NA | field index offset of source
content |
+| fieldOffset | Y | NA | offset of Elasticsearch
index field name list |
### Modify configuration: sinkParams of Elasticsearch sort task
-| Parameter | Required | DefaultValue |Remark |
-| ------------ | ------------ | ------------ | ------------ |
-|httpHosts | Y | NA | Hosts of Elasticsearch |
-|username | Y | NA | Username of Elasticsearch |
-|password | Y | NA | Password of Elasticsearch |
-|isUseIndexId | N | false | Create index id or not |
-|bulkSizeMb | N | 10 | Max content size per bulk(MB) |
-|flushInterval | N | 60 | Max interval between flushing operation(Second)
|
-|keywordMaxLength | N | 32767 | Max keyword length(Byte) |
-|bulkAction | N | 4000 | Max index request per bulk |
-|maxConnect | N | 10 | Max opening HTTP connect |
-|concurrentRequests | N | 5 | Max concurrent requests per HTTP connect |
-
-## Modify configuration file: sid_es_v3.conf
-
-- The file name include sort task name plus the postfix ".conf".
-- Get the configuration data from the file in classpath, it can not support
online updating.
-- Get the configuration data from InlongManager URL, it can support online
updating.
+| Parameter | Required | DefaultValue | Remark
|
+|--------------------|-------------|----------------|-------------------------------------------------|
+| httpHosts | Y | NA | Hosts of Elasticsearch
|
+| username | Y | NA | Username of
Elasticsearch |
+| password | Y | NA | Password of
Elasticsearch |
+| isUseIndexId | N | false | Create index id or not
|
+| bulkSizeMb | N | 10 | Max content size per
bulk(MB) |
+| flushInterval | N | 60 | Max interval between
flushing operation(Second) |
+| keywordMaxLength | N | 32767 | Max keyword length(Byte)
|
+| bulkAction | N | 4000 | Max index request per
bulk |
+| maxConnect | N | 10 | Max opening HTTP connect
|
+| concurrentRequests | N | 5 | Max concurrent requests
per HTTP connect |
+
### Modify configuration: sid_es_v3.conf
+- The file name include sort task name plus the postfix ".conf".
-| Parameter | Required |Type | DefaultValue |Remark |
-| ------------ | ------------ | ------------ | ------------ | ------------ |
-|sortClusterName | Y |String | NA | inlong-sort-standalone cluster id |
-|sortTaskId | Y | String |NA | Sort task name |
-|cacheZones | Y | JsonObject<String, JsonObject> |NA | Cache cluster
list, Map<cacheClusterName, CacheCluster> |
+| Parameter | Required | Type |
DefaultValue | Remark |
+|-----------------|------------|--------------------------------------|---------------|---------------------------------------------------------------|
+| sortClusterName | Y | String | NA
| inlong-sort-standalone cluster id |
+| sortTaskId | Y | String | NA
| Sort task name |
+| cacheZones | Y | JsonObject<String, JsonObject> | NA
| Cache cluster list, Map<cacheClusterName, CacheCluster> |
### Modify configuration: CacheCluster
-| Parameter | Required |Type | DefaultValue |Remark |
-| ------------ | ------------ | ------------ | ------------ | ------------ |
-|zoneName | Y |String | NA | cache cluster name |
-|zoneType | Y | String |NA | [pulsar,tube,kafka] |
-|serviceUrl | Y | String |NA | Pulsar serviceUrl or Kafka broker list |
-|authentication | Y | String |NA | Pulsar authentication |
-|cacheZoneProperties | N | Map<String,String> |NA | Cache consumer
configuration |
-|topics | N | List<Topic> |NA | Topic list of Cache consumer |
+| Parameter | Required | Type | DefaultValue
| Remark |
+|---------------------|-----------|--------------------------|-----------------|----------------------------------------|
+| zoneName | Y | String | NA
| cache cluster name |
+| zoneType | Y | String | NA
| [pulsar,tube,kafka] |
+| serviceUrl | Y | String | NA
| Pulsar serviceUrl or Kafka broker list |
+| authentication | Y | String | NA
| Pulsar authentication |
+| cacheZoneProperties | N | Map<String,String> | NA
| Cache consumer configuration |
+| topics | N | List<Topic> | NA
| Topic list of Cache consumer |
### Modify configuration: Topic
-| Parameter | Required |Type | DefaultValue |Remark |
-| ------------ | ------------ | ------------ | ------------ | ------------ |
-|topic | Y |String | NA | cache topic name |
-|partitionCnt | Y | Integer |NA | cache topic partition count |
-|topicProperties | N | Map<String,String> |NA | Cache topic
configuration |
+| Parameter | Required | Type | DefaultValue |
Remark |
+|-----------------|-------------|--------------------------|----------------|-----------------------------|
+| topic | Y | String | NA |
cache topic name |
+| partitionCnt | Y | Integer | NA |
cache topic partition count |
+| topicProperties | N | Map<String,String> | NA |
Cache topic configuration |
-## Start inlong-sort-standalone application
-At last, execute the shell file "./bin/sort-start.sh" for starting
sort-standalone, you can check the log file "sort.log".
diff --git a/docs/modules/sort-standalone/hive_example.md
b/docs/modules/sort-standalone/hive_example.md
index 218f8face0..be6bc67be2 100644
--- a/docs/modules/sort-standalone/hive_example.md
+++ b/docs/modules/sort-standalone/hive_example.md
@@ -27,8 +27,8 @@
sortSource.type=org.apache.inlong.sort.standalone.source.sortsdk.SortSdkSource
sortClusterConfig.type=file
sortClusterConfig.file=SortClusterConfig.conf
sortSourceConfig.QueryConsumeConfigType=file
-#sortTaskId.conf
+# manager config example
#sortClusterConfig.type=manager
#sortSourceConfig.QueryConsumeConfigType=manager
#managerUrlLoaderType=org.apache.inlong.sort.standalone.config.loader.CommonPropertiesManagerUrlLoader
@@ -38,40 +38,101 @@ sortSourceConfig.QueryConsumeConfigType=file
```
### Example: conf/SortClusterConfig.conf
-
```
{
- "clusterName": "hivev3-sz-sz1",
- "sortTasks": [{
- "name": "sid_hive_inlong6th_v3",
- "type": "HIVE",
- "idParams": [{
- "inlongGroupId": "atta",
- "inlongStreamId": "0fc00000046",
- "separator": "|",
- "partitionIntervalMs": 3600000,
- "idRootPath":
"/user/hive/warehouse/t_inlong_v1_0fc00000046",
- "partitionSubPath": "/{yyyyMMdd}/{yyyyMMddHH}",
- "hiveTableName": "t_inlong_v1_0fc00000046",
- "partitionFieldName": "dt",
- "partitionFieldPattern": "yyyyMMddHH",
- "msgTimeFieldPattern": "yyyy-MM-dd HH:mm:ss",
- "maxPartitionOpenDelayHour": 8
- }],
- "sinkParams": {
- "hdfsPath": "hdfs://10.160.139.123:9000",
- "maxFileOpenDelayMinute": "5",
- "tokenOvertimeMinute": "60",
- "maxOutputFileSizeGb": "2",
- "hiveJdbcUrl": "jdbc:hive2://10.160.142.179:10000",
- "hiveDatabase": "default",
- "hiveUsername": "hive",
- "hivePassword": "hive"
- }
- }]
+ "data":{
+ "clusterName":"hivev3-sz-sz1",
+ "sortTasks":[
+ {
+ "idParams":[
+ {
+ "inlongGroupId":"0fc00000046",
+ "inlongStreamId":"",
+ "separator":"|",
+ "partitionIntervalMs":3600000,
+
"idRootPath":"/user/hive/warehouse/t_inlong_v1_0fc00000046",
+
"partitionSubPath":"/{yyyyMMdd}/{yyyyMMddHH}",
+
"hiveTableName":"t_inlong_v1_0fc00000046",
+ "partitionFieldName":"dt",
+
"partitionFieldPattern":"yyyyMMddHH",
+
"msgTimeFieldPattern":"yyyy-MM-dd HH:mm:ss",
+ "maxPartitionOpenDelayHour":8
+ },
+ {
+ "inlongGroupId":"03600000045",
+ "inlongStreamId":"",
+ "separator":"|",
+ "partitionIntervalMs":3600000,
+
"idRootPath":"/user/hive/warehouse/t_inlong_v1_03600000045",
+
"partitionSubPath":"/{yyyyMMdd}/{yyyyMMddHH}",
+
"hiveTableName":"t_inlong_v1_03600000045",
+ "partitionFieldName":"dt",
+
"partitionFieldPattern":"yyyyMMddHH",
+
"msgTimeFieldPattern":"yyyy-MM-dd HH:mm:ss",
+ "maxPartitionOpenDelayHour":8
+ },
+ {
+ "inlongGroupId":"05100054990",
+ "inlongStreamId":"",
+ "separator":"|",
+ "partitionIntervalMs":3600000,
+
"idRootPath":"/user/hive/warehouse/t_inlong_v1_05100054990",
+
"partitionSubPath":"/{yyyyMMdd}/{yyyyMMddHH}",
+
"hiveTableName":"t_inlong_v1_05100054990",
+ "partitionFieldName":"dt",
+
"partitionFieldPattern":"yyyyMMddHH",
+
"msgTimeFieldPattern":"yyyy-MM-dd HH:mm:ss",
+ "maxPartitionOpenDelayHour":8
+ },
+ {
+ "inlongGroupId":"09c00014434",
+ "inlongStreamId":"",
+ "separator":"|",
+ "partitionIntervalMs":3600000,
+
"idRootPath":"/user/hive/warehouse/t_inlong_v1_09c00014434",
+
"partitionSubPath":"/{yyyyMMdd}/{yyyyMMddHH}",
+
"hiveTableName":"t_inlong_v1_09c00014434",
+ "partitionFieldName":"dt",
+
"partitionFieldPattern":"yyyyMMddHH",
+
"msgTimeFieldPattern":"yyyy-MM-dd HH:mm:ss",
+ "maxPartitionOpenDelayHour":8
+ },
+ {
+ "inlongGroupId":"0c900035509",
+ "inlongStreamId":"",
+ "separator":"|",
+ "partitionIntervalMs":3600000,
+
"idRootPath":"/user/hive/warehouse/t_inlong_v1_0c900035509",
+
"partitionSubPath":"/{yyyyMMdd}/{yyyyMMddHH}",
+
"hiveTableName":"t_inlong_v1_0c900035509",
+ "partitionFieldName":"dt",
+
"partitionFieldPattern":"yyyyMMddHH",
+
"msgTimeFieldPattern":"yyyy-MM-dd HH:mm:ss",
+ "maxPartitionOpenDelayHour":8
+ }
+ ],
+ "name":"sid_hive_inlong6th_v3",
+ "sinkParams":{
+ "hdfsPath":"hdfs://127.0.0.1:9000",
+ "maxFileOpenDelayMinute":"5",
+ "tokenOvertimeMinute":"60",
+ "maxOutputFileSizeGb":"2",
+
"hiveJdbcUrl":"jdbc:hive2://127.0.0.2:10000",
+ "hiveDatabase":"default",
+ "hiveUsername":"hive",
+ "hivePassword":"hive"
+ },
+ "type":"HIVE"
+ }
+ ]
+ },
+ "errCode":0,
+ "md5":"md5",
+ "result":true
}
```
+
### Example: conf/sid_hive_inlong6th_v3.conf
```
@@ -97,103 +158,100 @@ sortSourceConfig.QueryConsumeConfigType=file
## Modify configuration file:conf/common.properties
-| Parameter | Required | DefaultValue |Remark |
-| ------------ | ------------ | ------------ | ------------ |
-|clusterId | Y | NA | inlong-sort-standalone cluster id |
-|nodeId | N | Local IP | Current node id |
-|metricDomains | N | Sort | domain name of metric |
-|metricDomains.Sort.domainListeners | N |
org.apache.inlong.sort.standalone.metrics.prometheus.PrometheusMetricListener |
class name list of metric listener, separated by space |
-|metricDomains.Sort.snapshotInterval | N | 60000 | interval snapshoting metric
data(millisecond) |
-|prometheusHttpPort | N | 8080 | HTTP server port of prometheus simple client |
-|sortChannel.type | N |
org.apache.inlong.sort.standalone.channel.BufferQueueChannel | Channel class
name |
-|sortSink.type | Y | NA | Sink class name |
-|sortSource.type | N |
org.apache.inlong.sort.standalone.source.sortsdk.SortSdkSource | Source class
name |
-|sortClusterConfig.type | N | manager | Loader source of cluster
configuration data: [file,manager,UserDefinedClassName]. |
-|sortClusterConfig.file | N | SortClusterConfig.conf | File name in class
resource when sortClusterConfig.type=file. |
-|sortClusterConfig.managerUrl | N | NA | The parameter is the cluster
configuration URL of InlongManager when sortClusterConfig.type=manager.
<br/>For example:http://${manager
ip:port}/api/inlong/manager/openapi/sort/standalone/getClusterConfig |
-|sortSourceConfig.QueryConsumeConfigType | N | manager | Loader source of
sort task configuration data: [file,manager,UserDefinedClassName]. <br/>Sort
task configuration file is ${sortTaskId}.conf in the class resource when
sortSourceConfig.QueryConsumeConfigType=file. |
-|sortSourceConfig.managerUrl | N | NA | The parameter is the sort task
configuration URL of InlongManager when sortClusterConfig.type=manager.
<br/>For example:http://${manager
ip:port}/api/inlong/manager/openapi/sort/standalone/getSortSource |
+| Parameter | Required | DefaultValue
| Remark
|
+|-----------------------------------------|------------|-------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| clusterId | Y | NA
|
inlong-sort-standalone cluster id
|
+| nodeId | N | Local IP
| Current node id
|
+| metricDomains | N | Sort
| domain name of metric
|
+| metricDomains.Sort.domainListeners | N |
org.apache.inlong.sort.standalone.metrics.prometheus.PrometheusMetricListener |
class name list of metric listener, separated by space
|
+| metricDomains.Sort.snapshotInterval | N | 60000
| interval snapshoting
metric data(millisecond)
|
+| prometheusHttpPort | N | 8080
| HTTP server port of
prometheus simple client
|
+| sortChannel.type | N |
org.apache.inlong.sort.standalone.channel.BufferQueueChannel |
Channel class name
|
+| sortSink.type | Y | NA
| Sink class name
|
+| sortSource.type | N |
org.apache.inlong.sort.standalone.source.sortsdk.SortSdkSource |
Source class name
|
+| sortClusterConfig.type | N | manager
| Loader source of
cluster configuration data: [file,manager,UserDefinedClassName].
|
+| sortClusterConfig.file | N |
SortClusterConfig.conf |
File name in class resource when sortClusterConfig.type=file.
|
+| sortClusterConfig.managerUrl | N | NA
| The parameter is the
cluster configuration URL of InlongManager when sortClusterConfig.type=manager.
<br/>For example:http://${manager
ip:port}/api/inlong/manager/openapi/sort/standalone/getClusterConfig |
+| sortSourceConfig.QueryConsumeConfigType | N | manager
| Loader source of sort
task configuration data: [file,manager,UserDefinedClassName]. <br/>Sort task
configuration file is ${sortTaskId}.conf in the class resource when
sortSourceConfig.QueryConsumeConfigType=file. |
+| sortSourceConfig.managerUrl | N | NA
| The parameter is the
sort task configuration URL of InlongManager when
sortClusterConfig.type=manager. <br/>For example:http://${manager
ip:port}/api/inlong/manager/openapi/sort/standalone/getSortSource |
## Modify configuration file: SortClusterConfig.conf
- Get cluster configuration data from the file:SortClusterConfig.conf in
classpath, it can not support online updating.
- Get cluster configuration data from InlongManager URL, it can support online
updating.
-| Parameter | Required |Type | DefaultValue |Remark |
-| ------------ | ------------ | ------------ | ------------ | ------------ |
-|clusterName | Y |String | NA | inlong-sort-standalone cluster id |
-|sortTasks | Y | JsonArray<SortTaskConfig> |NA | Sort task list |
+| Parameter | Required | Type | DefaultValue
| Remark |
+|-------------|------------|---------------------------------|----------------|-----------------------------------|
+| clusterName | Y | String | NA
| inlong-sort-standalone cluster id |
+| sortTasks | Y | JsonArray<SortTaskConfig> | NA
| Sort task list |
### Modify configuration: SortTaskConfig
-| Parameter | Required | DefaultValue |Remark |
-| ------------ | ------------ | ------------ | ------------ |
-|name | Y | NA | sort task name |
-|type | Y | NA | sort task type, for example:HIVE("hive"), TUBE("tube"),
KAFKA("kafka"), PULSAR("pulsar"), ElasticSearch("ElasticSearch"), UNKNOWN("n")
|
-|idParams | Y | NA | Inlong DataStream configuration |
-|sinkParams | Y | NA | Sort task parameters |
+| Parameter | Required | DefaultValue | Remark
|
+|-------------|-------------|---------------|----------------------------------------------------------------------------------------------------------------------------------------|
+| name | Y | NA | sort task name
|
+| type | Y | NA | sort task type, for
example:HIVE("hive"), TUBE("tube"), KAFKA("kafka"), PULSAR("pulsar"),
ElasticSearch("ElasticSearch"), UNKNOWN("n") |
+| idParams | Y | NA | Inlong DataStream configuration
|
+| sinkParams | Y | NA | Sort task parameters
|
### Modify configuration: idParams of Hive sort task
-| Parameter | Required | DefaultValue |Remark |
-| ------------ | ------------ | ------------ | ------------ |
-|inlongGroupId | Y | NA | inlongGroupId |
-|inlongStreamId | Y | NA | inlongStreamId |
-|separator | Y | NA | separator of Inlong datastream in data source |
-|partitionIntervalMs | N | 3600000 | partition interval(millisecond) |
-|idRootPath | Y | NA | HDFS root path of Inlong DataStream |
-|partitionSubPath | Y | NA | partition sub path of Inlong DataStream |
-|hiveTableName | Y | NA | Hive table name of Inlong DataStream |
-|partitionFieldName | N | dt | partition field name of Inlong DataStream |
-|partitionFieldPattern | Y | NA | Date format of partition field value,
the type have {yyyyMMdd},{yyyyMMddHH},{yyyyMMddHHmm} |
-|msgTimeFieldPattern | Y | NA | Date format of message generation time, it
support Java date format |
-|maxPartitionOpenDelayHour | N | 8 | Max delay hour of partition(hour) |
+| Parameter | Required | DefaultValue | Remark
|
+|---------------------------|------------|-----------------|--------------------------------------------------------------------------------------------|
+| inlongGroupId | Y | NA | inlongGroupId
|
+| inlongStreamId | Y | NA | inlongStreamId
|
+| separator | Y | NA | separator of
Inlong datastream in data source |
+| partitionIntervalMs | N | 3600000 | partition
interval(millisecond)
|
+| idRootPath | Y | NA | HDFS root path of
Inlong DataStream |
+| partitionSubPath | Y | NA | partition sub
path of Inlong DataStream |
+| hiveTableName | Y | NA | Hive table name
of Inlong DataStream |
+| partitionFieldName | N | dt | partition field
name of Inlong DataStream |
+| partitionFieldPattern | Y | NA | Date format of
partition field value, the type have {yyyyMMdd},{yyyyMMddHH},{yyyyMMddHHmm} |
+| msgTimeFieldPattern | Y | NA | Date format of
message generation time, it support Java date format |
+| maxPartitionOpenDelayHour | N | 8 | Max delay hour of
partition(hour) |
### Modify configuration: sinkParams of Hive sort task
-| Parameter | Required | DefaultValue |Remark |
-| ------------ | ------------ | ------------ | ------------ |
-|hdfsPath | Y | NA | NameNode URL of HDFS |
-|maxFileOpenDelayMinute | N | 5 | Max writing delay minutes of simple HDFS
file(minute) |
-|tokenOvertimeMinute | N | 60 | token overtime of Inlong Data
Stream(minute) |
-|maxOutputFileSizeGb | N | 2 | Max file size of simple HDFS file(GB) |
-|hiveJdbcUrl | Y | NA | JDBC URL of Hive |
-|hiveDatabase | Y | NA | Hive database |
-|hiveUsername | Y | NA | Hive username |
-|hivePassword | Y | NA | Hive password |
+| Parameter | Required | DefaultValue | Remark
|
+|------------------------|------------|-----------------|-------------------------------------------------------|
+| hdfsPath | Y | NA | NameNode URL of HDFS
|
+| maxFileOpenDelayMinute | N | 5 | Max writing delay
minutes of simple HDFS file(minute) |
+| tokenOvertimeMinute | N | 60 | token overtime of
Inlong Data Stream(minute) |
+| maxOutputFileSizeGb | N | 2 | Max file size of
simple HDFS file(GB) |
+| hiveJdbcUrl | Y | NA | JDBC URL of Hive
|
+| hiveDatabase | Y | NA | Hive database
|
+| hiveUsername | Y | NA | Hive username
|
+| hivePassword | Y | NA | Hive password
|
## Modify configuration file: sid_hive_inlong6th_v3.conf
-- The file name include sort task name plus the postfix ".conf".
-- Get the configuration data from the file in classpath, it can not support
online updating.
-- Get the configuration data from InlongManager URL, it can support online
updating.
+- The file name include sort task name plus the postfix ".conf".
### Modify configuration: sid_hive_inlong6th_v3.conf
-| Parameter | Required |Type | DefaultValue |Remark |
-| ------------ | ------------ | ------------ | ------------ | ------------ |
-|sortClusterName | Y |String | NA | inlong-sort-standalone cluster id |
-|sortTaskId | Y | String |NA | Sort task name |
-|cacheZones | Y | JsonObject<String, JsonObject> |NA | Cache cluster
list, Map<cacheClusterName, CacheCluster> |
+| Parameter | Required | Type |
DefaultValue | Remark |
+|-----------------|-------------|--------------------------------------|----------------|---------------------------------------------------------------|
+| sortClusterName | Y | String | NA
| inlong-sort-standalone cluster id |
+| sortTaskId | Y | String | NA
| Sort task name |
+| cacheZones | Y | JsonObject<String, JsonObject> | NA
| Cache cluster list, Map<cacheClusterName, CacheCluster> |
### Modify configuration: CacheCluster
-| Parameter | Required |Type | DefaultValue |Remark |
-| ------------ | ------------ | ------------ | ------------ | ------------ |
-|zoneName | Y |String | NA | cache cluster name |
-|zoneType | Y | String |NA | [pulsar,tube,kafka] |
-|serviceUrl | Y | String |NA | Pulsar serviceUrl or Kafka broker list |
-|authentication | Y | String |NA | Pulsar authentication |
-|cacheZoneProperties | N | Map<String,String> |NA | Cache consumer
configuration |
-|topics | N | List<Topic> |NA | Topic list of Cache consumer |
+| Parameter | Required | Type | DefaultValue
| Remark |
+|---------------------|-------------|--------------------------|----------------|----------------------------------------|
+| zoneName | Y | String | NA
| cache cluster name |
+| zoneType | Y | String | NA
| [pulsar,tube,kafka] |
+| serviceUrl | Y | String | NA
| Pulsar serviceUrl or Kafka broker list |
+| authentication | Y | String | NA
| Pulsar authentication |
+| cacheZoneProperties | N | Map<String,String> | NA
| Cache consumer configuration |
+| topics | N | List<Topic> | NA
| Topic list of Cache consumer |
### Modify configuration: Topic
-| Parameter | Required |Type | DefaultValue |Remark |
-| ------------ | ------------ | ------------ | ------------ | ------------ |
-|topic | Y |String | NA | cache topic name |
-|partitionCnt | Y | Integer |NA | cache topic partition count |
-|topicProperties | N | Map<String,String> |NA | Cache topic
configuration |
+| Parameter | Required | Type | DefaultValue |
Remark |
+|-----------------|------------|--------------------------|-----------------|-----------------------------|
+| topic | Y | String | NA |
cache topic name |
+| partitionCnt | Y | Integer | NA |
cache topic partition count |
+| topicProperties | N | Map<String,String> | NA |
Cache topic configuration |
+
-## Start inlong-sort-standalone application
-At last, execute the shell file "./bin/sort-start.sh" for starting
sort-standalone, you can check the log file "sort.log".
diff --git a/docs/modules/sort-standalone/overview.md
b/docs/modules/sort-standalone/overview.md
index ebbb9f5964..b3e0b0632c 100644
--- a/docs/modules/sort-standalone/overview.md
+++ b/docs/modules/sort-standalone/overview.md
@@ -3,7 +3,7 @@ title: Overview
sidebar_position: 1
---
-## Overview
+## Introduction
Inlong sort standalone is a module responsible for consuming the data stream
reported by users from the cache layer and distributing it to different data
stores. It supports hive, elasticsearch, CLS and other data stores.
Inlong sort standalone relies on inlong manager to manage system metadata.
Inlong sort standalone is deployed by cluster and aggregates and distributes
tasks by target storage.
@@ -23,20 +23,21 @@ After the inlong data stream is distributed offline, it
goes online again. If th
If the consumption location at the time of offline is no longer within the
life cycle of the cache layer, consumption starts from the latest location of
the cache layer.
### message queues supported by the cache layer
-- Inlong-tubemq
+- Inlong-Tube
- Apache Pulsar
+- Apache Kafka
### supported data storage
-- Apache hive (currently only supports sequence file format)
+- Apache Hive (currently only supports sequence file format)
- Apache Pulsar
- Apache Kafka
+- Elasticsearch
+- ClickHouse
+- Tencent CLS
### Future planning
-#### support more types of cache layer message queues
-Apache Kafka, etc.
-
#### support more kinds of data storage
-HBase, elasticsearch, etc.
+HBase, etc.
#### support more file formats written to Apache hive
Orc file, etc.
diff --git a/docs/modules/sort-standalone/quick_start.md
b/docs/modules/sort-standalone/quick_start.md
index f72dbb9c71..482c63bf72 100644
--- a/docs/modules/sort-standalone/quick_start.md
+++ b/docs/modules/sort-standalone/quick_start.md
@@ -6,185 +6,102 @@ sidebar_position: 2
Module archive is in the
directory:inlong-sort-standalone/sort-standalone-dist/target/, the archive file
is apache-inlong-sort-standalone-${project.version}-bin.tar.gz.
## Start inlong-sort-standalone application
-At first, decompress the archive file, execute the shell file
"./bin/sort-start.sh".
+- Decompress the archive file;
+- Prepare conf/common.properties (and other config files if choosing local
file config mode);
+- Start SortStandalone ```./bin/sort-start.sh```
+- You can check the log file "sort.log".
## Configuration file:conf/common.properties
-| Parameter | Required | DefaultValue |Remark |
-| ------------ | ------------ | ------------ | ------------ |
-|clusterId | Y | NA | inlong-sort-standalone cluster id |
-|sortSource.type | N |
org.apache.inlong.sort.standalone.source.readapi.ReadApiSource | Source class
name |
-|sortChannel.type | N |
org.apache.inlong.sort.standalone.channel.BufferQueueChannel | Channel class
name |
-|sortSink.type | N | org.apache.inlong.sort.standalone.sink.hive.HiveSink |
Sink class name |
-|sortClusterConfig.type | N |
org.apache.inlong.sort.standalone.config.loader.ClassResourceSortClusterConfigLoader
| Configuration data loader class name |
-|sortClusterConfig.managerPath | N | NA | For
loader:org.apache.inlong.sort.standalone.config.loader.ManagerSortClusterConfigLoader,
the parameter is the URL of InlongManager. For example:http://${manager
ip:port}/api/inlong/manager/openapi/sort/standalone/getClusterConfig |
-|eventFormatHandler | N |
org.apache.inlong.sort.standalone.sink.hive.DefaultEventFormatHandler |
Formater class name |
-|maxThreads | N | 10 | sink thread number |
-|reloadInterval | N | 60000 | interval updating Configuration
data(millisecond) |
-|processInterval | N | 100 | interval processing data(millisecond) |
-|metricDomains | N | Sort | domain name of metric |
-|metricDomains.Sort.domainListeners | N |
org.apache.inlong.sort.standalone.metrics.prometheus.PrometheusMetricListener |
class name list of metric listener, separated by space |
-|prometheusHttpPort | N | 8080 | HTTP server port of prometheus simple client |
-|metricDomains.Sort.snapshotInterval | N | 60000 | interval snapshoting metric
data(millisecond) |
+| Parameter | Required | DefaultValue
| Remark
|
+|-----------------------------------------|----------|----------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------|
+| clusterId | Y | NA
|
inlong-sort-standalone cluster id
|
+| sortSource.type | N |
org.apache.inlong.sort.standalone.source.readapi.SortSdkSource
| Source class name
|
+| sortChannel.type | N |
org.apache.inlong.sort.standalone.channel.BufferQueueChannel
| Channel class name
|
+| sortSink.type | N |
org.apache.inlong.sort.standalone.sink.hive.HiveSink
| Sink class name
|
+| sortClusterConfig.type | Y | manager
| ClusterConfig
configuration type. File config mode please choose: ***file*** |
+| sortSourceConfig.QueryConsumeConfigType | Y | manager
| SourceConfig
configuration type. File config mode please choose: ***file*** |
+| sortClusterConfig.file | N | SortClusterConfig.conf
| Sort cluster config
file name. ***Required in file config type.*** |
+| managerUrlLoaderType | N |
org.apache.inlong.sort.standalone.config.loader.CommonPropertiesManagerUrlLoader
| The type of manager url loader. ***Required in manager config type.***
|
+| sortClusterConfig.managerUrl | N |
http://${manager_ip}:{manager_port}/inlong/manager/openapi/sort/getClusterConfig
| The manager config interface to cluster config. ***Required in manager
config type.*** |
+ | sortSourceConfig.managerUrl | N |
http://${manager_ip}:{manager_port}/inlong/manager/openapi/sort/getSortSource
| The manager config interface to sort source config. ***Required in manager
config mode.*** |
+| eventFormatHandler | N |
org.apache.inlong.sort.standalone.sink.hive.DefaultEventFormatHandler
| Formatter class name
|
+| maxThreads | N | 10
| sink thread number
|
+| reloadInterval | N | 60000
| interval updating
Configuration data(millisecond) |
+| processInterval | N | 100
| interval processing
data(millisecond) |
+| metricDomains | N | Sort
| domain name of
metric |
+| metricDomains.Sort.domainListeners | N |
org.apache.inlong.sort.standalone.metrics.prometheus.PrometheusMetricListener
| class name list of metric listener, separated by space
|
+| prometheusHttpPort | N | 8080
| HTTP server port of
prometheus simple client |
+| metricDomains.Sort.snapshotInterval | N | 60000
| interval snapshoting
metric data(millisecond) |
## SortClusterConfig
-- Get SortClusterConfig from the file:SortClusterConfig.conf in classpath, but
it can not support online updating.
-- Get SortClusterConfig from InlongManager URL, but it can support online
updating.
+The SortClusterConfig can be acquired from local file or manager, depending on
the ***sortClusterConfig.type*** configured in common.properties.
-| Parameter | Required | DefaultValue |Remark |
-| ------------ | ------------ | ------------ | ------------ |
-|clusterName | Y | NA | inlong-sort-standalone cluster id |
-|sortTasks | Y | NA | Sort task list |
+| Parameter | Required | DefaultValue | Remark
|
+|-------------|-------------|----------------|-----------------------------------|
+| clusterName | Y | NA | inlong-sort-standalone cluster
id |
+| sortTasks | Y | NA | Sort task list
|
+
+### SortTask
+Sort Task includes ***idParams and sinkParams*** which represent the config of
***stream and dataNode*** respectively
+
+The idParams and sinkParams are totally different among each type of sort
tasks. There are two examples of hive task and pulsar task
+
+| Parameter | Required | DefaultValue | Remark
|
+|------------|------------|----------------|----------------------------------------------------------------------------------------------------------------------------------------|
+| name | Y | NA | sort task name
|
+| type | Y | NA | sort task type, for
example:HIVE("hive"), TUBE("tube"), KAFKA("kafka"), PULSAR("pulsar"),
ElasticSearch("ElasticSearch"), UNKNOWN("n") |
+| idParams | Y | NA | List of Inlong DataStream
configuration
|
+| sinkParams | Y | NA | Sort task parameters
|
-### SortTaskConfig
-| Parameter | Required | DefaultValue |Remark |
-| ------------ | ------------ | ------------ | ------------ |
-|name | Y | NA | sort task name |
-|type | Y | NA | sort task type, for example:HIVE("hive"), TUBE("tube"),
KAFKA("kafka"), PULSAR("pulsar"), ElasticSearch("ElasticSearch"), UNKNOWN("n")
|
-|idParams | Y | NA | Inlong DataStream configuration |
-|sinkParams | Y | NA | Sort task parameters |
### idParams content of Hive sort task
-| Parameter | Required | DefaultValue |Remark |
-| ------------ | ------------ | ------------ | ------------ |
-|inlongGroupId | Y | NA | inlongGroupId |
-|inlongStreamId | Y | NA | inlongStreamId |
-|separator | Y | NA | separtor |
-|partitionIntervalMs | N | 3600000 | partition interval(millisecond) |
-|idRootPath | Y | NA | HDFS root path of Inlong DataStream |
-|partitionSubPath | Y | NA | partition sub path of Inlong DataStream |
-|hiveTableName | Y | NA | Hive table name of Inlong DataStream |
-|partitionFieldName | N | dt | partition field name of Inlong DataStream |
-|partitionFieldPattern | Y | NA | Date format of partition field value,
the type have {yyyyMMdd},{yyyyMMddHH},{yyyyMMddHHmm} |
-|msgTimeFieldPattern | Y | NA | Date format of message generation time, it
support Java date format |
-|maxPartitionOpenDelayHour | N | 8 | Max delay hour of partition(hour) |
+| Parameter | Required | DefaultValue | Remark
|
+|---------------------------|------------|---------------|--------------------------------------------------------------------------------------------|
+| inlongGroupId | Y | NA | inlongGroupId
|
+| inlongStreamId | Y | NA | inlongStreamId
|
+| separator | Y | NA | separator
|
+| partitionIntervalMs | N | 3600000 | partition
interval(millisecond)
|
+| idRootPath | Y | NA | HDFS root path of
Inlong DataStream |
+| partitionSubPath | Y | NA | partition sub path
of Inlong DataStream |
+| hiveTableName | Y | NA | Hive table name of
Inlong DataStream |
+| partitionFieldName | N | dt | partition field
name of Inlong DataStream |
+| partitionFieldPattern | Y | NA | Date format of
partition field value, the type have {yyyyMMdd},{yyyyMMddHH},{yyyyMMddHHmm} |
+| msgTimeFieldPattern | Y | NA | Date format of
message generation time, it support Java date format |
+| maxPartitionOpenDelayHour | N | 8 | Max delay hour of
partition(hour) |
### sinkParams content of Hive sort task
-| Parameter | Required | DefaultValue |Remark |
-| ------------ | ------------ | ------------ | ------------ |
-|hdfsPath | Y | NA | NameNode URL of HDFS |
-|maxFileOpenDelayMinute | N | 5 | Max writing delay minute of simple HDFS
file(minute) |
-|tokenOvertimeMinute | N | 60 | token overtime of Inlong Data
Stream(minute) |
-|maxOutputFileSizeGb | N | 2 | Max file size of simple HDFS file(GB) |
-|hiveJdbcUrl | Y | NA | JDBC URL of Hive |
-|hiveDatabase | Y | NA | Hive database |
-|hiveUsername | Y | NA | Hive username |
-|hivePassword | Y | NA | Hive password |
+| Parameter | Required | DefaultValue | Remark
|
+|------------------------|------------|-----------------|------------------------------------------------------|
+| hdfsPath | Y | NA | NameNode URL of HDFS
|
+| maxFileOpenDelayMinute | N | 5 | Max writing delay
minute of simple HDFS file(minute) |
+| tokenOvertimeMinute | N | 60 | token overtime of
Inlong Data Stream(minute) |
+| maxOutputFileSizeGb | N | 2 | Max file size of
simple HDFS file(GB) |
+| hiveJdbcUrl | Y | NA | JDBC URL of Hive
|
+| hiveDatabase | Y | NA | Hive database
|
+| hiveUsername | Y | NA | Hive username
|
+| hivePassword | Y | NA | Hive password
|
### idParams content of Pulsar sort task
-| Parameter | Required | DefaultValue |Remark |
-| ------------ | ------------ | ------------ | ------------ |
-|inlongGroupId | Y | NA | inlongGroupId |
-|inlongStreamId | Y | NA | inlongStreamId |
-|topic | Y | NA | Pulsarçš„Topic |
+| Parameter | Required | DefaultValue | Remark |
+|----------------|------------|-----------------|----------------|
+| inlongGroupId | Y | NA | inlongGroupId |
+| inlongStreamId | Y | NA | inlongStreamId |
+| topic | Y | NA | Topic of MQ |
### sinkParams content of Pulsar sort task
-| Parameter | Required | DefaultValue |Remark |
-| ------------ | ------------ | ------------ | ------------ |
-|serviceUrl | Y | NA | Pulsar service URL |
-|authentication | Y | NA | Pulsar authentication |
-|enableBatching | N | true | enableBatching |
-|batchingMaxBytes | N | 5242880 | batchingMaxBytes |
-|batchingMaxMessages | N | 3000 | batchingMaxMessages |
-|batchingMaxPublishDelay | N | 1 | batchingMaxPublishDelay |
-|maxPendingMessages | N | 1000 | maxPendingMessages |
-|maxPendingMessagesAcrossPartitions | N | 50000 |
maxPendingMessagesAcrossPartitions |
-|sendTimeout | N | 0 | sendTimeout |
-|compressionType | N | NONE | compressionType |
-|blockIfQueueFull | N | true | blockIfQueueFull |
-|roundRobinRouterBatchingPartitionSwitchFrequency | N | 10 |
roundRobinRouterBatchingPartitionSwitchFrequency |
+| Parameter | Required |
DefaultValue | Remark |
+|--------------------------------------------------|-------------|----------------|--------------------------------------------------|
+| serviceUrl | Y | NA
| Pulsar service URL |
+| authentication | Y | NA
| Pulsar authentication |
+| enableBatching | N | true
| enableBatching |
+| batchingMaxBytes | N | 5242880
| batchingMaxBytes |
+| batchingMaxMessages | N | 3000
| batchingMaxMessages |
+| batchingMaxPublishDelay | N | 1
| batchingMaxPublishDelay |
+| maxPendingMessages | N | 1000
| maxPendingMessages |
+| maxPendingMessagesAcrossPartitions | N | 50000
| maxPendingMessagesAcrossPartitions |
+| sendTimeout | N | 0
| sendTimeout |
+| compressionType | N | NONE
| compressionType |
+| blockIfQueueFull | N | true
| blockIfQueueFull |
+| roundRobinRouterBatchingPartitionSwitchFrequency | N | 10
| roundRobinRouterBatchingPartitionSwitchFrequency |
-### Sample of Hive sort task
-```
-{
- "data":{
- "clusterName":"hivev3-sz-sz1",
- "sortTasks":[
- {
- "idParams":[
- {
- "inlongGroupId":"0fc00000046",
- "inlongStreamId":"",
- "separator":"|",
- "partitionIntervalMs":3600000,
-
"idRootPath":"/user/hive/warehouse/t_inlong_v1_0fc00000046",
-
"partitionSubPath":"/{yyyyMMdd}/{yyyyMMddHH}",
-
"hiveTableName":"t_inlong_v1_0fc00000046",
- "partitionFieldName":"dt",
-
"partitionFieldPattern":"yyyyMMddHH",
-
"msgTimeFieldPattern":"yyyy-MM-dd HH:mm:ss",
- "maxPartitionOpenDelayHour":8
- },
- {
- "inlongGroupId":"03600000045",
- "inlongStreamId":"",
- "separator":"|",
- "partitionIntervalMs":3600000,
-
"idRootPath":"/user/hive/warehouse/t_inlong_v1_03600000045",
-
"partitionSubPath":"/{yyyyMMdd}/{yyyyMMddHH}",
-
"hiveTableName":"t_inlong_v1_03600000045",
- "partitionFieldName":"dt",
-
"partitionFieldPattern":"yyyyMMddHH",
-
"msgTimeFieldPattern":"yyyy-MM-dd HH:mm:ss",
- "maxPartitionOpenDelayHour":8
- },
- {
- "inlongGroupId":"05100054990",
- "inlongStreamId":"",
- "separator":"|",
- "partitionIntervalMs":3600000,
-
"idRootPath":"/user/hive/warehouse/t_inlong_v1_05100054990",
-
"partitionSubPath":"/{yyyyMMdd}/{yyyyMMddHH}",
-
"hiveTableName":"t_inlong_v1_05100054990",
- "partitionFieldName":"dt",
-
"partitionFieldPattern":"yyyyMMddHH",
-
"msgTimeFieldPattern":"yyyy-MM-dd HH:mm:ss",
- "maxPartitionOpenDelayHour":8
- },
- {
- "inlongGroupId":"09c00014434",
- "inlongStreamId":"",
- "separator":"|",
- "partitionIntervalMs":3600000,
-
"idRootPath":"/user/hive/warehouse/t_inlong_v1_09c00014434",
-
"partitionSubPath":"/{yyyyMMdd}/{yyyyMMddHH}",
-
"hiveTableName":"t_inlong_v1_09c00014434",
- "partitionFieldName":"dt",
-
"partitionFieldPattern":"yyyyMMddHH",
-
"msgTimeFieldPattern":"yyyy-MM-dd HH:mm:ss",
- "maxPartitionOpenDelayHour":8
- },
- {
- "inlongGroupId":"0c900035509",
- "inlongStreamId":"",
- "separator":"|",
- "partitionIntervalMs":3600000,
-
"idRootPath":"/user/hive/warehouse/t_inlong_v1_0c900035509",
-
"partitionSubPath":"/{yyyyMMdd}/{yyyyMMddHH}",
-
"hiveTableName":"t_inlong_v1_0c900035509",
- "partitionFieldName":"dt",
-
"partitionFieldPattern":"yyyyMMddHH",
-
"msgTimeFieldPattern":"yyyy-MM-dd HH:mm:ss",
- "maxPartitionOpenDelayHour":8
- }
- ],
- "name":"sid_hive_inlong6th_v3",
- "sinkParams":{
- "hdfsPath":"hdfs://127.0.0.1:9000",
- "maxFileOpenDelayMinute":"5",
- "tokenOvertimeMinute":"60",
- "maxOutputFileSizeGb":"2",
-
"hiveJdbcUrl":"jdbc:hive2://127.0.0.2:10000",
- "hiveDatabase":"default",
- "hiveUsername":"hive",
- "hivePassword":"hive"
- },
- "type":"HIVE"
- }
- ]
- },
- "errCode":0,
- "md5":"md5",
- "result":true
-}
-```
