This is an automated email from the ASF dual-hosted git repository. dockerzhang pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push: new 7012648891 [INLONG-10089][Audit] Adjust the service configuration of Audit (#10090) 7012648891 is described below commit 7012648891f55b4df46512dc138802135383124e Author: doleyzi <43397300+dole...@users.noreply.github.com> AuthorDate: Fri Apr 26 20:54:44 2024 +0800 [INLONG-10089][Audit] Adjust the service configuration of Audit (#10090) --- inlong-audit/conf/audit-proxy-kafka.conf | 8 ++-- inlong-audit/conf/audit-proxy-pulsar.conf | 8 ++-- inlong-audit/conf/audit-proxy-tubemq.conf | 8 ++-- .../apache_inlong_audit_clickhouse.sql | 5 +- ...aggregate.sql => apache_inlong_audit_mysql.sql} | 35 ++++++++++++-- inlong-audit/sql/apache_inlong_audit_starrocks.sql | 49 ++++++++++++++++++++ .../sql/audit-store/apache_inlong_audit.sql | 53 ---------------------- .../sql/audit-store/clickhouse-changes-1.12.0.sql | 24 ---------- .../sql/audit-store/clickhouse-changes-1.9.0.sql | 25 ---------- .../sql/audit-store/mysql-changes-1.12.0.sql | 24 ---------- .../sql/audit-store/mysql-changes-1.9.0.sql | 25 ---------- 11 files changed, 96 insertions(+), 168 deletions(-) diff --git a/inlong-audit/conf/audit-proxy-kafka.conf b/inlong-audit/conf/audit-proxy-kafka.conf index 8e700c31bf..8f5dec8d68 100644 --- a/inlong-audit/conf/audit-proxy-kafka.conf +++ b/inlong-audit/conf/audit-proxy-kafka.conf @@ -23,10 +23,10 @@ # in this case called 'agent' agent1.sources = tcp-source -agent1.channels = ch-msg1 -agent1.sinks = kafka-sink-msg1 +agent1.channels = ch-msg1 ch-msg2 +agent1.sinks = kafka-sink-msg1 pulsar-sink-msg2 -agent1.sources.tcp-source.channels = ch-msg1 +agent1.sources.tcp-source.channels = ch-msg1 ch-msg2 agent1.sources.tcp-source.type = org.apache.inlong.audit.source.SimpleTcpSource agent1.sources.tcp-source.msg-factory-name = org.apache.inlong.audit.source.ServerMessageFactory agent1.sources.tcp-source.host = 0.0.0.0 @@ -43,7 +43,7 @@ agent1.sources.tcp-source.metric-recovery-path=./data/recovery agent1.sources.tcp-source.set=10 agent1.channels.ch-msg1.type = memory -agent1.channels.ch-msg1.capacity = 10000 +agent1.channels.ch-msg1.capacity = 100000 agent1.channels.ch-msg1.keep-alive = 0 agent1.channels.ch-msg1.transactionCapacity = 200 diff --git a/inlong-audit/conf/audit-proxy-pulsar.conf b/inlong-audit/conf/audit-proxy-pulsar.conf index 633faff83b..040ce65662 100644 --- a/inlong-audit/conf/audit-proxy-pulsar.conf +++ b/inlong-audit/conf/audit-proxy-pulsar.conf @@ -22,10 +22,10 @@ # in this case called 'agent' agent1.sources = tcp-source -agent1.channels = ch-msg1 -agent1.sinks = pulsar-sink-msg1 +agent1.channels = ch-msg1 ch-msg2 +agent1.sinks = pulsar-sink-msg1 pulsar-sink-msg2 -agent1.sources.tcp-source.channels = ch-msg1 +agent1.sources.tcp-source.channels = ch-msg1 ch-msg2 agent1.sources.tcp-source.type = org.apache.inlong.audit.source.SimpleTcpSource agent1.sources.tcp-source.msg-factory-name = org.apache.inlong.audit.source.ServerMessageFactory agent1.sources.tcp-source.host = 0.0.0.0 @@ -44,7 +44,7 @@ agent1.sources.tcp-source.metric-cache-size=1000000 agent1.sources.tcp-source.set=10 agent1.channels.ch-msg1.type = memory -agent1.channels.ch-msg1.capacity = 10000 +agent1.channels.ch-msg1.capacity = 100000 agent1.channels.ch-msg1.keep-alive = 0 agent1.channels.ch-msg1.transactionCapacity = 200 diff --git a/inlong-audit/conf/audit-proxy-tubemq.conf b/inlong-audit/conf/audit-proxy-tubemq.conf index d4b9cc95f0..13ab207502 100644 --- a/inlong-audit/conf/audit-proxy-tubemq.conf +++ b/inlong-audit/conf/audit-proxy-tubemq.conf @@ -23,10 +23,10 @@ # in this case called 'agent' agent1.sources = tcp-source -agent1.channels = ch-msg1 -agent1.sinks = tube-sink-msg1 +agent1.channels = ch-msg1 ch-msg2 +agent1.sinks = tube-sink-msg1 pulsar-sink-msg2 -agent1.sources.tcp-source.channels = ch-msg1 +agent1.sources.tcp-source.channels = ch-msg1 ch-msg2 agent1.sources.tcp-source.type = org.apache.inlong.audit.source.SimpleTcpSource agent1.sources.tcp-source.msg-factory-name = org.apache.inlong.audit.source.ServerMessageFactory agent1.sources.tcp-source.message-handler-name = org.apache.inlong.audit.source.ServerMessageHandler @@ -46,7 +46,7 @@ agent1.sources.tcp-source.metric-recovery-path=./data/recovery agent1.sources.tcp-source.set=10 agent1.channels.ch-msg1.type = memory -agent1.channels.ch-msg1.capacity = 10000 +agent1.channels.ch-msg1.capacity = 100000 agent1.channels.ch-msg1.keep-alive = 0 agent1.channels.ch-msg1.transactionCapacity = 200 diff --git a/inlong-audit/sql/audit-store/apache_inlong_audit_clickhouse.sql b/inlong-audit/sql/apache_inlong_audit_clickhouse.sql similarity index 91% rename from inlong-audit/sql/audit-store/apache_inlong_audit_clickhouse.sql rename to inlong-audit/sql/apache_inlong_audit_clickhouse.sql index 936305c73b..7e96f5e8bc 100644 --- a/inlong-audit/sql/audit-store/apache_inlong_audit_clickhouse.sql +++ b/inlong-audit/sql/apache_inlong_audit_clickhouse.sql @@ -24,6 +24,7 @@ USE apache_inlong_audit; -- ---------------------------- -- Table structure for audit_data +-- The table creation statement of the audit flow table is used to record the real-time flow data of the audit. -- ---------------------------- CREATE TABLE IF NOT EXISTS `audit_data` ( @@ -42,5 +43,5 @@ CREATE TABLE IF NOT EXISTS `audit_data` `delay` Int64 COMMENT 'Message delay', `update_time` DateTime COMMENT 'Update time' ) ENGINE = MergeTree - ORDER BY inlong_group_id - SETTINGS index_granularity = 8192; +ORDER BY inlong_group_id +SETTINGS index_granularity = 8192; diff --git a/inlong-audit/sql/audit-service/apcache_inlong_audit_aggregate.sql b/inlong-audit/sql/apache_inlong_audit_mysql.sql similarity index 73% rename from inlong-audit/sql/audit-service/apcache_inlong_audit_aggregate.sql rename to inlong-audit/sql/apache_inlong_audit_mysql.sql index 110d7f884d..4914fd7854 100644 --- a/inlong-audit/sql/audit-service/apcache_inlong_audit_aggregate.sql +++ b/inlong-audit/sql/apache_inlong_audit_mysql.sql @@ -27,6 +27,32 @@ CREATE DATABASE IF NOT EXISTS apache_inlong_audit; USE apache_inlong_audit; +-- ---------------------------- +-- Table structure for audit_data +-- The table creation statement of the audit flow table is used to record the real-time flow data of the audit. +-- ---------------------------- +CREATE TABLE IF NOT EXISTS `audit_data` +( + `id` int(32) NOT NULL PRIMARY KEY AUTO_INCREMENT COMMENT 'Incremental primary key', + `ip` varchar(32) NOT NULL DEFAULT '' COMMENT 'Client IP', + `docker_id` varchar(100) NOT NULL DEFAULT '' COMMENT 'Client docker id', + `thread_id` varchar(50) NOT NULL DEFAULT '' COMMENT 'Client thread id', + `sdk_ts` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'SDK timestamp', + `packet_id` BIGINT NOT NULL DEFAULT '0' COMMENT 'Packet id', + `log_ts` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Log timestamp', + `inlong_group_id` varchar(100) NOT NULL DEFAULT '' COMMENT 'The target inlong group id', + `inlong_stream_id` varchar(100) NOT NULL DEFAULT '' COMMENT 'The target inlong stream id', + `audit_id` varchar(100) NOT NULL DEFAULT '' COMMENT 'Audit id', + `audit_tag` varchar(100) DEFAULT '' COMMENT 'Audit tag', + `audit_version` BIGINT DEFAULT -1 COMMENT 'Audit version', + `count` BIGINT NOT NULL DEFAULT '0' COMMENT 'Message count', + `size` BIGINT NOT NULL DEFAULT '0' COMMENT 'Message size', + `delay` BIGINT NOT NULL DEFAULT '0' COMMENT 'Message delay count', + `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Update time', + INDEX group_stream_audit_id (`inlong_group_id`, `inlong_stream_id`, `audit_id`, `log_ts`) +) ENGINE = InnoDB +DEFAULT CHARSET = UTF8 COMMENT ='Inlong audit data table'; + -- ---------------------------- -- Table structure for audit_data_temp -- You can create daily partitions or hourly partitions through the log_ts field. @@ -70,7 +96,8 @@ DEFAULT CHARSET = utf8 COMMENT ='Inlong audit data day table'; -- ---------------------------- -- Table structure for selector -- ---------------------------- -CREATE TABLE IF NOT EXISTS `leader_selector` ( +CREATE TABLE IF NOT EXISTS `leader_selector` +( `service_id` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL, `leader_id` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci NOT NULL, `last_seen_active` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, @@ -80,7 +107,8 @@ CREATE TABLE IF NOT EXISTS `leader_selector` ( -- ---------------------------- -- Table structure for audit id config -- ---------------------------- -CREATE TABLE IF NOT EXISTS `audit_id_config` ( +CREATE TABLE IF NOT EXISTS `audit_id_config` +( `audit_id` varchar(128) NOT NULL DEFAULT '' COMMENT 'Audit id', `status` int(11) DEFAULT '1' COMMENT 'Audit source config status. 0:Offline,1:Online', `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Update time', @@ -91,7 +119,8 @@ CREATE TABLE IF NOT EXISTS `audit_id_config` ( -- ---------------------------- -- Table structure for audit source config -- ---------------------------- -CREATE TABLE IF NOT EXISTS `audit_source_config` ( +CREATE TABLE IF NOT EXISTS `audit_source_config` +( `source_name` varchar(32) NOT NULL DEFAULT '' COMMENT 'Source_name', `jdbc_url` varchar(256) NOT NULL DEFAULT '' COMMENT 'Jdbc url', `jdbc_driver_class` varchar(128) NOT NULL DEFAULT '' COMMENT 'Jdbc driver class', diff --git a/inlong-audit/sql/apache_inlong_audit_starrocks.sql b/inlong-audit/sql/apache_inlong_audit_starrocks.sql new file mode 100644 index 0000000000..d05425f9ee --- /dev/null +++ b/inlong-audit/sql/apache_inlong_audit_starrocks.sql @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +-- ---------------------------- +-- Database for InLong Audit +-- ---------------------------- +CREATE DATABASE IF NOT EXISTS apache_inlong_audit; + +USE apache_inlong_audit; + +-- ---------------------------- +-- Table structure for audit_data +-- The table creation statement of the audit flow table is used to record the real-time flow data of the audit. +-- ---------------------------- +CREATE TABLE IF NOT EXISTS `audit_data` +( + `log_ts` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT "Log timestamp", + `audit_id` varchar(100) NOT NULL DEFAULT "" COMMENT "Audit id", + `audit_tag` varchar(100) NOT NULL DEFAULT "" COMMENT "Audit tag", + `inlong_group_id` varchar(100) NOT NULL DEFAULT "" COMMENT "The target inlong group id", + `inlong_stream_id` varchar(100) NOT NULL DEFAULT "" COMMENT "The target inlong stream id", + `audit_version` bigint(20) NOT NULL DEFAULT "-1" COMMENT "Audit version", + `ip` varchar(32) NOT NULL DEFAULT "" COMMENT "Client IP", + `docker_id` varchar(100) NOT NULL DEFAULT "" COMMENT "Client docker id", + `thread_id` varchar(50) NOT NULL DEFAULT "" COMMENT "Client thread id", + `sdk_ts` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT "SDK timestamp", + `packet_id` bigint(20) NOT NULL DEFAULT "0" COMMENT "Packet id", + `count` bigint(20) NOT NULL DEFAULT "0" COMMENT "Message count", + `size` bigint(20) NOT NULL DEFAULT "0" COMMENT "Message size", + `delay` bigint(20) NOT NULL DEFAULT "0" COMMENT "Message delay count", + `update_time` datetime NULL DEFAULT CURRENT_TIMESTAMP COMMENT "Update time" +) ENGINE=OLAP +PRIMARY KEY(`log_ts`,`audit_id`,`audit_tag`,`inlong_group_id`,`inlong_stream_id`,`audit_version`,`ip`,`docker_id`,`thread_id`,`sdk_ts`,`packet_id`) +COMMENT "Aduit data" +DISTRIBUTED BY HASH(`log_ts`,`audit_id`,`audit_tag`) BUCKETS 32; diff --git a/inlong-audit/sql/audit-store/apache_inlong_audit.sql b/inlong-audit/sql/audit-store/apache_inlong_audit.sql deleted file mode 100644 index a2b007534a..0000000000 --- a/inlong-audit/sql/audit-store/apache_inlong_audit.sql +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -SET NAMES utf8; -SET FOREIGN_KEY_CHECKS = 0; -SET SQL_MODE = "NO_AUTO_VALUE_ON_ZERO"; -SET time_zone = "+00:00"; - --- ---------------------------- --- Database for InLong Audit --- ---------------------------- -CREATE DATABASE IF NOT EXISTS apache_inlong_audit; - -USE apache_inlong_audit; - --- ---------------------------- --- Table structure for audit_data --- ---------------------------- -CREATE TABLE IF NOT EXISTS `audit_data` -( - `id` int(32) NOT NULL PRIMARY KEY AUTO_INCREMENT COMMENT 'Incremental primary key', - `ip` varchar(32) NOT NULL DEFAULT '' COMMENT 'Client IP', - `docker_id` varchar(100) NOT NULL DEFAULT '' COMMENT 'Client docker id', - `thread_id` varchar(50) NOT NULL DEFAULT '' COMMENT 'Client thread id', - `sdk_ts` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'SDK timestamp', - `packet_id` BIGINT NOT NULL DEFAULT '0' COMMENT 'Packet id', - `log_ts` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT 'Log timestamp', - `inlong_group_id` varchar(100) NOT NULL DEFAULT '' COMMENT 'The target inlong group id', - `inlong_stream_id` varchar(100) NOT NULL DEFAULT '' COMMENT 'The target inlong stream id', - `audit_id` varchar(100) NOT NULL DEFAULT '' COMMENT 'Audit id', - `audit_tag` varchar(100) DEFAULT '' COMMENT 'Audit tag', - `audit_version` BIGINT DEFAULT -1 COMMENT 'Audit version', - `count` BIGINT NOT NULL DEFAULT '0' COMMENT 'Message count', - `size` BIGINT NOT NULL DEFAULT '0' COMMENT 'Message size', - `delay` BIGINT NOT NULL DEFAULT '0' COMMENT 'Message delay count', - `update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT 'Update time', - INDEX group_stream_audit_id (`inlong_group_id`, `inlong_stream_id`, `audit_id`, `log_ts`) -) ENGINE = InnoDB - DEFAULT CHARSET = utf8 COMMENT ='Inlong audit data table'; diff --git a/inlong-audit/sql/audit-store/clickhouse-changes-1.12.0.sql b/inlong-audit/sql/audit-store/clickhouse-changes-1.12.0.sql deleted file mode 100644 index d6c0ee8bbd..0000000000 --- a/inlong-audit/sql/audit-store/clickhouse-changes-1.12.0.sql +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - --- When upgrading to version 1.12.0, please execute those SQLs in the clickhouse which audit used. - -USE `apache_inlong_audit`; - -ALTER TABLE audit_data ADD COLUMN audit_version Int64 DEFAULT -1 COMMENT 'Audit version' AFTER `audit_tag`; - - diff --git a/inlong-audit/sql/audit-store/clickhouse-changes-1.9.0.sql b/inlong-audit/sql/audit-store/clickhouse-changes-1.9.0.sql deleted file mode 100644 index 8bfe84d7fa..0000000000 --- a/inlong-audit/sql/audit-store/clickhouse-changes-1.9.0.sql +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - --- This is the SQL change file from version 1.8.0 to the current version 1.9.0. --- When upgrading to version 1.9.0, please execute those SQLs in the DB (such as MySQL) used by the Manager module. - -USE `apache_inlong_audit`; - -ALTER TABLE audit_data ADD COLUMN audit_tag String DEFAULT '' COMMENT 'Audit tag' after `audit_id`; - - diff --git a/inlong-audit/sql/audit-store/mysql-changes-1.12.0.sql b/inlong-audit/sql/audit-store/mysql-changes-1.12.0.sql deleted file mode 100644 index 4695bed02d..0000000000 --- a/inlong-audit/sql/audit-store/mysql-changes-1.12.0.sql +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - --- When upgrading to version 1.12.0, please execute those SQLs in the DB (such as MySQL) which audit used. - -USE `apache_inlong_audit`; - -ALTER TABLE audit_data ADD COLUMN audit_version BIGINT DEFAULT -1 COMMENT 'Audit version' AFTER `audit_tag`; - - diff --git a/inlong-audit/sql/audit-store/mysql-changes-1.9.0.sql b/inlong-audit/sql/audit-store/mysql-changes-1.9.0.sql deleted file mode 100644 index 29ade6c935..0000000000 --- a/inlong-audit/sql/audit-store/mysql-changes-1.9.0.sql +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - --- This is the SQL change file from version 1.8.0 to the current version 1.9.0. --- When upgrading to version 1.9.0, please execute those SQLs in the DB (such as MySQL) used by the Manager module. - -USE `apache_inlong_audit`; - -ALTER TABLE audit_data ADD COLUMN audit_tag varchar(100) DEFAULT '' COMMENT 'Audit tag' after `audit_id`; - -