This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new dcd5e5d [AuditPlugin] Modify load label of audit plugin to avoid load
confliction (#3681)
dcd5e5d is described below
commit dcd5e5df124e9a3927a64f5940e3200e077fb98b
Author: Mingyu Chen <[email protected]>
AuthorDate: Tue May 26 18:23:07 2020 +0800
[AuditPlugin] Modify load label of audit plugin to avoid load confliction
(#3681)
Change the load label of audit plugin as:
`audit_yyyyMMdd_HHmmss_feIdentity`.
The `feIdentity` is got from the FE which run this plugin, currently just
use FE's IP_editlog_port.
---
be/src/olap/rowset/alpha_rowset.cpp | 3 ++-
.../main/java/org/apache/doris/plugin/PluginContext.java | 15 +++++++++++++++
.../auditloader/src/main/assembly/plugin.properties | 2 +-
.../org/apache/doris/plugin/audit/AuditLoaderPlugin.java | 5 ++++-
.../org/apache/doris/plugin/audit/DorisStreamLoader.java | 11 +++++++----
5 files changed, 29 insertions(+), 7 deletions(-)
diff --git a/be/src/olap/rowset/alpha_rowset.cpp
b/be/src/olap/rowset/alpha_rowset.cpp
index 3ec52d4..8292913 100644
--- a/be/src/olap/rowset/alpha_rowset.cpp
+++ b/be/src/olap/rowset/alpha_rowset.cpp
@@ -168,7 +168,8 @@ OLAPStatus AlphaRowset::split_range(
std::shared_ptr<SegmentGroup> largest_segment_group =
_segment_group_with_largest_size();
if (largest_segment_group == nullptr ||
largest_segment_group->current_num_rows_per_row_block() == 0) {
LOG(WARNING) << "failed to get largest_segment_group. is null: " <<
(largest_segment_group == nullptr)
- << ". version: " << start_version() << "-" << end_version();
+ << ". version: " << start_version() << "-" << end_version()
+ << ". tablet: " << rowset_meta()->tablet_id();
ranges->emplace_back(start_key.to_tuple());
ranges->emplace_back(end_key.to_tuple());
return OLAP_SUCCESS;
diff --git a/fe/src/main/java/org/apache/doris/plugin/PluginContext.java
b/fe/src/main/java/org/apache/doris/plugin/PluginContext.java
index 1ba6137..fb061aa 100644
--- a/fe/src/main/java/org/apache/doris/plugin/PluginContext.java
+++ b/fe/src/main/java/org/apache/doris/plugin/PluginContext.java
@@ -17,10 +17,20 @@
package org.apache.doris.plugin;
+import org.apache.doris.common.Config;
+import org.apache.doris.service.FrontendOptions;
+
public class PluginContext {
// the dir path where the plugin's files saved
private String pluginPath;
+ // the identity of the FE which load this plugin. Some plugins need to
know which FE is running with it.
+ // currently, we just use FE ip and editlog port
+ private String feIdentity;
+
+ public PluginContext() {
+ this.feIdentity = FrontendOptions.getLocalHostAddress() + "_" +
Config.edit_log_port;
+ }
protected void setPluginPath(String pluginPath) {
this.pluginPath = pluginPath;
@@ -29,4 +39,9 @@ public class PluginContext {
public String getPluginPath() {
return pluginPath;
}
+
+ public String getFeIdentity() {
+ return feIdentity;
+ }
}
+
diff --git a/fe_plugins/auditloader/src/main/assembly/plugin.properties
b/fe_plugins/auditloader/src/main/assembly/plugin.properties
index 1226f9b..3ab896c 100755
--- a/fe_plugins/auditloader/src/main/assembly/plugin.properties
+++ b/fe_plugins/auditloader/src/main/assembly/plugin.properties
@@ -18,6 +18,6 @@
name=AuditLoader
type=AUDIT
description=load audit log to olap load, and user can view the statistic of
queries
-version=0.12.0
+version=0.12.1
java.version=1.8.0
classname=org.apache.doris.plugin.audit.AuditLoaderPlugin
diff --git
a/fe_plugins/auditloader/src/main/java/org/apache/doris/plugin/audit/AuditLoaderPlugin.java
b/fe_plugins/auditloader/src/main/java/org/apache/doris/plugin/audit/AuditLoaderPlugin.java
index c6d0823..8ce6579 100755
---
a/fe_plugins/auditloader/src/main/java/org/apache/doris/plugin/audit/AuditLoaderPlugin.java
+++
b/fe_plugins/auditloader/src/main/java/org/apache/doris/plugin/audit/AuditLoaderPlugin.java
@@ -107,6 +107,7 @@ public class AuditLoaderPlugin extends Plugin implements
AuditPlugin {
conf = new AuditLoaderConf();
conf.init(properties);
+ conf.feIdentity = ctx.getFeIdentity();
}
@Override
@@ -191,11 +192,13 @@ public class AuditLoaderPlugin extends Plugin implements
AuditPlugin {
public long maxBatchSize = 50 * 1024 * 1024;
public long maxBatchIntervalSec = 60;
- public String frontendHostPort = "127.0.0.1:9030";
+ public String frontendHostPort = "127.0.0.1:8030";
public String user = "root";
public String password = "";
public String database = "doris_audit_db__";
public String table = "doris_audit_tbl__";
+ // the identity of FE which run this plugin
+ public String feIdentity = "";
public void init(Map<String, String> properties) throws
PluginException {
try {
diff --git
a/fe_plugins/auditloader/src/main/java/org/apache/doris/plugin/audit/DorisStreamLoader.java
b/fe_plugins/auditloader/src/main/java/org/apache/doris/plugin/audit/DorisStreamLoader.java
index 5148233..6cb8512 100644
---
a/fe_plugins/auditloader/src/main/java/org/apache/doris/plugin/audit/DorisStreamLoader.java
+++
b/fe_plugins/auditloader/src/main/java/org/apache/doris/plugin/audit/DorisStreamLoader.java
@@ -41,6 +41,7 @@ public class DorisStreamLoader {
private String passwd;
private String loadUrlStr;
private String authEncoding;
+ private String feIdentity;
public DorisStreamLoader(AuditLoaderPlugin.AuditLoaderConf conf) {
this.hostPort = conf.frontendHostPort;
@@ -51,6 +52,8 @@ public class DorisStreamLoader {
this.loadUrlStr = String.format(loadUrlPattern, hostPort, db, tbl);
this.authEncoding =
Base64.getEncoder().encodeToString(String.format("%s:%s", user,
passwd).getBytes(StandardCharsets.UTF_8));
+ // currently, FE identity is FE's IP, so we replace the "." in IP to
make it suitable for label
+ this.feIdentity = conf.feIdentity.replaceAll("\\.", "_");
}
public static void main(String[] args) {
@@ -98,10 +101,10 @@ public class DorisStreamLoader {
public LoadResponse loadBatch(StringBuilder sb) {
Calendar calendar = Calendar.getInstance();
- String label = String.format("audit_%s%02d%02d_%02d%02d%02d",
- calendar.get(Calendar.YEAR), calendar.get(Calendar.MONTH),
calendar.get(Calendar.DAY_OF_MONTH),
- calendar.get(Calendar.HOUR), calendar.get(Calendar.MINUTE),
calendar.get(Calendar.SECOND));
-
+ String label = String.format("audit_%s%02d%02d_%02d%02d%02d_%s",
+ calendar.get(Calendar.YEAR), calendar.get(Calendar.MONTH) + 1,
calendar.get(Calendar.DAY_OF_MONTH),
+ calendar.get(Calendar.HOUR_OF_DAY),
calendar.get(Calendar.MINUTE), calendar.get(Calendar.SECOND),
+ feIdentity);
HttpURLConnection feConn = null;
HttpURLConnection beConn = null;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
