This is an automated email from the ASF dual-hosted git repository.
yangzhg pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new 7e77b5e [Optimize] Using custom conf dir to save log config of Spring
(#6205)
7e77b5e is described below
commit 7e77b5ed7f29bf26e66509ba16342a6fa2d98341
Author: Mingyu Chen <morningman....@gmail.com>
AuthorDate: Thu Jul 15 11:13:51 2021 +0800
[Optimize] Using custom conf dir to save log config of Spring (#6205)
The log4j-config.xml will be generated at startup of FE and also when
modifying FE config.
But in some deploy environment such as k8s, the conf dir is not writable.
So change the dir of log4j-config.xml to Config.custom_conf_dir.
Also fix some small bugs:
1. Typo "less then" -> "less than"
2. Duplicated `exec_mem_limit` showed in SHOW ROUTINE LOAD
3. Allow MAXVALUE in single partition column table.
4. Add IP info for "intolerate index channel failure" msg.
Change-Id: Ib4e1182084219c41eae44d3a28110c0315fdbd7d
Co-authored-by: chenmingyu <chenmin...@baidu.com>
---
be/src/exec/tablet_sink.cpp | 3 +++
.../src/main/java/org/apache/doris/alter/RollupJobV2.java | 4 ++--
.../java/org/apache/doris/alter/SchemaChangeJobV2.java | 4 ++--
.../apache/doris/analysis/CreateMaterializedViewStmt.java | 2 +-
.../main/java/org/apache/doris/analysis/InsertStmt.java | 10 +++++++++-
.../java/org/apache/doris/analysis/PartitionKeyDesc.java | 14 ++++++++------
.../src/main/java/org/apache/doris/catalog/Database.java | 2 +-
.../src/main/java/org/apache/doris/catalog/OlapTable.java | 2 +-
.../src/main/java/org/apache/doris/common/Log4jConfig.java | 5 ++++-
.../src/main/java/org/apache/doris/httpv2/HttpServer.java | 3 ++-
.../org/apache/doris/httpv2/config/SpringLog4j2Config.java | 2 +-
.../java/org/apache/doris/load/loadv2/LoadLoadingTask.java | 2 +-
.../org/apache/doris/load/routineload/RoutineLoadJob.java | 7 +------
.../doris/load/routineload/RoutineLoadScheduler.java | 2 +-
.../java/org/apache/doris/planner/AssertNumRowsNode.java | 2 +-
.../main/java/org/apache/doris/qe/ConnectProcessor.java | 11 +++++++++--
.../doris/transaction/TabletQuorumFailedException.java | 2 +-
.../test/java/org/apache/doris/utframe/UtFrameUtils.java | 5 +++++
18 files changed, 53 insertions(+), 29 deletions(-)
diff --git a/be/src/exec/tablet_sink.cpp b/be/src/exec/tablet_sink.cpp
index d7d24d1..e978f99 100644
--- a/be/src/exec/tablet_sink.cpp
+++ b/be/src/exec/tablet_sink.cpp
@@ -26,6 +26,7 @@
#include "runtime/row_batch.h"
#include "runtime/runtime_state.h"
#include "runtime/tuple_row.h"
+#include "service/backend_options.h"
#include "service/brpc.h"
#include "util/brpc_stub_cache.h"
#include "util/debug/sanitizer_scopes.h"
@@ -473,6 +474,8 @@ Status IndexChannel::add_row(Tuple* tuple, int64_t
tablet_id) {
}
if (has_intolerable_failure()) {
+ std::stringstream ss;
+ ss << "index channel has intolerable failure. " <<
BackendOptions::get_localhost();
return Status::InternalError(ss.str());
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java
b/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java
index 470d4c5..dbd0204 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java
@@ -751,8 +751,8 @@ public class RollupJobV2 extends AlterJobV2 implements
GsonPostProcessable {
}
/**
- * This method is only used to deserialize the text mate which version is
less then 86.
- * If the meta version >=86, it will be deserialized by the `read` of
AlterJobV2 rather then here.
+ * This method is only used to deserialize the text mate which version is
less than 86.
+ * If the meta version >=86, it will be deserialized by the `read` of
AlterJobV2 rather than here.
*/
public static RollupJobV2 read(DataInput in) throws IOException {
Preconditions.checkState(Catalog.getCurrentCatalogJournalVersion() <
FeMetaVersion.VERSION_86);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
index bd22ba2..3292d86 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
@@ -979,8 +979,8 @@ public class SchemaChangeJobV2 extends AlterJobV2 {
}
/**
- * This method is only used to deserialize the text mate which version is
less then 86.
- * If the meta version >=86, it will be deserialized by the `read` of
AlterJobV2 rather then here.
+ * This method is only used to deserialize the text mate which version is
less than 86.
+ * If the meta version >=86, it will be deserialized by the `read` of
AlterJobV2 rather than here.
*/
public static SchemaChangeJobV2 read(DataInput in) throws IOException {
Preconditions.checkState(Catalog.getCurrentCatalogJournalVersion() <
FeMetaVersion.VERSION_86);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java
index c5607bd..f29daba 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java
@@ -245,7 +245,7 @@ public class CreateMaterializedViewStmt extends DdlStmt {
List<OrderByElement> orderByElements = selectStmt.getOrderByElements();
if (orderByElements.size() > mvColumnItemList.size()) {
- throw new AnalysisException("The number of columns in order clause
must be less then " + "the number of "
+ throw new AnalysisException("The number of columns in order clause
must be less than " + "the number of "
+ "columns in select clause");
}
if (beginIndexOfAggregation != -1 && (orderByElements.size() !=
(beginIndexOfAggregation))) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/InsertStmt.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/InsertStmt.java
index bd6f0ac..8ded93d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/InsertStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/InsertStmt.java
@@ -129,6 +129,12 @@ public class InsertStmt extends DdlStmt {
*/
private boolean isTransactionBegin = false;
+ private boolean isValuesOrConstantSelect = false;
+
+ public boolean isValuesOrConstantSelect() {
+ return isValuesOrConstantSelect;
+ }
+
public InsertStmt(InsertTarget target, String label, List<String> cols,
InsertSource source, List<String> hints) {
this.tblName = target.getTblName();
this.targetPartitionNames = target.getPartitionNames();
@@ -140,6 +146,8 @@ public class InsertStmt extends DdlStmt {
if (!Strings.isNullOrEmpty(label)) {
isUserSpecifiedLabel = true;
}
+
+ this.isValuesOrConstantSelect = (queryStmt instanceof SelectStmt &&
((SelectStmt) queryStmt).getTableRefs().isEmpty());
}
// Ctor for CreateTableAsSelectStmt
@@ -488,7 +496,7 @@ public class InsertStmt extends DdlStmt {
checkColumnCoverage(mentionedColumns, targetTable.getBaseSchema()) ;
// handle VALUES() or SELECT constant list
- if (queryStmt instanceof SelectStmt && ((SelectStmt)
queryStmt).getTableRefs().isEmpty()) {
+ if (isValuesOrConstantSelect) {
SelectStmt selectStmt = (SelectStmt) queryStmt;
if (selectStmt.getValueList() != null) {
// INSERT INTO VALUES(...)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionKeyDesc.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionKeyDesc.java
index 3237ee2..da019d7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionKeyDesc.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionKeyDesc.java
@@ -106,20 +106,22 @@ public class PartitionKeyDesc {
}
}
- // currently, we do not support MAXVALUE in partition range values.
eg: ("100", "200", MAXVALUE);
- // maybe support later.
- if (lowerValues != null) {
+ // Currently, we do not support MAXVALUE in multi partition range
values. eg: ("100", "200", MAXVALUE);
+ // Because we still don’t support expressing such values on the BE
side.
+ // Maybe support later.
+ // But we can support MAXVALUE in single partition values.
+ if (lowerValues != null && lowerValues.size() > 1) {
for (PartitionValue lowerVal : lowerValues) {
if (lowerVal.isMax()) {
- throw new AnalysisException("Not support MAXVALUE in
partition range values.");
+ throw new AnalysisException("Not support MAXVALUE in multi
partition range values.");
}
}
}
- if (upperValues != null) {
+ if (upperValues != null && upperValues.size() > 1) {
for (PartitionValue upperVal : upperValues) {
if (upperVal.isMax()) {
- throw new AnalysisException("Not support MAXVALUE in
partition range values.");
+ throw new AnalysisException("Not support MAXVALUE in multi
partition range values.");
}
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java
index f1b7e82..2226cd1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java
@@ -74,7 +74,7 @@ public class Database extends MetaObject implements Writable {
private static final Logger LOG = LogManager.getLogger(Database.class);
// empirical value.
- // assume that the time a lock is held by thread is less then 100ms
+ // assume that the time a lock is held by thread is less than 100ms
public static final long TRY_LOCK_TIMEOUT_MS = 100L;
private long id;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
index 478e076..a237627 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
@@ -281,7 +281,7 @@ public class OlapTable extends Table {
indexName = getIndexNameById(indexId);
Preconditions.checkState(indexName != null);
}
- // Nullable when meta is less then VERSION_74
+ // Nullable when meta is less than VERSION_74
if (keysType == null) {
keysType = this.keysType;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java
b/fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java
index 98944fd..71b22f1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/Log4jConfig.java
@@ -108,6 +108,8 @@ public class Log4jConfig extends XmlConfiguration {
private static String logXmlConfTemplate;
// dir of fe.conf
public static String confDir;
+ // custom conf dir
+ public static String customConfDir;
private static void reconfig() throws IOException {
String newXmlConfTemplate = xmlConfTemplate;
@@ -180,7 +182,7 @@ public class Log4jConfig extends XmlConfiguration {
System.out.println(newXmlConfTemplate);
System.out.println("=====");
logXmlConfTemplate = newXmlConfTemplate;
- SpringLog4j2Config.writeSpringLogConf(confDir);
+ SpringLog4j2Config.writeSpringLogConf(customConfDir);
// new SimpleLog4jConfiguration with xmlConfTemplate
ByteArrayInputStream bis = new
ByteArrayInputStream(newXmlConfTemplate.getBytes("UTF-8"));
@@ -221,6 +223,7 @@ public class Log4jConfig extends XmlConfiguration {
verboseModules = Config.sys_log_verbose_modules;
auditModules = Config.audit_log_modules;
confDir = dorisConfDir;
+ customConfDir = Config.custom_config_dir;
reconfig();
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/HttpServer.java
b/fe/fe-core/src/main/java/org/apache/doris/httpv2/HttpServer.java
index 62af813..0a1b654 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/HttpServer.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/HttpServer.java
@@ -18,6 +18,7 @@
package org.apache.doris.httpv2;
import org.apache.doris.PaloFe;
+import org.apache.doris.common.Config;
import org.apache.doris.httpv2.config.SpringLog4j2Config;
import org.springframework.boot.autoconfigure.SpringBootApplication;
@@ -85,7 +86,7 @@ public class HttpServer extends SpringBootServletInitializer {
// To avoid some unexpected behavior.
System.setProperty("spring.devtools.restart.enabled", "false");
System.setProperty("spring.http.multipart.location",
PaloFe.DORIS_HOME_DIR);
- properties.put("logging.config", dorisHome + "/conf/" +
SpringLog4j2Config.SPRING_LOG_XML_FILE);
+ properties.put("logging.config", Config.custom_config_dir + "/" +
SpringLog4j2Config.SPRING_LOG_XML_FILE);
new SpringApplicationBuilder()
.sources(HttpServer.class)
.properties(properties)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/httpv2/config/SpringLog4j2Config.java
b/fe/fe-core/src/main/java/org/apache/doris/httpv2/config/SpringLog4j2Config.java
index d18c2b5..21842c5 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/httpv2/config/SpringLog4j2Config.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/httpv2/config/SpringLog4j2Config.java
@@ -35,7 +35,7 @@ public class SpringLog4j2Config {
Writer writer = null;
try {
// log4j2-spring.xml file path
- File file = new File(confDir + SPRING_LOG_XML_FILE);
+ File file = new File(confDir + "/" + SPRING_LOG_XML_FILE);
if (!file.exists()) {
file.createNewFile();
//write file
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadLoadingTask.java
b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadLoadingTask.java
index f9ad624..2e0c7bb 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadLoadingTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadLoadingTask.java
@@ -140,7 +140,7 @@ public class LoadLoadingTask extends LoadTask {
private void actualExecute(Coordinator curCoordinator) throws Exception {
int waitSecond = (int) (getLeftTimeMs() / 1000);
if (waitSecond <= 0) {
- throw new LoadException("failed to execute plan when the left time
is less then 0");
+ throw new LoadException("failed to execute plan when the left time
is less than 0");
}
if (LOG.isDebugEnabled()) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java
b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java
index f55e904..8213514 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadJob.java
@@ -302,7 +302,6 @@ public abstract class RoutineLoadJob extends
AbstractTxnStateChangeCallback impl
}
jobProperties.put(LoadStmt.TIMEZONE, stmt.getTimezone());
jobProperties.put(LoadStmt.STRICT_MODE,
String.valueOf(stmt.isStrictMode()));
- jobProperties.put(LoadStmt.EXEC_MEM_LIMIT,
String.valueOf(stmt.getExecMemLimit()));
if (Strings.isNullOrEmpty(stmt.getFormat()) ||
stmt.getFormat().equals("csv")) {
jobProperties.put(PROPS_FORMAT, "csv");
jobProperties.put(PROPS_STRIP_OUTER_ARRAY, "false");
@@ -522,11 +521,7 @@ public abstract class RoutineLoadJob extends
AbstractTxnStateChangeCallback impl
@Override
public long getMemLimit() {
- String value = jobProperties.get(LoadStmt.EXEC_MEM_LIMIT);
- if (value == null) {
- return DEFAULT_EXEC_MEM_LIMIT;
- }
- return Long.valueOf(value);
+ return execMemLimit;
}
public String getTimezone() {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadScheduler.java
b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadScheduler.java
index 5454481..df15bdc 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadScheduler.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadScheduler.java
@@ -82,7 +82,7 @@ public class RoutineLoadScheduler extends MasterDaemon {
if (desiredConcurrentTaskNum <= 0) {
// the job will be rescheduled later.
LOG.info(new LogBuilder(LogKey.ROUTINE_LOAD_JOB,
routineLoadJob.getId())
- .add("msg", "the current concurrent num
is less then or equal to zero, "
+ .add("msg", "the current concurrent num
is less than or equal to zero, "
+ "job will be rescheduled later")
.build());
continue;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/planner/AssertNumRowsNode.java
b/fe/fe-core/src/main/java/org/apache/doris/planner/AssertNumRowsNode.java
index f8196e5..388f6fa 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/AssertNumRowsNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/AssertNumRowsNode.java
@@ -29,7 +29,7 @@ import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
/**
- * Assert num rows node is used to determine whether the number of rows is
less then desired num of rows.
+ * Assert num rows node is used to determine whether the number of rows is
less than desired num of rows.
* The rows are the result of subqueryString.
* If the number of rows is more than the desired num of rows, the query will
be cancelled.
* The cancelled reason will be reported by Backend and displayed back to the
user.
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java
index 727559a..6aae692 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java
@@ -17,7 +17,7 @@
package org.apache.doris.qe;
-import avro.shaded.com.google.common.collect.Lists;
+import org.apache.doris.analysis.InsertStmt;
import org.apache.doris.analysis.KillStmt;
import org.apache.doris.analysis.SqlParser;
import org.apache.doris.analysis.SqlScanner;
@@ -50,6 +50,7 @@ import org.apache.doris.thrift.TMasterOpRequest;
import org.apache.doris.thrift.TMasterOpResult;
import org.apache.doris.thrift.TUniqueId;
+import com.google.common.collect.Lists;
import com.google.common.base.Strings;
import org.apache.logging.log4j.LogManager;
@@ -147,7 +148,13 @@ public class ConnectProcessor {
if (!ctx.getState().isQuery() && (parsedStmt != null &&
parsedStmt.needAuditEncryption())) {
ctx.getAuditEventBuilder().setStmt(parsedStmt.toSql());
} else {
- ctx.getAuditEventBuilder().setStmt(origStmt);
+ if (parsedStmt instanceof InsertStmt &&
((InsertStmt)parsedStmt).isValuesOrConstantSelect()) {
+ // INSERT INTO VALUES may be very long, so we only log at most
1K bytes.
+ int length = Math.min(1024, origStmt.length());
+ ctx.getAuditEventBuilder().setStmt(origStmt.substring(0,
length));
+ } else {
+ ctx.getAuditEventBuilder().setStmt(origStmt);
+ }
}
Catalog.getCurrentAuditEventProcessor().handleAuditEvent(ctx.getAuditEventBuilder().build());
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/transaction/TabletQuorumFailedException.java
b/fe/fe-core/src/main/java/org/apache/doris/transaction/TabletQuorumFailedException.java
index cf49602..2aa3a3b 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/transaction/TabletQuorumFailedException.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/transaction/TabletQuorumFailedException.java
@@ -25,7 +25,7 @@ import java.util.Set;
public class TabletQuorumFailedException extends TransactionException {
private static final String TABLET_QUORUM_FAILED_MSG = "Failed to commit
txn %s. "
- + "Tablet [%s] success replica num %s is less then quorum "
+ + "Tablet [%s] success replica num %s is less than quorum "
+ "replica num %s while error backends %s";
private long tabletId;
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/utframe/UtFrameUtils.java
b/fe/fe-core/src/test/java/org/apache/doris/utframe/UtFrameUtils.java
index 8b416dc..7686c0f 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/utframe/UtFrameUtils.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/utframe/UtFrameUtils.java
@@ -130,6 +130,11 @@ public class UtFrameUtils {
dorisHome =
Files.createTempDirectory("DORIS_HOME").toAbsolutePath().toString();
}
Config.plugin_dir = dorisHome + "/plugins";
+ Config.custom_config_dir = dorisHome + "/conf";
+ File file = new File(Config.custom_config_dir);
+ if (!file.exists()) {
+ file.mkdir();
+ }
int fe_http_port = findValidPort();
int fe_rpc_port = findValidPort();
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
