This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new ecbd4bc [fix](catalog) Fix bug that The MetaObject lock design of fe
would cause some problems with consistent meta when catalog do replay operation
(#6650)
ecbd4bc is described below
commit ecbd4bcae06c3c154139dce52995caab8c3a2ac7
Author: caiconghui <55968745+caicong...@users.noreply.github.com>
AuthorDate: Tue Feb 8 10:01:52 2022 +0800
[fix](catalog) Fix bug that The MetaObject lock design of fe would cause
some problems with consistent meta when catalog do replay operation (#6650)
1. If the table or db has been dropped，we will get write lock failed or
just skip or throw exception，
2. and if we recover table or db， we must ensure that unmark dropped state
after writing recover journal.
3. db.dropTable corresponds to db.createTable, I don't move
table.markDropped method to the db.dropTable,
for that all meta added to db or catalog must after writing recover
journal, so we must invoke markDropped
and unmarkDropped method outside the dropTable and createTable method.
---
.../main/java/org/apache/doris/alter/Alter.java | 123 ++++++++------
.../java/org/apache/doris/alter/AlterHandler.java | 2 +-
.../java/org/apache/doris/alter/AlterJobV2.java | 2 +-
.../doris/alter/MaterializedViewHandler.java | 10 +-
.../java/org/apache/doris/alter/RollupJob.java | 5 +-
.../java/org/apache/doris/alter/RollupJobV2.java | 4 +-
.../apache/doris/alter/SchemaChangeHandler.java | 8 +-
.../org/apache/doris/alter/SchemaChangeJob.java | 15 +-
.../org/apache/doris/alter/SchemaChangeJobV2.java | 4 +-
.../java/org/apache/doris/backup/RestoreJob.java | 51 ++++--
.../java/org/apache/doris/catalog/Catalog.java | 176 +++++++++++++--------
.../apache/doris/catalog/CatalogRecycleBin.java | 36 +++--
.../java/org/apache/doris/catalog/Database.java | 64 +++++---
.../main/java/org/apache/doris/catalog/Table.java | 67 ++++++++
.../org/apache/doris/catalog/TabletStatMgr.java | 5 +-
.../doris/clone/DynamicPartitionScheduler.java | 4 +-
.../org/apache/doris/clone/TabletSchedCtx.java | 10 +-
.../org/apache/doris/clone/TabletScheduler.java | 9 +-
.../apache/doris/common/util/MetaLockUtils.java | 27 +++-
.../doris/consistency/CheckConsistencyJob.java | 8 +-
.../org/apache/doris/http/rest/RowCountAction.java | 2 +-
.../doris/http/rest/TableRowCountAction.java | 4 +-
.../doris/httpv2/rest/TableRowCountAction.java | 2 +-
.../apache/doris/journal/bdbje/BDBJEJournal.java | 1 -
.../java/org/apache/doris/load/LoadChecker.java | 7 +-
.../apache/doris/load/loadv2/BrokerLoadJob.java | 2 +-
.../org/apache/doris/load/loadv2/SparkLoadJob.java | 2 +-
.../java/org/apache/doris/master/MasterImpl.java | 6 +-
.../org/apache/doris/master/ReportHandler.java | 11 +-
.../doris/transaction/DatabaseTransactionMgr.java | 19 +--
.../doris/transaction/GlobalTransactionMgr.java | 2 +-
.../org/apache/doris/catalog/DatabaseTest.java | 21 ++-
.../org/apache/doris/catalog/InfoSchemaDbTest.java | 3 +-
.../java/org/apache/doris/catalog/TableTest.java | 34 +++-
.../doris/common/util/MetaLockUtilsTest.java | 62 +++++++-
35 files changed, 550 insertions(+), 258 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java
b/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java
index 85eaaaf..2667a0c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java
@@ -57,6 +57,7 @@ import org.apache.doris.common.DdlException;
import org.apache.doris.common.MetaNotFoundException;
import org.apache.doris.common.UserException;
import org.apache.doris.common.util.DynamicPartitionUtil;
+import org.apache.doris.common.util.MetaLockUtils;
import org.apache.doris.common.util.PropertyAnalyzer;
import org.apache.doris.persist.AlterViewInfo;
import org.apache.doris.persist.BatchModifyPartitionsInfo;
@@ -77,6 +78,7 @@ import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.Arrays;
+import java.util.Comparator;
import java.util.List;
import java.util.Map;
@@ -158,7 +160,7 @@ public class Alter {
} else if (currentAlterOps.hasPartitionOp()) {
Preconditions.checkState(alterClauses.size() == 1);
AlterClause alterClause = alterClauses.get(0);
- olapTable.writeLock();
+ olapTable.writeLockOrDdlException();
try {
if (alterClause instanceof DropPartitionClause) {
if (!((DropPartitionClause)
alterClause).isTempPartition()) {
@@ -217,7 +219,7 @@ public class Alter {
private void processModifyTableComment(Database db, OlapTable tbl,
AlterClause alterClause)
throws DdlException {
- tbl.writeLock();
+ tbl.writeLockOrDdlException();
try {
ModifyTableCommentClause clause = (ModifyTableCommentClause)
alterClause;
tbl.setComment(clause.getComment());
@@ -231,7 +233,7 @@ public class Alter {
private void processModifyColumnComment(Database db, OlapTable tbl,
List<AlterClause> alterClauses)
throws DdlException {
- tbl.writeLock();
+ tbl.writeLockOrDdlException();
try {
// check first
Map<String, String> colToComment = Maps.newHashMap();
@@ -304,14 +306,15 @@ public class Alter {
}
public void processModifyEngine(Database db, Table externalTable,
ModifyEngineClause clause) throws DdlException {
- if (externalTable.getType() != TableType.MYSQL) {
- throw new DdlException("Only support modify table engine from
MySQL to ODBC");
+ externalTable.writeLockOrDdlException();
+ try {
+ if (externalTable.getType() != TableType.MYSQL) {
+ throw new DdlException("Only support modify table engine from
MySQL to ODBC");
+ }
+ processModifyEngineInternal(db, externalTable,
clause.getProperties(), false);
+ } finally {
+ externalTable.writeUnlock();
}
-
- processModifyEngineInternal(db, externalTable, clause.getProperties());
- ModifyTableEngineOperationLog log = new
ModifyTableEngineOperationLog(db.getId(),
- externalTable.getId(), clause.getProperties());
- Catalog.getCurrentCatalog().getEditLog().logModifyTableEngine(log);
LOG.info("modify table {}'s engine from MySQL to ODBC",
externalTable.getName());
}
@@ -324,10 +327,15 @@ public class Alter {
if (mysqlTable == null) {
return;
}
- processModifyEngineInternal(db, mysqlTable, log.getProperties());
+ mysqlTable.writeLock();
+ try {
+ processModifyEngineInternal(db, mysqlTable, log.getProperties(),
true);
+ } finally {
+ mysqlTable.writeUnlock();
+ }
}
- private void processModifyEngineInternal(Database db, Table externalTable,
Map<String, String> prop) {
+ private void processModifyEngineInternal(Database db, Table externalTable,
Map<String, String> prop, boolean isReplay) {
MysqlTable mysqlTable = (MysqlTable) externalTable;
Map<String, String> newProp = Maps.newHashMap(prop);
newProp.put(OdbcTable.ODBC_HOST, mysqlTable.getHost());
@@ -346,8 +354,18 @@ public class Alter {
LOG.warn("Should not happen", e);
return;
}
- db.dropTable(mysqlTable.getName());
- db.createTable(odbcTable);
+ odbcTable.writeLock();
+ try {
+ db.dropTable(mysqlTable.getName());
+ db.createTable(odbcTable);
+ if (!isReplay) {
+ ModifyTableEngineOperationLog log = new
ModifyTableEngineOperationLog(db.getId(),
+ externalTable.getId(), prop);
+
Catalog.getCurrentCatalog().getEditLog().logModifyTableEngine(log);
+ }
+ } finally {
+ odbcTable.writeUnlock();
+ }
}
public void processAlterTable(AlterTableStmt stmt) throws UserException {
@@ -393,7 +411,7 @@ public class Alter {
((SchemaChangeHandler)
schemaChangeHandler).updatePartitionsInMemoryMeta(
db, tableName, partitionNames, properties);
OlapTable olapTable = (OlapTable) table;
- olapTable.writeLock();
+ olapTable.writeLockOrDdlException();
try {
modifyPartitionsProperty(db, olapTable, partitionNames,
properties);
} finally {
@@ -415,27 +433,31 @@ public class Alter {
ReplaceTableClause clause = (ReplaceTableClause) alterClauses.get(0);
String newTblName = clause.getTblName();
boolean swapTable = clause.isSwapTable();
- Table newTbl = db.getTableOrMetaException(newTblName, TableType.OLAP);
- OlapTable olapNewTbl = (OlapTable) newTbl;
- db.writeLock();
- origTable.writeLock();
+ db.writeLockOrDdlException();
try {
- String oldTblName = origTable.getName();
- // First, we need to check whether the table to be operated on can
be renamed
- olapNewTbl.checkAndSetName(oldTblName, true);
- if (swapTable) {
- origTable.checkAndSetName(newTblName, true);
+ Table newTbl = db.getTableOrMetaException(newTblName,
TableType.OLAP);
+ OlapTable olapNewTbl = (OlapTable) newTbl;
+ List<Table> tableList = Lists.newArrayList(origTable, newTbl);
+ tableList.sort((Comparator.comparing(Table::getId)));
+ MetaLockUtils.writeLockTablesOrMetaException(tableList);
+ try {
+ String oldTblName = origTable.getName();
+ // First, we need to check whether the table to be operated on
can be renamed
+ olapNewTbl.checkAndSetName(oldTblName, true);
+ if (swapTable) {
+ origTable.checkAndSetName(newTblName, true);
+ }
+ replaceTableInternal(db, origTable, olapNewTbl, swapTable,
false);
+ // write edit log
+ ReplaceTableOperationLog log = new
ReplaceTableOperationLog(db.getId(), origTable.getId(), olapNewTbl.getId(),
swapTable);
+ Catalog.getCurrentCatalog().getEditLog().logReplaceTable(log);
+ LOG.info("finish replacing table {} with table {}, is swap:
{}", oldTblName, newTblName, swapTable);
+ } finally {
+ MetaLockUtils.writeUnlockTables(tableList);
}
- replaceTableInternal(db, origTable, olapNewTbl, swapTable, false);
- // write edit log
- ReplaceTableOperationLog log = new
ReplaceTableOperationLog(db.getId(), origTable.getId(), olapNewTbl.getId(),
swapTable);
- Catalog.getCurrentCatalog().getEditLog().logReplaceTable(log);
- LOG.info("finish replacing table {} with table {}, is swap: {}",
oldTblName, newTblName, swapTable);
} finally {
- origTable.writeUnlock();
db.writeUnlock();
}
-
}
public void replayReplaceTable(ReplaceTableOperationLog log) throws
MetaNotFoundException {
@@ -446,11 +468,15 @@ public class Alter {
Database db = Catalog.getCurrentCatalog().getDbOrMetaException(dbId);
OlapTable origTable = db.getTableOrMetaException(origTblId,
TableType.OLAP);
OlapTable newTbl = db.getTableOrMetaException(newTblId,
TableType.OLAP);
-
+ List<Table> tableList = Lists.newArrayList(origTable, newTbl);
+ tableList.sort((Comparator.comparing(Table::getId)));
+ MetaLockUtils.writeLockTablesOrMetaException(tableList);
try {
replaceTableInternal(db, origTable, newTbl, log.isSwapTable(),
true);
} catch (DdlException e) {
LOG.warn("should not happen", e);
+ } finally {
+ MetaLockUtils.writeUnlockTables(tableList);
}
LOG.info("finish replay replacing table {} with table {}, is swap:
{}", origTblId, newTblId, log.isSwapTable());
}
@@ -505,25 +531,28 @@ public class Alter {
}
private void modifyViewDef(Database db, View view, String inlineViewDef,
long sqlMode, List<Column> newFullSchema) throws DdlException {
- db.writeLock();
- view.writeLock();
+ db.writeLockOrDdlException();
try {
- view.setInlineViewDefWithSqlMode(inlineViewDef, sqlMode);
+ view.writeLockOrDdlException();
try {
- view.init();
- } catch (UserException e) {
- throw new DdlException("failed to init view stmt", e);
+ view.setInlineViewDefWithSqlMode(inlineViewDef, sqlMode);
+ try {
+ view.init();
+ } catch (UserException e) {
+ throw new DdlException("failed to init view stmt", e);
+ }
+ view.setNewFullSchema(newFullSchema);
+ String viewName = view.getName();
+ db.dropTable(viewName);
+ db.createTable(view);
+
+ AlterViewInfo alterViewInfo = new AlterViewInfo(db.getId(),
view.getId(), inlineViewDef, newFullSchema, sqlMode);
+
Catalog.getCurrentCatalog().getEditLog().logModifyViewDef(alterViewInfo);
+ LOG.info("modify view[{}] definition to {}", viewName,
inlineViewDef);
+ } finally {
+ view.writeUnlock();
}
- view.setNewFullSchema(newFullSchema);
- String viewName = view.getName();
- db.dropTable(viewName);
- db.createTable(view);
-
- AlterViewInfo alterViewInfo = new AlterViewInfo(db.getId(),
view.getId(), inlineViewDef, newFullSchema, sqlMode);
-
Catalog.getCurrentCatalog().getEditLog().logModifyViewDef(alterViewInfo);
- LOG.info("modify view[{}] definition to {}", viewName,
inlineViewDef);
} finally {
- view.writeUnlock();
db.writeUnlock();
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/AlterHandler.java
b/fe/fe-core/src/main/java/org/apache/doris/alter/AlterHandler.java
index 2f825b2..9aefdbb 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/AlterHandler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/AlterHandler.java
@@ -420,7 +420,7 @@ public abstract class AlterHandler extends MasterDaemon {
Database db =
Catalog.getCurrentCatalog().getDbOrMetaException(task.getDbId());
OlapTable tbl = db.getTableOrMetaException(task.getTableId(),
Table.TableType.OLAP);
- tbl.writeLock();
+ tbl.writeLockOrMetaException();
try {
Partition partition = tbl.getPartition(task.getPartitionId());
if (partition == null) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/AlterJobV2.java
b/fe/fe-core/src/main/java/org/apache/doris/alter/AlterJobV2.java
index 6a31be1..c8c8815 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/AlterJobV2.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/AlterJobV2.java
@@ -194,7 +194,7 @@ public abstract class AlterJobV2 implements Writable {
throw new AlterCancelException(e.getMessage());
}
- tbl.writeLock();
+ tbl.writeLockOrAlterCancelException();
try {
boolean isStable = tbl.isStable(Catalog.getCurrentSystemInfo(),
Catalog.getCurrentCatalog().getTabletScheduler(),
db.getClusterName());
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java
b/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java
index e46378b..af07dbb 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java
@@ -174,7 +174,7 @@ public class MaterializedViewHandler extends AlterHandler {
*/
public void processCreateMaterializedView(CreateMaterializedViewStmt
addMVClause, Database db, OlapTable olapTable)
throws DdlException, AnalysisException {
- olapTable.writeLock();
+ olapTable.writeLockOrDdlException();
try {
olapTable.checkStableAndNormal(db.getClusterName());
if (olapTable.existTempPartitions()) {
@@ -229,7 +229,7 @@ public class MaterializedViewHandler extends AlterHandler {
Map<String, RollupJobV2> rollupNameJobMap = new LinkedHashMap<>();
// save job id for log
Set<Long> logJobIdSet = new HashSet<>();
- olapTable.writeLock();
+ olapTable.writeLockOrDdlException();
try {
if (olapTable.existTempPartitions()) {
throw new DdlException("Can not alter table when there are
temp partitions in table");
@@ -711,7 +711,7 @@ public class MaterializedViewHandler extends AlterHandler {
public void processBatchDropRollup(List<AlterClause> dropRollupClauses,
Database db, OlapTable olapTable)
throws DdlException, MetaNotFoundException {
- olapTable.writeLock();
+ olapTable.writeLockOrDdlException();
try {
if (olapTable.existTempPartitions()) {
throw new DdlException("Can not alter table when there are
temp partitions in table");
@@ -747,7 +747,7 @@ public class MaterializedViewHandler extends AlterHandler {
public void processDropMaterializedView(DropMaterializedViewStmt
dropMaterializedViewStmt, Database db,
OlapTable olapTable) throws
DdlException, MetaNotFoundException {
- olapTable.writeLock();
+ olapTable.writeLockOrDdlException();
try {
// check table state
if (olapTable.getState() != OlapTableState.NORMAL) {
@@ -884,7 +884,7 @@ public class MaterializedViewHandler extends AlterHandler {
try {
Database db =
Catalog.getCurrentCatalog().getDbOrMetaException(dbId);
OlapTable olapTable = db.getTableOrMetaException(tableId,
Table.TableType.OLAP);
- olapTable.writeLock();
+ olapTable.writeLockOrMetaException();
try {
if (olapTable.getState() == olapTableState) {
return;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJob.java
b/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJob.java
index 9b83815..1ab6cd2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJob.java
@@ -643,7 +643,10 @@ public class RollupJob extends AlterJob {
return -1;
}
- olapTable.writeLock();
+ if (!olapTable.writeLockIfExist()) {
+ LOG.warn("unknown table, tableName=" + olapTable.getName());
+ return -1;
+ }
try {
// if all previous transaction has finished, then check base and
rollup replica num
synchronized (this) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java
b/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java
index e344c8f..81a9f7e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java
@@ -281,7 +281,7 @@ public class RollupJobV2 extends AlterJobV2 implements
GsonPostProcessable {
// create all rollup replicas success.
// add rollup index to catalog
- tbl.writeLock();
+ tbl.writeLockOrAlterCancelException();
try {
Preconditions.checkState(tbl.getState() == OlapTableState.ROLLUP);
addRollupIndexToCatalog(tbl);
@@ -427,7 +427,7 @@ public class RollupJobV2 extends AlterJobV2 implements
GsonPostProcessable {
* all tasks are finished. check the integrity.
* we just check whether all rollup replicas are healthy.
*/
- tbl.writeLock();
+ tbl.writeLockOrAlterCancelException();
try {
Preconditions.checkState(tbl.getState() == OlapTableState.ROLLUP);
for (Map.Entry<Long, MaterializedIndex> entry :
this.partitionIdToRollupIndex.entrySet()) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
index 4abdf90..db90d30 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
@@ -1621,7 +1621,7 @@ public class SchemaChangeHandler extends AlterHandler {
@Override
public void process(List<AlterClause> alterClauses, String clusterName,
Database db, OlapTable olapTable)
throws UserException {
- olapTable.writeLock();
+ olapTable.writeLockOrDdlException();
try {
// index id -> index schema
Map<Long, LinkedList<Column>> indexSchemaMap = new HashMap<>();
@@ -1723,7 +1723,7 @@ public class SchemaChangeHandler extends AlterHandler {
@Override
public void processExternalTable(List<AlterClause> alterClauses, Database
db, Table externalTable)
throws UserException {
- externalTable.writeLock();
+ externalTable.writeLockOrDdlException();
try {
// copy the external table schema columns
List<Column> newSchema = Lists.newArrayList();
@@ -1804,7 +1804,7 @@ public class SchemaChangeHandler extends AlterHandler {
updatePartitionInMemoryMeta(db, olapTable.getName(),
partition.getName(), isInMemory);
}
- olapTable.writeLock();
+ olapTable.writeLockOrDdlException();
try {
Catalog.getCurrentCatalog().modifyTableInMemoryMeta(db, olapTable,
properties);
} finally {
@@ -1932,7 +1932,7 @@ public class SchemaChangeHandler extends AlterHandler {
AlterJobV2 schemaChangeJobV2 = null;
OlapTable olapTable = db.getOlapTableOrDdlException(tableName);
- olapTable.writeLock();
+ olapTable.writeLockOrDdlException();
try {
if (olapTable.getState() != OlapTableState.SCHEMA_CHANGE &&
olapTable.getState() != OlapTableState.WAITING_STABLE) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJob.java
b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJob.java
index f781d65..f2df913 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJob.java
@@ -584,15 +584,9 @@ public class SchemaChangeJob extends AlterJob {
long replicaId = schemaChangeTask.getReplicaId();
// update replica's info
- OlapTable olapTable;
- try {
- Database db =
Catalog.getCurrentCatalog().getDbOrMetaException(dbId);
- olapTable = db.getTableOrMetaException(tableId,
Table.TableType.OLAP);
- } catch (MetaNotFoundException e) {
- LOG.warn(e.getMessage());
- return;
- }
- olapTable.writeLock();
+ Database db = Catalog.getCurrentCatalog().getDbOrMetaException(dbId);
+ OlapTable olapTable = db.getTableOrMetaException(tableId,
Table.TableType.OLAP);
+ olapTable.writeLockOrMetaException();
try {
Preconditions.checkState(olapTable.getState() ==
OlapTableState.SCHEMA_CHANGE);
@@ -668,12 +662,11 @@ public class SchemaChangeJob extends AlterJob {
OlapTable olapTable;
try {
olapTable = db.getTableOrMetaException(tableId,
Table.TableType.OLAP);
+ olapTable.writeLockOrMetaException();
} catch (MetaNotFoundException e) {
LOG.warn(e.getMessage());
return -1;
}
-
- olapTable.writeLock();
try {
synchronized (this) {
boolean hasUnfinishedPartition = false;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
index 1ad1493..2d77da2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
@@ -307,7 +307,7 @@ public class SchemaChangeJobV2 extends AlterJobV2 {
// create all replicas success.
// add all shadow indexes to catalog
- tbl.writeLock();
+ tbl.writeLockOrAlterCancelException();
try {
Preconditions.checkState(tbl.getState() ==
OlapTableState.SCHEMA_CHANGE);
addShadowIndexToCatalog(tbl);
@@ -465,7 +465,7 @@ public class SchemaChangeJobV2 extends AlterJobV2 {
* all tasks are finished. check the integrity.
* we just check whether all new replicas are healthy.
*/
- tbl.writeLock();
+ tbl.writeLockOrAlterCancelException();
try {
Preconditions.checkState(tbl.getState() ==
OlapTableState.SCHEMA_CHANGE);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
index 8e92395..151ab13 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
@@ -445,7 +445,9 @@ public class RestoreJob extends AbstractJob {
}
OlapTable olapTbl = (OlapTable) tbl;
- olapTbl.writeLock();
+ if (!olapTbl.writeLockIfExist()) {
+ continue;
+ }
try {
if (olapTbl.getState() != OlapTableState.NORMAL) {
status = new Status(ErrCode.COMMON_ERROR,
@@ -783,7 +785,12 @@ public class RestoreJob extends AbstractJob {
// add restored tables
for (Table tbl : restoredTbls) {
- db.writeLock();
+ if (!db.writeLockIfExist()) {
+ status = new Status(ErrCode.COMMON_ERROR, "Database " +
db.getFullName()
+ + " has been dropped");
+ return;
+ }
+ tbl.writeLock();
try {
if (!db.createTable(tbl)) {
status = new Status(ErrCode.COMMON_ERROR, "Table " +
tbl.getName()
@@ -791,6 +798,7 @@ public class RestoreJob extends AbstractJob {
return;
}
} finally {
+ tbl.writeUnlock();
db.writeUnlock();
}
}
@@ -1109,9 +1117,11 @@ public class RestoreJob extends AbstractJob {
// restored tables
for (Table restoreTbl : restoredTbls) {
db.writeLock();
+ restoreTbl.writeLock();
try {
db.createTable(restoreTbl);
} finally {
+ restoreTbl.writeUnlock();
db.writeUnlock();
}
if (restoreTbl.getType() != TableType.OLAP) {
@@ -1387,7 +1397,9 @@ public class RestoreJob extends AbstractJob {
continue;
}
OlapTable olapTbl = (OlapTable) tbl;
- tbl.writeLock();
+ if (!tbl.writeLockIfExist()) {
+ continue;
+ }
try {
Map<Long, Pair<Long, Long>> map =
restoredVersionInfo.rowMap().get(tblId);
for (Map.Entry<Long, Pair<Long, Long>> entry : map.entrySet())
{
@@ -1550,22 +1562,29 @@ public class RestoreJob extends AbstractJob {
// remove restored tbls
for (Table restoreTbl : restoredTbls) {
LOG.info("remove restored table when cancelled: {}",
restoreTbl.getName());
- if (restoreTbl.getType() == TableType.OLAP) {
- OlapTable restoreOlapTable = (OlapTable) restoreTbl;
- restoreOlapTable.writeLock();
+ if (db.writeLockIfExist()) {
try {
- for (Partition part :
restoreOlapTable.getPartitions()) {
- for (MaterializedIndex idx :
part.getMaterializedIndices(IndexExtState.VISIBLE)) {
- for (Tablet tablet : idx.getTablets()) {
-
Catalog.getCurrentInvertedIndex().deleteTablet(tablet.getId());
+ if (restoreTbl.getType() == TableType.OLAP) {
+ OlapTable restoreOlapTable = (OlapTable)
restoreTbl;
+ restoreOlapTable.writeLock();
+ try {
+ for (Partition part :
restoreOlapTable.getPartitions()) {
+ for (MaterializedIndex idx :
part.getMaterializedIndices(IndexExtState.VISIBLE)) {
+ for (Tablet tablet : idx.getTablets())
{
+
Catalog.getCurrentInvertedIndex().deleteTablet(tablet.getId());
+ }
+ }
}
+ db.dropTable(restoreTbl.getName());
+ } finally {
+ restoreTbl.writeUnlock();
}
}
} finally {
- restoreTbl.writeUnlock();
+ db.writeUnlock();
}
}
- db.dropTableWithLock(restoreTbl.getName());
+
}
// remove restored partitions
@@ -1574,9 +1593,11 @@ public class RestoreJob extends AbstractJob {
if (restoreTbl == null) {
continue;
}
+ if (!restoreTbl.writeLockIfExist()) {
+ continue;
+ }
LOG.info("remove restored partition in table {} when
cancelled: {}",
restoreTbl.getName(), entry.second.getName());
- restoreTbl.writeLock();
try {
restoreTbl.dropPartition(dbId, entry.second.getName(),
true /* force drop */);
} finally {
@@ -1625,7 +1646,9 @@ public class RestoreJob extends AbstractJob {
}
OlapTable olapTbl = (OlapTable) tbl;
- tbl.writeLock();
+ if (!tbl.writeLockIfExist()) {
+ continue;
+ }
try {
if (olapTbl.getState() == OlapTableState.RESTORE
|| olapTbl.getState() ==
OlapTableState.RESTORE_WITH_LOAD) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java
index e29a341..304c04b 100755
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java
@@ -2760,7 +2760,26 @@ public class Catalog {
// save table names for recycling
Set<String> tableNames = db.getTableNamesWithLock();
- unprotectDropDb(db, stmt.isForceDrop(), false);
+ List<Table> tableList = db.getTablesOnIdOrder();
+ MetaLockUtils.writeLockTables(tableList);
+ try {
+ if (!stmt.isForceDrop()) {
+ for (Table table : tableList) {
+ if (table.getType() == TableType.OLAP) {
+ OlapTable olapTable = (OlapTable) table;
+ if (olapTable.getState() !=
OlapTableState.NORMAL) {
+ throw new DdlException("The table [" +
olapTable.getState() + "]'s state is " + olapTable.getState() + ", cannot be
dropped." +
+ " please cancel the operation on
olap table firstly. If you want to forcibly drop(cannot be recovered)," +
+ " please use \"DROP table
FORCE\".");
+ }
+ }
+ }
+ }
+ unprotectDropDb(db, stmt.isForceDrop(), false);
+ } finally {
+ MetaLockUtils.writeUnlockTables(tableList);
+ }
+
if (!stmt.isForceDrop()) {
Catalog.getCurrentRecycleBin().recycleDatabase(db,
tableNames);
} else {
@@ -2790,13 +2809,9 @@ public class Catalog {
icebergTableCreationRecordMgr.deregisterDb(db);
}
for (Table table : db.getTables()) {
- table.writeLock();
- try {
- unprotectDropTable(db, table, isForeDrop, isReplay);
- } finally {
- table.writeUnlock();
- }
+ unprotectDropTable(db, table, isForeDrop, isReplay);
}
+ db.markDropped();
}
public void replayDropLinkDb(DropLinkDbAndUpdateDbInfo info) {
@@ -2822,7 +2837,13 @@ public class Catalog {
db.writeLock();
try {
Set<String> tableNames = db.getTableNamesWithLock();
- unprotectDropDb(db, isForceDrop, true);
+ List<Table> tableList = db.getTablesOnIdOrder();
+ MetaLockUtils.writeLockTables(tableList);
+ try {
+ unprotectDropDb(db, isForceDrop, true);
+ } finally {
+ MetaLockUtils.writeUnlockTables(tableList);
+ }
if (!isForceDrop) {
Catalog.getCurrentRecycleBin().recycleDatabase(db,
tableNames);
} else {
@@ -2853,6 +2874,9 @@ public class Catalog {
if (!tryLock(false)) {
throw new DdlException("Failed to acquire catalog lock. Try
again");
}
+ db.writeLock();
+ List<Table> tableList = db.getTablesOnIdOrder();
+ MetaLockUtils.writeLockTables(tableList);
try {
if (fullNameToDb.containsKey(db.getFullName())) {
throw new DdlException("Database[" + db.getFullName() + "]
already exist.");
@@ -2868,7 +2892,10 @@ public class Catalog {
// log
RecoverInfo recoverInfo = new RecoverInfo(db.getId(), -1L, -1L);
editLog.logRecoverDb(recoverInfo);
+ db.unmarkDropped();
} finally {
+ MetaLockUtils.writeUnlockTables(tableList);
+ db.writeUnlock();
unlock();
}
@@ -2880,7 +2907,7 @@ public class Catalog {
String tableName = recoverStmt.getTableName();
Database db = this.getDbOrDdlException(dbName);
- db.writeLock();
+ db.writeLockOrDdlException();
try {
if (db.getTable(tableName).isPresent()) {
ErrorReport.reportDdlException(ErrorCode.ERR_TABLE_EXISTS_ERROR, tableName);
@@ -2899,7 +2926,7 @@ public class Catalog {
Database db = this.getDbOrDdlException(dbName);
OlapTable olapTable = db.getOlapTableOrDdlException(tableName);
- olapTable.writeLock();
+ olapTable.writeLockOrDdlException();
try {
String partitionName = recoverStmt.getPartitionName();
if (olapTable.getPartition(partitionName) != null) {
@@ -2929,24 +2956,33 @@ public class Catalog {
public void alterDatabaseQuota(AlterDatabaseQuotaStmt stmt) throws
DdlException {
String dbName = stmt.getDbName();
Database db = this.getDbOrDdlException(dbName);
-
QuotaType quotaType = stmt.getQuotaType();
- if (quotaType == QuotaType.DATA) {
- db.setDataQuotaWithLock(stmt.getQuota());
- } else if (quotaType == QuotaType.REPLICA) {
- db.setReplicaQuotaWithLock(stmt.getQuota());
+ db.writeLockOrDdlException();
+ try {
+ if (quotaType == QuotaType.DATA) {
+ db.setDataQuota(stmt.getQuota());
+ } else if (quotaType == QuotaType.REPLICA) {
+ db.setReplicaQuota(stmt.getQuota());
+ }
+ long quota = stmt.getQuota();
+ DatabaseInfo dbInfo = new DatabaseInfo(dbName, "", quota,
quotaType);
+ editLog.logAlterDb(dbInfo);
+ } finally {
+ db.writeUnlock();
}
- long quota = stmt.getQuota();
- DatabaseInfo dbInfo = new DatabaseInfo(dbName, "", quota, quotaType);
- editLog.logAlterDb(dbInfo);
}
public void replayAlterDatabaseQuota(String dbName, long quota, QuotaType
quotaType) throws MetaNotFoundException {
Database db = this.getDbOrMetaException(dbName);
- if (quotaType == QuotaType.DATA) {
- db.setDataQuotaWithLock(quota);
- } else if (quotaType == QuotaType.REPLICA) {
- db.setReplicaQuotaWithLock(quota);
+ db.writeLock();
+ try {
+ if (quotaType == QuotaType.DATA) {
+ db.setDataQuota(quota);
+ } else if (quotaType == QuotaType.REPLICA) {
+ db.setReplicaQuota(quota);
+ }
+ } finally {
+ db.writeUnlock();
}
}
@@ -3314,8 +3350,7 @@ public class Catalog {
// check again
table = db.getOlapTableOrDdlException(tableName);
-
- table.writeLock();
+ table.writeLockOrDdlException();
try {
olapTable = (OlapTable) table;
if (olapTable.getState() != OlapTableState.NORMAL) {
@@ -4467,10 +4502,13 @@ public class Catalog {
}
}
- public void replayCreateTable(String dbName, Table table) {
+ public void replayCreateTable(String dbName, Table table) throws
MetaNotFoundException {
Database db = this.fullNameToDb.get(dbName);
- db.createTableWithLock(table, true, false);
-
+ try {
+ db.createTableWithLock(table, true, false);
+ } catch (DdlException e) {
+ throw new MetaNotFoundException(e.getMessage());
+ }
if (!isCheckpointThread()) {
// add to inverted index
if (table.getType() == TableType.OLAP) {
@@ -4597,7 +4635,7 @@ public class Catalog {
// check database
Database db = this.getDbOrDdlException(dbName);
- db.writeLock();
+ db.writeLockOrDdlException();
try {
Table table = db.getTableNullable(tableName);
if (table == null) {
@@ -5004,7 +5042,7 @@ public class Catalog {
OlapTable olapTable = (OlapTable) table;
// use try lock to avoid blocking a long time.
// if block too long, backend report rpc will timeout.
- if (!olapTable.tryWriteLock(Table.TRY_LOCK_TIMEOUT_MS,
TimeUnit.MILLISECONDS)) {
+ if (!olapTable.tryWriteLockIfExist(Table.TRY_LOCK_TIMEOUT_MS,
TimeUnit.MILLISECONDS)) {
LOG.warn("try get table {} writelock but failed when
checking backend storage medium", table.getName());
continue;
}
@@ -5356,42 +5394,45 @@ public class Catalog {
// entry of rename table operation
public void renameTable(Database db, Table table, TableRenameClause
tableRenameClause) throws DdlException {
- db.writeLock();
- table.writeLock();
+ db.writeLockOrDdlException();
try {
- if (table instanceof OlapTable) {
- OlapTable olapTable = (OlapTable) table;
- if (olapTable.getState() != OlapTableState.NORMAL) {
- throw new DdlException("Table[" + olapTable.getName() + "]
is under " + olapTable.getState());
+ table.writeLockOrDdlException();
+ try {
+ if (table instanceof OlapTable) {
+ OlapTable olapTable = (OlapTable) table;
+ if (olapTable.getState() != OlapTableState.NORMAL) {
+ throw new DdlException("Table[" + olapTable.getName()
+ "] is under " + olapTable.getState());
+ }
}
- }
- String oldTableName = table.getName();
- String newTableName = tableRenameClause.getNewTableName();
- if (oldTableName.equals(newTableName)) {
- throw new DdlException("Same table name");
- }
+ String oldTableName = table.getName();
+ String newTableName = tableRenameClause.getNewTableName();
+ if (oldTableName.equals(newTableName)) {
+ throw new DdlException("Same table name");
+ }
- // check if name is already used
- if (db.getTable(newTableName).isPresent()) {
- throw new DdlException("Table name[" + newTableName + "] is
already used");
- }
+ // check if name is already used
+ if (db.getTable(newTableName).isPresent()) {
+ throw new DdlException("Table name[" + newTableName + "]
is already used");
+ }
- if (table.getType() == TableType.OLAP) {
- // olap table should also check if any rollup has same name as
"newTableName"
- ((OlapTable) table).checkAndSetName(newTableName, false);
- } else {
- table.setName(newTableName);
- }
+ if (table.getType() == TableType.OLAP) {
+ // olap table should also check if any rollup has same
name as "newTableName"
+ ((OlapTable) table).checkAndSetName(newTableName, false);
+ } else {
+ table.setName(newTableName);
+ }
- db.dropTable(oldTableName);
- db.createTable(table);
+ db.dropTable(oldTableName);
+ db.createTable(table);
- TableInfo tableInfo = TableInfo.createForTableRename(db.getId(),
table.getId(), newTableName);
- editLog.logTableRename(tableInfo);
- LOG.info("rename table[{}] to {}", oldTableName, newTableName);
+ TableInfo tableInfo =
TableInfo.createForTableRename(db.getId(), table.getId(), newTableName);
+ editLog.logTableRename(tableInfo);
+ LOG.info("rename table[{}] to {}", oldTableName, newTableName);
+ } finally {
+ table.writeUnlock();
+ }
} finally {
- table.writeUnlock();
db.writeUnlock();
}
}
@@ -5412,16 +5453,16 @@ public class Catalog {
db.writeLock();
try {
Table table = db.getTableOrMetaException(tableId);
- String tableName = table.getName();
- db.dropTable(tableName);
table.writeLock();
try {
+ String tableName = table.getName();
+ db.dropTable(tableName);
table.setName(newTableName);
+ db.createTable(table);
+ LOG.info("replay rename table[{}] to {}", tableName,
newTableName);
} finally {
table.writeUnlock();
}
- db.createTable(table);
- LOG.info("replay rename table[{}] to {}", tableName, newTableName);
} finally {
db.writeUnlock();
}
@@ -5534,7 +5575,7 @@ public class Catalog {
}
public void renameRollup(Database db, OlapTable table, RollupRenameClause
renameClause) throws DdlException {
- table.writeLock();
+ table.writeLockOrDdlException();
try {
if (table.getState() != OlapTableState.NORMAL) {
throw new DdlException("Table[" + table.getName() + "] is
under " + table.getState());
@@ -5595,7 +5636,7 @@ public class Catalog {
}
public void renamePartition(Database db, OlapTable table,
PartitionRenameClause renameClause) throws DdlException {
- table.writeLock();
+ table.writeLockOrDdlException();
try {
if (table.getState() != OlapTableState.NORMAL) {
throw new DdlException("Table[" + table.getName() + "] is
under " + table.getState());
@@ -5806,8 +5847,7 @@ public class Catalog {
}
public void modifyDefaultDistributionBucketNum(Database db, OlapTable
olapTable, ModifyDistributionClause modifyDistributionClause) throws
DdlException {
- olapTable.writeLock();
-
+ olapTable.writeLockOrDdlException();
try {
if (olapTable.isColocateTable()) {
throw new DdlException("Cannot change default bucket number of
colocate table.");
@@ -6838,7 +6878,7 @@ public class Catalog {
// before replacing, we need to check again.
// Things may be changed outside the table lock.
olapTable = (OlapTable) db.getTableOrDdlException(copiedTbl.getId());
- olapTable.writeLock();
+ olapTable.writeLockOrDdlException();
try {
if (olapTable.getState() != OlapTableState.NORMAL) {
throw new DdlException("Table' state is not NORMAL: " +
olapTable.getState());
@@ -7040,7 +7080,7 @@ public class Catalog {
// Convert table's distribution type from random to hash.
// random distribution is no longer supported.
public void convertDistributionType(Database db, OlapTable tbl) throws
DdlException {
- tbl.writeLock();
+ tbl.writeLockOrDdlException();
try {
if (!tbl.convertRandomDistributionToHashDistribution()) {
throw new DdlException("Table " + tbl.getName() + " is not
random distributed");
@@ -7190,7 +7230,7 @@ public class Catalog {
}
Database db = this.getDbOrMetaException(meta.getDbId());
Table table = db.getTableOrMetaException(meta.getTableId());
- table.writeLock();
+ table.writeLockOrMetaException();
try {
Replica replica = tabletInvertedIndex.getReplica(tabletId,
backendId);
if (replica == null) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java
index 5d5e903..e3d61be 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java
@@ -366,15 +366,17 @@ public class CatalogRecycleBin extends MasterDaemon
implements Writable {
if (!table.getName().equals(tableName)) {
continue;
}
-
- db.createTable(table);
-
- iterator.remove();
- idToRecycleTime.remove(table.getId());
-
- // log
- RecoverInfo recoverInfo = new RecoverInfo(dbId, table.getId(),
-1L);
-
Catalog.getCurrentCatalog().getEditLog().logRecoverTable(recoverInfo);
+ table.writeLock();
+ try {
+ db.createTable(table);
+ iterator.remove();
+ idToRecycleTime.remove(table.getId());
+ // log
+ RecoverInfo recoverInfo = new RecoverInfo(dbId, table.getId(),
-1L);
+
Catalog.getCurrentCatalog().getEditLog().logRecoverTable(recoverInfo);
+ } finally {
+ table.writeUnlock();
+ }
LOG.info("recover db[{}] with table[{}]: {}", dbId, table.getId(),
table.getName());
return true;
}
@@ -391,13 +393,17 @@ public class CatalogRecycleBin extends MasterDaemon
implements Writable {
if (tableInfo.getTable().getId() != tableId) {
continue;
}
-
Preconditions.checkState(tableInfo.getDbId() == db.getId());
-
- db.createTable(tableInfo.getTable());
- iterator.remove();
- idToRecycleTime.remove(tableInfo.getTable().getId());
- LOG.info("replay recover table[{}]", tableId);
+ Table table = tableInfo.getTable();
+ table.writeLock();
+ try {
+ db.createTable(tableInfo.getTable());
+ iterator.remove();
+ idToRecycleTime.remove(tableInfo.getTable().getId());
+ LOG.info("replay recover table[{}]", tableId);
+ } finally {
+ table.writeUnlock();
+ }
break;
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java
index 1803c78..3817c34 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java
@@ -95,6 +95,8 @@ public class Database extends MetaObject implements Writable {
private volatile long replicaQuotaSize;
+ private volatile boolean isDropped;
+
public enum DbState {
NORMAL, LINK, MOVE
}
@@ -126,6 +128,14 @@ public class Database extends MetaObject implements
Writable {
this.dbEncryptKey = new DatabaseEncryptKey();
}
+ public void markDropped() {
+ isDropped = true;
+ }
+
+ public void unmarkDropped() {
+ isDropped = false;
+ }
+
public void readLock() {
this.rwLock.readLock().lock();
}
@@ -155,6 +165,26 @@ public class Database extends MetaObject implements
Writable {
return this.rwLock.writeLock().isHeldByCurrentThread();
}
+ public boolean writeLockIfExist() {
+ if (!isDropped) {
+ this.rwLock.writeLock().lock();
+ return true;
+ }
+ return false;
+ }
+
+ public <E extends Exception> void writeLockOrException(E e) throws E {
+ writeLock();
+ if (isDropped) {
+ writeUnlock();
+ throw e;
+ }
+ }
+
+ public void writeLockOrDdlException() throws DdlException {
+ writeLockOrException(new DdlException("unknown db, dbName=" +
fullQualifiedName));
+ }
+
public long getId() {
return id;
}
@@ -172,26 +202,16 @@ public class Database extends MetaObject implements
Writable {
}
}
- public void setDataQuotaWithLock(long newQuota) {
+ public void setDataQuota(long newQuota) {
Preconditions.checkArgument(newQuota >= 0L);
LOG.info("database[{}] set quota from {} to {}", fullQualifiedName,
dataQuotaBytes, newQuota);
- writeLock();
- try {
- this.dataQuotaBytes = newQuota;
- } finally {
- writeUnlock();
- }
+ this.dataQuotaBytes = newQuota;
}
- public void setReplicaQuotaWithLock(long newQuota) {
+ public void setReplicaQuota(long newQuota) {
Preconditions.checkArgument(newQuota >= 0L);
LOG.info("database[{}] set replica quota from {} to {}",
fullQualifiedName, replicaQuotaSize, newQuota);
- writeLock();
- try {
- this.replicaQuotaSize = newQuota;
- } finally {
- writeUnlock();
- }
+ this.replicaQuotaSize = newQuota;
}
public long getDataQuota() {
@@ -302,12 +322,12 @@ public class Database extends MetaObject implements
Writable {
}
// return pair <success?, table exist?>
- public Pair<Boolean, Boolean> createTableWithLock(Table table, boolean
isReplay, boolean setIfNotExist) {
+ public Pair<Boolean, Boolean> createTableWithLock(Table table, boolean
isReplay, boolean setIfNotExist) throws DdlException {
boolean result = true;
// if a table is already exists, then edit log won't be executed
// some caller of this method may need to know this message
boolean isTableExist = false;
- writeLock();
+ writeLockOrDdlException();
try {
String tableName = table.getName();
if (Catalog.isStoredTableNamesLowerCase()) {
@@ -349,18 +369,10 @@ public class Database extends MetaObject implements
Writable {
nameToTable.put(table.getName(), table);
lowerCaseToTableName.put(tableName.toLowerCase(), tableName);
}
+ table.unmarkDropped();
return result;
}
- public void dropTableWithLock(String tableName) {
- writeLock();
- try {
- dropTable(tableName);
- } finally {
- writeUnlock();
- }
- }
-
public void dropTable(String tableName) {
if (Catalog.isStoredTableNamesLowerCase()) {
tableName = tableName.toLowerCase();
@@ -370,6 +382,7 @@ public class Database extends MetaObject implements
Writable {
this.nameToTable.remove(tableName);
this.idToTable.remove(table.getId());
this.lowerCaseToTableName.remove(tableName.toLowerCase());
+ table.markDropped();
}
}
@@ -490,6 +503,7 @@ public class Database extends MetaObject implements
Writable {
public Table getTableOrMetaException(long tableId) throws
MetaNotFoundException {
return getTableOrException(tableId, t -> new
MetaNotFoundException("unknown table, tableId=" + t));
}
+
@SuppressWarnings("unchecked")
public <T extends Table> T getTableOrMetaException(String tableName,
TableType tableType) throws MetaNotFoundException {
Table table = getTableOrMetaException(tableName);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java
index 919a94c..a0c1fb1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java
@@ -17,8 +17,11 @@
package org.apache.doris.catalog;
+import org.apache.doris.alter.AlterCancelException;
import org.apache.doris.analysis.CreateTableStmt;
+import org.apache.doris.common.DdlException;
import org.apache.doris.common.FeMetaVersion;
+import org.apache.doris.common.MetaNotFoundException;
import org.apache.doris.common.io.Text;
import org.apache.doris.common.io.Writable;
import org.apache.doris.common.util.SqlUtils;
@@ -54,6 +57,8 @@ public class Table extends MetaObject implements Writable {
// assume that the time a lock is held by thread is less then 100ms
public static final long TRY_LOCK_TIMEOUT_MS = 100L;
+ public volatile boolean isDropped = false;
+
public enum TableType {
MYSQL,
ODBC,
@@ -135,6 +140,14 @@ public class Table extends MetaObject implements Writable {
this.createTime = Instant.now().getEpochSecond();
}
+ public void markDropped() {
+ isDropped = true;
+ }
+
+ public void unmarkDropped() {
+ isDropped = false;
+ }
+
public void readLock() {
this.rwLock.readLock().lock();
}
@@ -156,6 +169,14 @@ public class Table extends MetaObject implements Writable {
this.rwLock.writeLock().lock();
}
+ public boolean writeLockIfExist() {
+ if (!isDropped) {
+ this.rwLock.writeLock().lock();
+ return true;
+ }
+ return false;
+ }
+
public boolean tryWriteLock(long timeout, TimeUnit unit) {
try {
return this.rwLock.writeLock().tryLock(timeout, unit);
@@ -173,6 +194,52 @@ public class Table extends MetaObject implements Writable {
return this.rwLock.writeLock().isHeldByCurrentThread();
}
+ public <E extends Exception> void writeLockOrException(E e) throws E {
+ writeLock();
+ if (isDropped) {
+ writeUnlock();
+ throw e;
+ }
+ }
+
+ public void writeLockOrDdlException() throws DdlException {
+ writeLockOrException(new DdlException("unknown table, tableName=" +
name));
+ }
+
+ public void writeLockOrMetaException() throws MetaNotFoundException {
+ writeLockOrException(new MetaNotFoundException("unknown table,
tableName=" + name));
+ }
+
+ public void writeLockOrAlterCancelException() throws AlterCancelException {
+ writeLockOrException(new AlterCancelException("unknown table,
tableName=" + name));
+ }
+
+ public boolean tryWriteLockOrMetaException(long timeout, TimeUnit unit)
throws MetaNotFoundException {
+ return tryWriteLockOrException(timeout, unit, new
MetaNotFoundException("unknown table, tableName=" + name));
+ }
+
+ public <E extends Exception> boolean tryWriteLockOrException(long timeout,
TimeUnit unit, E e) throws E {
+ if (tryWriteLock(timeout, unit)) {
+ if (isDropped) {
+ writeUnlock();
+ throw e;
+ }
+ return true;
+ }
+ return false;
+ }
+
+ public boolean tryWriteLockIfExist(long timeout, TimeUnit unit) {
+ if (tryWriteLock(timeout, unit)) {
+ if (isDropped) {
+ writeUnlock();
+ return false;
+ }
+ return true;
+ }
+ return false;
+ }
+
public boolean isTypeRead() {
return isTypeRead;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletStatMgr.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletStatMgr.java
index ed02c39..9755822 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletStatMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletStatMgr.java
@@ -91,9 +91,10 @@ public class TabletStatMgr extends MasterDaemon {
if (table.getType() != TableType.OLAP) {
continue;
}
-
OlapTable olapTable = (OlapTable) table;
- table.writeLock();
+ if (!table.writeLockIfExist()) {
+ continue;
+ }
try {
for (Partition partition : olapTable.getAllPartitions()) {
long version = partition.getVisibleVersion();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java
b/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java
index 4a6fc1d..e85b620 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java
@@ -406,7 +406,9 @@ public class DynamicPartitionScheduler extends MasterDaemon
{
}
for (DropPartitionClause dropPartitionClause :
dropPartitionClauses) {
- olapTable.writeLock();
+ if (!olapTable.writeLockIfExist()) {
+ continue;
+ }
try {
Catalog.getCurrentCatalog().dropPartition(db, olapTable,
dropPartitionClause);
clearDropPartitionFailedMsg(olapTable.getId());
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/clone/TabletSchedCtx.java
b/fe/fe-core/src/main/java/org/apache/doris/clone/TabletSchedCtx.java
index c67c3cc..7af0af3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/clone/TabletSchedCtx.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/clone/TabletSchedCtx.java
@@ -678,8 +678,7 @@ public class TabletSchedCtx implements
Comparable<TabletSchedCtx> {
Database db = Catalog.getCurrentCatalog().getDbNullable(dbId);
if (db != null) {
Table table = db.getTableNullable(tblId);
- if (table != null) {
- table.writeLock();
+ if (table != null && table.writeLockIfExist()) {
try {
List<Replica> cloneReplicas = Lists.newArrayList();
tablet.getReplicas().stream().filter(r -> r.getState()
== ReplicaState.CLONE).forEach(r -> {
@@ -833,10 +832,9 @@ public class TabletSchedCtx implements
Comparable<TabletSchedCtx> {
}
// 1. check the tablet status first
- Database db = Catalog.getCurrentCatalog().getDbOrException(dbId, s ->
new SchedException(Status.UNRECOVERABLE, "db does not exist"));
- OlapTable olapTable = (OlapTable) db.getTableOrException(tblId, s ->
new SchedException(Status.UNRECOVERABLE, "tbl does not exist"));
-
- olapTable.writeLock();
+ Database db = Catalog.getCurrentCatalog().getDbOrException(dbId, s ->
new SchedException(Status.UNRECOVERABLE, "db " + dbId + " does not exist"));
+ OlapTable olapTable = (OlapTable) db.getTableOrException(tblId, s ->
new SchedException(Status.UNRECOVERABLE, "tbl " + tabletId + " does not
exist"));
+ olapTable.writeLockOrException(new
SchedException(Status.UNRECOVERABLE, "table " + olapTable.getName() + " does
not exist"));
try {
Partition partition = olapTable.getPartition(partitionId);
if (partition == null) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/clone/TabletScheduler.java
b/fe/fe-core/src/main/java/org/apache/doris/clone/TabletScheduler.java
index 1215da2..39573e1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/clone/TabletScheduler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/clone/TabletScheduler.java
@@ -469,10 +469,11 @@ public class TabletScheduler extends MasterDaemon {
stat.counterTabletScheduled.incrementAndGet();
Pair<TabletStatus, TabletSchedCtx.Priority> statusPair;
- // check this tablet again
- Database db = catalog.getDbOrException(tabletCtx.getDbId(), s -> new
SchedException(Status.UNRECOVERABLE, "db does not exist"));
- OlapTable tbl = (OlapTable)
db.getTableOrException(tabletCtx.getTblId(), s -> new
SchedException(Status.UNRECOVERABLE, "tbl does not exist"));
- tbl.writeLock();
+ Database db =
Catalog.getCurrentCatalog().getDbOrException(tabletCtx.getDbId(),
+ s -> new SchedException(Status.UNRECOVERABLE, "db " +
tabletCtx.getDbId() + " does not exist"));
+ OlapTable tbl = (OlapTable)
db.getTableOrException(tabletCtx.getTblId(),
+ s -> new SchedException(Status.UNRECOVERABLE, "tbl " +
tabletCtx.getTblId() + " does not exist"));
+ tbl.writeLockOrException(new SchedException(Status.UNRECOVERABLE,
"table " + tbl.getName() + " does not exist"));
try {
boolean isColocateTable =
colocateTableIndex.isColocateTable(tbl.getId());
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/common/util/MetaLockUtils.java
b/fe/fe-core/src/main/java/org/apache/doris/common/util/MetaLockUtils.java
index ed0e06a..42346a9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/util/MetaLockUtils.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/MetaLockUtils.java
@@ -19,6 +19,7 @@ package org.apache.doris.common.util;
import org.apache.doris.catalog.Database;
import org.apache.doris.catalog.Table;
+import org.apache.doris.common.MetaNotFoundException;
import java.util.List;
import java.util.concurrent.TimeUnit;
@@ -60,13 +61,33 @@ public class MetaLockUtils {
}
}
- public static boolean tryWriteLockTables(List<Table> tableList, long
timeout, TimeUnit unit) {
+ public static void writeLockTablesOrMetaException(List<Table> tableList)
throws MetaNotFoundException {
for (int i = 0; i < tableList.size(); i++) {
- if (!tableList.get(i).tryWriteLock(timeout, unit)) {
+ try {
+ tableList.get(i).writeLockOrMetaException();
+ } catch (MetaNotFoundException e) {
for (int j = i - 1; j >= 0; j--) {
tableList.get(j).writeUnlock();
}
- return false;
+ throw e;
+ }
+ }
+ }
+
+ public static boolean tryWriteLockTablesOrMetaException(List<Table>
tableList, long timeout, TimeUnit unit) throws MetaNotFoundException {
+ for (int i = 0; i < tableList.size(); i++) {
+ try {
+ if (!tableList.get(i).tryWriteLockOrMetaException(timeout,
unit)) {
+ for (int j = i - 1; j >= 0; j--) {
+ tableList.get(j).writeUnlock();
+ }
+ return false;
+ }
+ } catch (MetaNotFoundException e) {
+ for (int j = i - 1; j >= 0; j--) {
+ tableList.get(j).writeUnlock();
+ }
+ throw e;
}
}
return true;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/consistency/CheckConsistencyJob.java
b/fe/fe-core/src/main/java/org/apache/doris/consistency/CheckConsistencyJob.java
index d0cbb49..4d32786 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/consistency/CheckConsistencyJob.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/consistency/CheckConsistencyJob.java
@@ -215,7 +215,10 @@ public class CheckConsistencyJob {
if (state != JobState.RUNNING) {
// failed to send task. set tablet's checked version and version
hash to avoid choosing it again
- table.writeLock();
+ if (!table.writeLockIfExist()) {
+ LOG.debug("table[{}] does not exist", tabletMeta.getTableId());
+ return false;
+ }
try {
tablet.setCheckedVersion(checkedVersion, checkedVersionHash);
} finally {
@@ -261,11 +264,10 @@ public class CheckConsistencyJob {
boolean isConsistent = true;
Table table = db.getTableNullable(tabletMeta.getTableId());
- if (table == null) {
+ if (table == null || !table.writeLockIfExist()) {
LOG.warn("table[{}] does not exist", tabletMeta.getTableId());
return -1;
}
- table.writeLock();
try {
OlapTable olapTable = (OlapTable) table;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/http/rest/RowCountAction.java
b/fe/fe-core/src/main/java/org/apache/doris/http/rest/RowCountAction.java
index e134fc3..4648065 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/http/rest/RowCountAction.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/http/rest/RowCountAction.java
@@ -74,7 +74,7 @@ public class RowCountAction extends RestBaseAction {
Catalog catalog = Catalog.getCurrentCatalog();
Database db = catalog.getDbOrDdlException(dbName);
OlapTable olapTable = db.getOlapTableOrDdlException(tableName);
- olapTable.writeLock();
+ olapTable.writeLockOrDdlException();
try {
for (Partition partition : olapTable.getAllPartitions()) {
long version = partition.getVisibleVersion();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/http/rest/TableRowCountAction.java
b/fe/fe-core/src/main/java/org/apache/doris/http/rest/TableRowCountAction.java
index 3b75547..1b4a8ed 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/http/rest/TableRowCountAction.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/http/rest/TableRowCountAction.java
@@ -84,13 +84,13 @@ public class TableRowCountAction extends RestBaseAction {
throw new DorisHttpException(HttpResponseStatus.BAD_REQUEST,
e.getMessage());
}
- table.writeLock();
+ table.readLock();
try {
OlapTable olapTable = (OlapTable) table;
resultMap.put("status", 200);
resultMap.put("size", olapTable.proximateRowCount());
} finally {
- table.writeUnlock();
+ table.readUnlock();
}
} catch (DorisHttpException e) {
// status code should conforms to HTTP semantic
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/TableRowCountAction.java
b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/TableRowCountAction.java
index 8d9d7f9..a473bc0 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/TableRowCountAction.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/TableRowCountAction.java
@@ -74,7 +74,7 @@ public class TableRowCountAction extends RestBaseController {
resultMap.put("status", 200);
resultMap.put("size", olapTable.proximateRowCount());
} finally {
- olapTable.readLock();
+ olapTable.readUnlock();
}
} catch (DorisHttpException e) {
// status code should conforms to HTTP semantic
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBJEJournal.java
b/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBJEJournal.java
index c4c1b71..e4b4ac6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBJEJournal.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBJEJournal.java
@@ -157,7 +157,6 @@ public class BDBJEJournal implements Journal {
} catch (InterruptedException e1) {
e1.printStackTrace();
}
- continue;
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/LoadChecker.java
b/fe/fe-core/src/main/java/org/apache/doris/load/LoadChecker.java
index 3793acc..17fdea6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/LoadChecker.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/LoadChecker.java
@@ -331,7 +331,12 @@ public class LoadChecker extends MasterDaemon {
// concurrent problems
// table in tables are ordered.
- MetaLockUtils.writeLockTables(tables);
+ try {
+ MetaLockUtils.writeLockTablesOrMetaException(tables);
+ } catch (UserException e) {
+ load.cancelLoadJob(job, CancelType.LOAD_RUN_FAIL, "table does not
exist. dbId: " + job.getDbId() + ", err: " + e.getMessage());
+ return;
+ }
try {
TabletInvertedIndex invertedIndex =
Catalog.getCurrentInvertedIndex();
for (Replica replica : job.getFinishedReplicas()) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java
b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java
index 6f03f4f..cefd28e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java
@@ -277,6 +277,7 @@ public class BrokerLoadJob extends BulkLoadJob {
try {
db = getDb();
tableList =
db.getTablesOnIdOrderOrThrowException(Lists.newArrayList(fileGroupAggInfo.getAllTableIds()));
+ MetaLockUtils.writeLockTablesOrMetaException(tableList);
} catch (MetaNotFoundException e) {
LOG.warn(new LogBuilder(LogKey.LOAD_JOB, id)
.add("database_id", dbId)
@@ -285,7 +286,6 @@ public class BrokerLoadJob extends BulkLoadJob {
cancelJobWithoutCheck(new
FailMsg(FailMsg.CancelType.LOAD_RUN_FAIL, e.getMessage()), true, true);
return;
}
- MetaLockUtils.writeLockTables(tableList);
try {
LOG.info(new LogBuilder(LogKey.LOAD_JOB, id)
.add("txn_id", transactionId)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/SparkLoadJob.java
b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/SparkLoadJob.java
index 9185437..d5e3136 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/SparkLoadJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/SparkLoadJob.java
@@ -626,7 +626,7 @@ public class SparkLoadJob extends BulkLoadJob {
.build());
Database db = getDb();
List<Table> tableList =
db.getTablesOnIdOrderOrThrowException(Lists.newArrayList(tableToLoadPartitions.keySet()));
- MetaLockUtils.writeLockTables(tableList);
+ MetaLockUtils.writeLockTablesOrMetaException(tableList);
try {
Catalog.getCurrentGlobalTransactionMgr().commitTransaction(
dbId, tableList, transactionId, commitInfos,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/master/MasterImpl.java
b/fe/fe-core/src/main/java/org/apache/doris/master/MasterImpl.java
index 55de348..fdf971e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/master/MasterImpl.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/master/MasterImpl.java
@@ -338,12 +338,11 @@ public class MasterImpl {
LOG.debug("push report state: {}", pushState.name());
OlapTable olapTable = (OlapTable) db.getTableNullable(tableId);
- if (olapTable == null) {
+ if (olapTable == null || !olapTable.writeLockIfExist()) {
AgentTaskQueue.removeTask(backendId, TTaskType.REALTIME_PUSH,
signature);
LOG.warn("finish push replica error, cannot find table[" + tableId
+ "] when push finished");
return;
}
- olapTable.writeLock();
try {
Partition partition = olapTable.getPartition(partitionId);
if (partition == null) {
@@ -552,12 +551,11 @@ public class MasterImpl {
LOG.debug("push report state: {}", pushState.name());
OlapTable olapTable = (OlapTable) db.getTableNullable(tableId);
- if (olapTable == null) {
+ if (olapTable == null || !olapTable.writeLockIfExist()) {
AgentTaskQueue.removeTask(backendId, TTaskType.REALTIME_PUSH,
signature);
LOG.warn("finish push replica error, cannot find table[" + tableId
+ "] when push finished");
return;
}
- olapTable.writeLock();
try {
Partition partition = olapTable.getPartition(partitionId);
if (partition == null) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java
b/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java
index 3775afb..fabc2c7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/master/ReportHandler.java
@@ -416,10 +416,9 @@ public class ReportHandler extends Daemon {
long tabletId = tabletIds.get(i);
long tableId = tabletMeta.getTableId();
OlapTable olapTable = (OlapTable) db.getTableNullable(tableId);
- if (olapTable == null) {
+ if (olapTable == null || !olapTable.writeLockIfExist()) {
continue;
}
- olapTable.writeLock();
try {
long partitionId = tabletMeta.getPartitionId();
Partition partition = olapTable.getPartition(partitionId);
@@ -545,10 +544,9 @@ public class ReportHandler extends Daemon {
long tabletId = tabletIds.get(i);
long tableId = tabletMeta.getTableId();
OlapTable olapTable = (OlapTable) db.getTableNullable(tableId);
- if (olapTable == null) {
+ if (olapTable == null || !olapTable.writeLockIfExist()) {
continue;
}
- olapTable.writeLock();
try {
long partitionId = tabletMeta.getPartitionId();
Partition partition = olapTable.getPartition(partitionId);
@@ -827,10 +825,9 @@ public class ReportHandler extends Daemon {
long tabletId = tabletIds.get(i);
long tableId = tabletMeta.getTableId();
OlapTable olapTable = (OlapTable) db.getTableNullable(tableId);
- if (olapTable == null) {
+ if (olapTable == null || !olapTable.writeLockIfExist()) {
continue;
}
- olapTable.writeLock();
try {
long partitionId = tabletMeta.getPartitionId();
Partition partition = olapTable.getPartition(partitionId);
@@ -949,7 +946,7 @@ public class ReportHandler extends Daemon {
Database db = Catalog.getCurrentCatalog().getDbOrMetaException(dbId);
OlapTable olapTable = db.getTableOrMetaException(tableId,
Table.TableType.OLAP);
- olapTable.writeLock();
+ olapTable.writeLockOrMetaException();
try {
Partition partition = olapTable.getPartition(partitionId);
if (partition == null) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java
b/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java
index 5cc8002..87cd981 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java
@@ -690,23 +690,8 @@ public class DatabaseTransactionMgr {
}
}
List<Long> tableIdList = transactionState.getTableIdList();
- // to be compatiable with old meta version, table List may be empty
- if (tableIdList.isEmpty()) {
- readLock();
- try {
- for (TableCommitInfo tableCommitInfo :
transactionState.getIdToTableCommitInfos().values()) {
- long tableId = tableCommitInfo.getTableId();
- if (!tableIdList.contains(tableId)) {
- tableIdList.add(tableId);
- }
- }
- } finally {
- readUnlock();
- }
- }
-
- List<Table> tableList =
db.getTablesOnIdOrderWithIgnoringWrongTableId(tableIdList);
- MetaLockUtils.writeLockTables(tableList);
+ List<Table> tableList =
db.getTablesOnIdOrderOrThrowException(tableIdList);
+ MetaLockUtils.writeLockTablesOrMetaException(tableList);
try {
boolean hasError = false;
Iterator<TableCommitInfo> tableCommitInfoIterator =
transactionState.getIdToTableCommitInfos().values().iterator();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/transaction/GlobalTransactionMgr.java
b/fe/fe-core/src/main/java/org/apache/doris/transaction/GlobalTransactionMgr.java
index 198b5a4..6aedf62 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/transaction/GlobalTransactionMgr.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/transaction/GlobalTransactionMgr.java
@@ -211,7 +211,7 @@ public class GlobalTransactionMgr implements Writable {
throws UserException {
StopWatch stopWatch = new StopWatch();
stopWatch.start();
- if (!MetaLockUtils.tryWriteLockTables(tableList, timeoutMillis,
TimeUnit.MILLISECONDS)) {
+ if (!MetaLockUtils.tryWriteLockTablesOrMetaException(tableList,
timeoutMillis, TimeUnit.MILLISECONDS)) {
throw new UserException("get tableList write lock timeout,
tableList=(" + StringUtils.join(tableList, ",") + ")");
}
try {
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/catalog/DatabaseTest.java
b/fe/fe-core/src/test/java/org/apache/doris/catalog/DatabaseTest.java
index 161d202..05a8fb6 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/catalog/DatabaseTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/DatabaseTest.java
@@ -17,7 +17,9 @@
package org.apache.doris.catalog;
+import org.apache.doris.alter.AlterCancelException;
import org.apache.doris.catalog.MaterializedIndex.IndexState;
+import org.apache.doris.common.DdlException;
import org.apache.doris.common.ExceptionChecker;
import org.apache.doris.common.FeConstants;
import org.apache.doris.common.MetaNotFoundException;
@@ -93,10 +95,27 @@ public class DatabaseTest {
db.writeLock();
try {
- Assert.assertTrue(db.tryWriteLock(0, TimeUnit.SECONDS));
+ Assert.assertTrue(db.tryWriteLock(1000, TimeUnit.SECONDS));
+ db.writeUnlock();
} finally {
db.writeUnlock();
}
+
+ db.markDropped();
+ Assert.assertFalse(db.writeLockIfExist());
+ Assert.assertFalse(db.isWriteLockHeldByCurrentThread());
+ db.unmarkDropped();
+ Assert.assertTrue(db.writeLockIfExist());
+ Assert.assertTrue(db.isWriteLockHeldByCurrentThread());
+ db.writeUnlock();
+ }
+
+ @Test
+ public void lockTestWithException() {
+ db.markDropped();
+ ExceptionChecker.expectThrowsWithMsg(DdlException.class,
+ "errCode = 2, detailMessage = unknown db, dbName=dbTest", ()
-> db.writeLockOrDdlException());
+ db.unmarkDropped();
}
@Test
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/catalog/InfoSchemaDbTest.java
b/fe/fe-core/src/test/java/org/apache/doris/catalog/InfoSchemaDbTest.java
index 0a808c9..422a7ac 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/catalog/InfoSchemaDbTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/InfoSchemaDbTest.java
@@ -17,6 +17,7 @@
package org.apache.doris.catalog;
+import org.apache.doris.common.DdlException;
import org.junit.Assert;
import org.junit.Test;
@@ -24,7 +25,7 @@ import java.io.IOException;
public class InfoSchemaDbTest {
@Test
- public void testNormal() throws IOException {
+ public void testNormal() throws IOException, DdlException {
Database db = new InfoSchemaDb();
Assert.assertFalse(db.createTable(null));
diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/TableTest.java
b/fe/fe-core/src/test/java/org/apache/doris/catalog/TableTest.java
index 7187384..6b0aa03 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/catalog/TableTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/TableTest.java
@@ -17,7 +17,11 @@
package org.apache.doris.catalog;
+import org.apache.doris.alter.AlterCancelException;
+import org.apache.doris.common.DdlException;
+import org.apache.doris.common.ExceptionChecker;
import org.apache.doris.common.FeConstants;
+import org.apache.doris.common.MetaNotFoundException;
import org.apache.doris.common.jmockit.Deencapsulation;
import org.apache.doris.thrift.TStorageType;
@@ -51,6 +55,7 @@ public class TableTest {
fakeCatalog = new FakeCatalog();
catalog = Deencapsulation.newInstance(Catalog.class);
table = new Table(Table.TableType.OLAP);
+ table.setName("test");
FakeCatalog.setCatalog(catalog);
FakeCatalog.setMetaVersion(FeConstants.meta_version);
}
@@ -64,14 +69,41 @@ public class TableTest {
table.readUnlock();
}
+ Assert.assertFalse(table.isWriteLockHeldByCurrentThread());
table.writeLock();
try {
- Assert.assertTrue(table.tryWriteLock(0, TimeUnit.SECONDS));
+ Assert.assertTrue(table.tryWriteLock(1000, TimeUnit.SECONDS));
+ Assert.assertTrue(table.isWriteLockHeldByCurrentThread());
+ table.writeUnlock();
} finally {
table.writeUnlock();
+ Assert.assertFalse(table.isWriteLockHeldByCurrentThread());
}
+
+ Assert.assertFalse(table.isWriteLockHeldByCurrentThread());
+ table.markDropped();
+ Assert.assertFalse(table.writeLockIfExist());
+ Assert.assertFalse(table.isWriteLockHeldByCurrentThread());
+ table.unmarkDropped();
+ Assert.assertTrue(table.writeLockIfExist());
+ Assert.assertTrue(table.writeLockIfExist());
+ Assert.assertTrue(table.isWriteLockHeldByCurrentThread());
+ table.writeUnlock();
}
+ @Test
+ public void lockTestWithException() {
+ table.markDropped();
+ ExceptionChecker.expectThrowsWithMsg(DdlException.class,
+ "errCode = 2, detailMessage = unknown table, tableName=test",
() -> table.writeLockOrDdlException());
+ ExceptionChecker.expectThrowsWithMsg(MetaNotFoundException.class,
+ "errCode = 7, detailMessage = unknown table, tableName=test",
() -> table.writeLockOrMetaException());
+ ExceptionChecker.expectThrowsWithMsg(AlterCancelException.class,
+ "errCode = 2, detailMessage = unknown table, tableName=test",
() -> table.writeLockOrAlterCancelException());
+ ExceptionChecker.expectThrowsWithMsg(MetaNotFoundException.class,
+ "errCode = 7, detailMessage = unknown table, tableName=test",
() -> table.tryWriteLockOrMetaException(1000, TimeUnit.MILLISECONDS));
+ table.unmarkDropped();
+ }
@Test
public void testSerialization() throws Exception {
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/common/util/MetaLockUtilsTest.java
b/fe/fe-core/src/test/java/org/apache/doris/common/util/MetaLockUtilsTest.java
index 47fb1ff..7abe526 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/common/util/MetaLockUtilsTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/common/util/MetaLockUtilsTest.java
@@ -20,14 +20,20 @@ package org.apache.doris.common.util;
import com.google.common.collect.Lists;
import org.apache.doris.catalog.Database;
import org.apache.doris.catalog.Table;
+import org.apache.doris.common.MetaNotFoundException;
import org.junit.Assert;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.ExpectedException;
import java.util.List;
import java.util.concurrent.TimeUnit;
public class MetaLockUtilsTest {
+ @Rule
+ public ExpectedException expectedException = ExpectedException.none();
+
@Test
public void testReadLockDatabases() {
List<Database> databaseList = Lists.newArrayList(new Database(), new
Database());
@@ -55,7 +61,7 @@ public class MetaLockUtilsTest {
}
@Test
- public void testWriteLockTables() {
+ public void testWriteLockTables() throws MetaNotFoundException {
List<Table> tableList = Lists.newArrayList(new
Table(Table.TableType.OLAP), new Table(Table.TableType.OLAP));
MetaLockUtils.writeLockTables(tableList);
Assert.assertTrue(tableList.get(0).isWriteLockHeldByCurrentThread());
@@ -63,14 +69,64 @@ public class MetaLockUtilsTest {
MetaLockUtils.writeUnlockTables(tableList);
Assert.assertFalse(tableList.get(0).isWriteLockHeldByCurrentThread());
Assert.assertFalse(tableList.get(1).isWriteLockHeldByCurrentThread());
- Assert.assertTrue(MetaLockUtils.tryWriteLockTables(tableList, 1,
TimeUnit.MILLISECONDS));
+
Assert.assertTrue(MetaLockUtils.tryWriteLockTablesOrMetaException(tableList, 1,
TimeUnit.MILLISECONDS));
Assert.assertTrue(tableList.get(0).isWriteLockHeldByCurrentThread());
Assert.assertTrue(tableList.get(1).isWriteLockHeldByCurrentThread());
MetaLockUtils.writeUnlockTables(tableList);
tableList.get(1).readLock();
- Assert.assertFalse(MetaLockUtils.tryWriteLockTables(tableList, 1,
TimeUnit.MILLISECONDS));
+
Assert.assertFalse(MetaLockUtils.tryWriteLockTablesOrMetaException(tableList,
1, TimeUnit.MILLISECONDS));
Assert.assertFalse(tableList.get(0).isWriteLockHeldByCurrentThread());
Assert.assertFalse(tableList.get(1).isWriteLockHeldByCurrentThread());
tableList.get(1).readUnlock();
}
+
+ @Test
+ public void testWriteLockTablesWithMetaNotFoundException() throws
MetaNotFoundException {
+ List<Table> tableList = Lists.newArrayList();
+ Table table1 = new Table(Table.TableType.OLAP);
+ Table table2 = new Table(Table.TableType.OLAP);
+ table2.setName("test2");
+ tableList.add(table1);
+ tableList.add(table2);
+ MetaLockUtils.writeLockTablesOrMetaException(tableList);
+ Assert.assertTrue(table1.isWriteLockHeldByCurrentThread());
+ Assert.assertTrue(table2.isWriteLockHeldByCurrentThread());
+ MetaLockUtils.writeUnlockTables(tableList);
+ Assert.assertFalse(table1.isWriteLockHeldByCurrentThread());
+ Assert.assertFalse(table2.isWriteLockHeldByCurrentThread());
+ table2.markDropped();
+ expectedException.expect(MetaNotFoundException.class);
+ expectedException.expectMessage("errCode = 7, detailMessage = unknown
table, tableName=test2");
+ try {
+ MetaLockUtils.writeLockTablesOrMetaException(tableList);
+ } finally {
+ Assert.assertFalse(table1.isWriteLockHeldByCurrentThread());
+ Assert.assertFalse(table2.isWriteLockHeldByCurrentThread());
+ }
+ }
+
+ @Test
+ public void testTryWriteLockTablesWithMetaNotFoundException() throws
MetaNotFoundException {
+ List<Table> tableList = Lists.newArrayList();
+ Table table1 = new Table(Table.TableType.OLAP);
+ Table table2 = new Table(Table.TableType.OLAP);
+ table2.setName("test2");
+ tableList.add(table1);
+ tableList.add(table2);
+ MetaLockUtils.tryWriteLockTablesOrMetaException(tableList, 1000,
TimeUnit.MILLISECONDS);
+ Assert.assertTrue(table1.isWriteLockHeldByCurrentThread());
+ Assert.assertTrue(table2.isWriteLockHeldByCurrentThread());
+ MetaLockUtils.writeUnlockTables(tableList);
+ Assert.assertFalse(table1.isWriteLockHeldByCurrentThread());
+ Assert.assertFalse(table2.isWriteLockHeldByCurrentThread());
+ table2.markDropped();
+ expectedException.expect(MetaNotFoundException.class);
+ expectedException.expectMessage("errCode = 7, detailMessage = unknown
table, tableName=test2");
+ try {
+ MetaLockUtils.tryWriteLockTablesOrMetaException(tableList, 1000,
TimeUnit.MILLISECONDS);
+ } finally {
+ Assert.assertFalse(table1.isWriteLockHeldByCurrentThread());
+ Assert.assertFalse(table2.isWriteLockHeldByCurrentThread());
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
