This is an automated email from the ASF dual-hosted git repository.
michaelsmith pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
The following commit(s) were added to refs/heads/master by this push:
new ec31324eb IMPALA-14502: Not tracking metrics in IncompleteTable
ec31324eb is described below
commit ec31324eb532652346c00b36ec42ca69e39b5d64
Author: stiga-huang <[email protected]>
AuthorDate: Wed Oct 15 13:33:41 2025 +0800
IMPALA-14502: Not tracking metrics in IncompleteTable
Tables that are in unloaded state are represented as IncompleteTable.
Table level metrics of them won't be used at all but occupy around 7KB
of memory for each table. This is a significant amount comparing to the
table name strings.
This patch skips initializing these metrics for IncompleteTable to save
memory usage. This reduces the initial memory requirement to launch
catalogd.
To avoid other codes unintentionally add new metrics to IncompleteTable,
overrides all Table methods that use metrics_ to return simple results,
e.g. IncompleteTable.getMedianTableLoadingTime() always returns 0.
IncompleteTable.getMetrics() shouldn't be used. Added a Precondition
check for this.
Tests:
- Verified in a heap dump file after loading 1.3M IncompleteTables that
the heap usage reduces to 2GB and only few instances of
com.codahale.metrics.Timer are created. Previously catalogd OOM in a
heap size of 18GB when running global IM, and the number of
com.codahale.metrics.Timer instances is similar to the number of
IncompleteTables.
- Passed CORE tests.
Change-Id: If0fcfeab99bbfbefe618d0abf7f2482a0cc5ef9f
Reviewed-on: http://gerrit.cloudera.org:8080/23547
Reviewed-by: Riza Suminto <[email protected]>
Tested-by: Impala Public Jenkins <[email protected]>
Reviewed-by: Michael Smith <[email protected]>
---
.../org/apache/impala/catalog/IncompleteTable.java | 46 ++++++++++++++++++++++
.../impala/catalog/events/MetastoreEvents.java | 2 +-
.../catalog/events/RenameTableBarrierEvent.java | 3 +-
3 files changed, 49 insertions(+), 2 deletions(-)
diff --git a/fe/src/main/java/org/apache/impala/catalog/IncompleteTable.java
b/fe/src/main/java/org/apache/impala/catalog/IncompleteTable.java
index 9dac3a20f..1bf085014 100644
--- a/fe/src/main/java/org/apache/impala/catalog/IncompleteTable.java
+++ b/fe/src/main/java/org/apache/impala/catalog/IncompleteTable.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.hive.metastore.IMetaStoreClient;
import org.apache.impala.common.ImpalaException;
import org.apache.impala.common.JniUtil;
+import org.apache.impala.common.Metrics;
import org.apache.impala.thrift.TCatalogObjectType;
import org.apache.impala.thrift.TErrorCode;
import org.apache.impala.thrift.TGetPartialCatalogObjectRequest;
@@ -217,4 +218,49 @@ public class IncompleteTable extends Table implements
FeIncompleteTable {
Throwables.propagateIfPossible(cause_, TableLoadingException.class);
throw new TableLoadingException(cause_.getMessage());
}
+
+ /**
+ * Don't initialize any metrics to save memory space.
+ */
+ @Override
+ public void initMetrics() {}
+
+ @Override
+ public Metrics getMetrics() {
+ Preconditions.checkState(false, "getMetrics() on IncompleteTable shouldn't
be used");
+ return null;
+ }
+
+ @Override
+ public long getMedianTableLoadingTime() { return 0; }
+
+ @Override
+ public long getMaxTableLoadingTime() { return 0; }
+
+ @Override
+ public long get75TableLoadingTime() { return 0; }
+
+ @Override
+ public long get95TableLoadingTime() { return 0; }
+
+ @Override
+ public long get99TableLoadingTime() { return 0; }
+
+ @Override
+ public long getTableLoadingCounts() { return 0; }
+
+ @Override
+ public void updateHMSLoadTableSchemaTime(long hmsLoadTimeNS) {}
+
+ @Override
+ public boolean removeFromVersionsForInflightEvents(
+ boolean isInsertEvent, long versionNumber) {
+ return false;
+ }
+
+ @Override
+ public boolean addToVersionsForInflightEvents(
+ boolean isInsertEvent, long versionNumber) {
+ return false;
+ }
}
diff --git
a/fe/src/main/java/org/apache/impala/catalog/events/MetastoreEvents.java
b/fe/src/main/java/org/apache/impala/catalog/events/MetastoreEvents.java
index e3f35672a..938a7dd32 100644
--- a/fe/src/main/java/org/apache/impala/catalog/events/MetastoreEvents.java
+++ b/fe/src/main/java/org/apache/impala/catalog/events/MetastoreEvents.java
@@ -1608,7 +1608,7 @@ public class MetastoreEvents {
protected void process() throws MetastoreNotificationException,
CatalogException {
Timer.Context context = null;
org.apache.impala.catalog.Table tbl = catalog_.getTableNoThrow(dbName_,
tblName_);
- if (tbl != null) {
+ if (tbl != null && !(tbl instanceof IncompleteTable)) {
context =
tbl.getMetrics().getTimer(TBL_EVENTS_PROCESS_DURATION).time();
}
try {
diff --git
a/fe/src/main/java/org/apache/impala/catalog/events/RenameTableBarrierEvent.java
b/fe/src/main/java/org/apache/impala/catalog/events/RenameTableBarrierEvent.java
index b3bdc5d68..70fb3b844 100644
---
a/fe/src/main/java/org/apache/impala/catalog/events/RenameTableBarrierEvent.java
+++
b/fe/src/main/java/org/apache/impala/catalog/events/RenameTableBarrierEvent.java
@@ -23,6 +23,7 @@ import com.google.common.base.Preconditions;
import org.apache.impala.catalog.CatalogException;
import org.apache.impala.catalog.Db;
+import org.apache.impala.catalog.IncompleteTable;
import org.apache.impala.catalog.Table;
import
org.apache.impala.catalog.events.MetastoreEvents.DerivedMetastoreEventContext;
import
org.apache.impala.catalog.events.MetastoreEvents.DerivedMetastoreTableEvent;
@@ -198,7 +199,7 @@ public class RenameTableBarrierEvent extends
DerivedMetastoreTableEvent {
Db db = catalog_.getDb(getDbName());
Table table = null;
if (db != null) table = db.getTable(getTableName());
- if (table != null) {
+ if (table != null && !(table instanceof IncompleteTable)) {
context =
table.getMetrics().getTimer(TBL_EVENTS_PROCESS_DURATION).time();
}
String fqTableName = getFullyQualifiedTblName();
