This is an automated email from the ASF dual-hosted git repository.
stigahuang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
The following commit(s) were added to refs/heads/master by this push:
new ac18820e7 IMPALA-12579: Add logs for loading table list in catalogd
startup
ac18820e7 is described below
commit ac18820e7be39f1dc95c447f791b56aa012daee9
Author: stiga-huang <[email protected]>
AuthorDate: Tue Nov 28 16:45:51 2023 +0800
IMPALA-12579: Add logs for loading table list in catalogd startup
Catalogd will load db and table list from HMS at startup. It could take
long for large dbs. This patch adds logs for this step so users won't
be confused and thinking catalogd is "hanging". Also improve the
function loading logs to show the number of loaded functions.
Tests:
- Verified the logs locally
Change-Id: I280c7b490bc57b38d867aeff29ae8ffcfec40c81
Reviewed-on: http://gerrit.cloudera.org:8080/20734
Reviewed-by: Andrew Sherman <[email protected]>
Tested-by: Impala Public Jenkins <[email protected]>
---
.../org/apache/impala/catalog/CatalogServiceCatalog.java | 12 +++++++++---
1 file changed, 9 insertions(+), 3 deletions(-)
diff --git
a/fe/src/main/java/org/apache/impala/catalog/CatalogServiceCatalog.java
b/fe/src/main/java/org/apache/impala/catalog/CatalogServiceCatalog.java
index 12cbd7b87..bddc3f321 100644
--- a/fe/src/main/java/org/apache/impala/catalog/CatalogServiceCatalog.java
+++ b/fe/src/main/java/org/apache/impala/catalog/CatalogServiceCatalog.java
@@ -1771,8 +1771,7 @@ public class CatalogServiceCatalog extends Catalog {
db.addFunction(f, false);
f.setCatalogVersion(incrementAndGetCatalogVersion());
}
-
- LOG.info("Loaded native functions for database: " + db.getName());
+ LOG.info("Loaded {} native functions for database: {}", funcs.size(),
db.getName());
}
/**
@@ -1788,6 +1787,7 @@ public class CatalogServiceCatalog extends Catalog {
return;
}
LOG.info("Loading Java functions for database: " + db.getName());
+ int numFuncs = 0;
for (org.apache.hadoop.hive.metastore.api.Function function: functions) {
try {
HiveJavaFunctionFactoryImpl factory =
@@ -1796,12 +1796,13 @@ public class CatalogServiceCatalog extends Catalog {
for (Function fn: javaFunction.extract()) {
db.addFunction(fn);
fn.setCatalogVersion(incrementAndGetCatalogVersion());
+ ++numFuncs;
}
} catch (Exception | LinkageError e) {
LOG.error("Skipping function load: " + function.getFunctionName(), e);
}
}
- LOG.info("Loaded Java functions for database: " + db.getName());
+ LOG.info("Loaded {} Java functions for database: {}", numFuncs,
db.getName());
}
/**
@@ -1932,6 +1933,8 @@ public class CatalogServiceCatalog extends Catalog {
loadJavaFunctions(newDb, javaFns);
newDb.setCatalogVersion(incrementAndGetCatalogVersion());
+ LOG.info("Loading table list for database: {}", dbName);
+ int numTables = 0;
List<TTableName> tblsToBackgroundLoad = new ArrayList<>();
for (TableMeta tblMeta: getTableMetaFromHive(msClient, dbName,
/*tblName*/null)) {
String tableName = tblMeta.getTableName().toLowerCase();
@@ -1945,10 +1948,13 @@ public class CatalogServiceCatalog extends Catalog {
tblMeta.getComments());
incompleteTbl.setCatalogVersion(incrementAndGetCatalogVersion());
newDb.addTable(incompleteTbl);
+ ++numTables;
if (loadInBackground_) {
tblsToBackgroundLoad.add(new TTableName(dbName, tableName));
}
}
+ LOG.info("Loaded table list for database: {}. Number of tables: {}",
+ dbName, numTables);
if (existingDb != null) {
// Identify any removed functions and add them to the delta log.
