This is an automated email from the ASF dual-hosted git repository.
stigahuang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
The following commit(s) were added to refs/heads/master by this push:
new 58fd45f20 IMPALA-12876: Add catalogVersion and loaded timestamp in
query profiles
58fd45f20 is described below
commit 58fd45f20c22f8a062f603d6b96f62ee57d85ca9
Author: stiga-huang <[email protected]>
AuthorDate: Fri May 17 09:57:20 2024 +0800
IMPALA-12876: Add catalogVersion and loaded timestamp in query profiles
When debugging stale metadata, it'd be helpful to know what catalog
version of the tables are used and what's the time when catalogd loads
those versions. This patch exposes these info in the query profile for
each referenced table. E.g.
Original Table Versions: tpch.customer, 2249, 1726052668932, Wed Sep 11
19:04:28 CST 2024
tpch.nation, 2255, 1726052790140, Wed Sep 11 19:06:30 CST 2024
tpch.orders, 2257, 1726052803258, Wed Sep 11 19:06:43 CST 2024
tpch.lineitem, 2254, 1726052785384, Wed Sep 11 19:06:25 CST 2024
tpch.supplier, 2256, 1726052794235, Wed Sep 11 19:06:34 CST 2024
Each line consists of the table name, catalog version, loaded timestamp
and the timestamp string.
Implementation:
The loaded timestamp is updated whenever a CatalogObject updates its
catalog version in catalogd. It's passed to impalads with the
TCatalogObject broadcasted by statestore, or in DDL/DML responses.
Currently, the loaded timestamp is added for table, view, function, data
source, and hdfs cache pool in catalogd. However, only those of table
and view are applied used in impalad. For the loaded timestamp of other
types, users can check them in the /catalog WebUI of catalogd.
Tests:
- Adds e2e test
Change-Id: I94b2fd59ed5aca664d6db4448c61ad21a88a4f98
Reviewed-on: http://gerrit.cloudera.org:8080/21782
Reviewed-by: Impala Public Jenkins <[email protected]>
Tested-by: Impala Public Jenkins <[email protected]>
---
common/thrift/CatalogObjects.thrift | 3 +++
common/thrift/CatalogService.thrift | 3 +++
.../java/org/apache/impala/catalog/Catalog.java | 5 +++++
.../org/apache/impala/catalog/CatalogObject.java | 20 +++++++++++++-------
.../apache/impala/catalog/CatalogObjectImpl.java | 16 +++++++++++++++-
.../impala/catalog/CatalogServiceCatalog.java | 6 ++++++
.../org/apache/impala/catalog/CtasTargetTable.java | 6 ++++++
fe/src/main/java/org/apache/impala/catalog/Db.java | 2 ++
.../java/org/apache/impala/catalog/FeTable.java | 9 +++++++++
.../impala/catalog/IcebergTimeTravelTable.java | 6 ++++++
.../org/apache/impala/catalog/ImpaladCatalog.java | 5 +++--
.../main/java/org/apache/impala/catalog/Table.java | 2 ++
.../org/apache/impala/catalog/VirtualTable.java | 6 ++++++
.../impala/catalog/local/CatalogdMetaProvider.java | 21 +++++++++++++++++----
.../impala/catalog/local/DirectMetaProvider.java | 10 ++++++++++
.../org/apache/impala/catalog/local/LocalTable.java | 12 ++++++++++++
.../apache/impala/catalog/local/MetaProvider.java | 2 ++
.../java/org/apache/impala/service/Frontend.java | 10 ++++++++++
tests/query_test/test_observability.py | 1 +
tests/shell/test_shell_commandline.py | 19 +++++++++++++++++++
20 files changed, 150 insertions(+), 14 deletions(-)
diff --git a/common/thrift/CatalogObjects.thrift
b/common/thrift/CatalogObjects.thrift
index 30b856e7e..851ed3f1e 100644
--- a/common/thrift/CatalogObjects.thrift
+++ b/common/thrift/CatalogObjects.thrift
@@ -929,4 +929,7 @@ struct TCatalogObject {
// Set iff object type is HDFS_PARTITION
12: optional THdfsPartition hdfs_partition
+
+ // Timestamp when the catalog version changes in catalogd
+ 13: optional i64 last_modified_time_ms
}
diff --git a/common/thrift/CatalogService.thrift
b/common/thrift/CatalogService.thrift
index 728c35bb0..d9e451310 100644
--- a/common/thrift/CatalogService.thrift
+++ b/common/thrift/CatalogService.thrift
@@ -637,6 +637,9 @@ struct TGetPartialCatalogObjectResponse {
7: optional list<Types.TFunction> functions
// DataSource objects are small enough that we return them wholesale.
8: optional list<CatalogObjects.TDataSource> data_srcs
+
+ // Loaded time in catalogd corresponding to 'object_version_number'.
+ 9: optional i64 object_loaded_time_ms
}
diff --git a/fe/src/main/java/org/apache/impala/catalog/Catalog.java
b/fe/src/main/java/org/apache/impala/catalog/Catalog.java
index d24b7567f..46ef622ae 100644
--- a/fe/src/main/java/org/apache/impala/catalog/Catalog.java
+++ b/fe/src/main/java/org/apache/impala/catalog/Catalog.java
@@ -564,6 +564,7 @@ public abstract class Catalog implements AutoCloseable {
}
result.setType(db.getCatalogObjectType());
result.setCatalog_version(db.getCatalogVersion());
+ result.setLast_modified_time_ms(db.getLastLoadedTimeMs());
result.setDb(db.toThrift());
break;
}
@@ -579,6 +580,7 @@ public abstract class Catalog implements AutoCloseable {
try {
result.setType(table.getCatalogObjectType());
result.setCatalog_version(table.getCatalogVersion());
+ result.setLast_modified_time_ms(table.getLastLoadedTimeMs());
result.setTable(
isHumanReadable ? table.toHumanReadableThrift() :
table.toThrift());
} finally {
@@ -596,6 +598,7 @@ public abstract class Catalog implements AutoCloseable {
}
result.setType(fn.getCatalogObjectType());
result.setCatalog_version(fn.getCatalogVersion());
+ result.setLast_modified_time_ms(fn.getLastLoadedTimeMs());
result.setFn(fn.toThrift());
break;
}
@@ -607,6 +610,7 @@ public abstract class Catalog implements AutoCloseable {
}
result.setType(dataSrc.getCatalogObjectType());
result.setCatalog_version(dataSrc.getCatalogVersion());
+ result.setLast_modified_time_ms(dataSrc.getLastLoadedTimeMs());
result.setData_source(dataSrc.toThrift());
break;
}
@@ -618,6 +622,7 @@ public abstract class Catalog implements AutoCloseable {
}
result.setType(pool.getCatalogObjectType());
result.setCatalog_version(pool.getCatalogVersion());
+ result.setLast_modified_time_ms(pool.getLastLoadedTimeMs());
result.setCache_pool(pool.toThrift());
break;
}
diff --git a/fe/src/main/java/org/apache/impala/catalog/CatalogObject.java
b/fe/src/main/java/org/apache/impala/catalog/CatalogObject.java
index 39da7a9e4..23b970092 100644
--- a/fe/src/main/java/org/apache/impala/catalog/CatalogObject.java
+++ b/fe/src/main/java/org/apache/impala/catalog/CatalogObject.java
@@ -33,7 +33,7 @@ public interface CatalogObject extends HasName {
* same topic update. "invalidation" form means only the name will be
included. "none"
* form means return nothing, i.e. null.
*/
- static enum ThriftObjectType {
+ enum ThriftObjectType {
FULL,
DESCRIPTOR_ONLY,
INVALIDATION,
@@ -41,21 +41,27 @@ public interface CatalogObject extends HasName {
}
// Returns the TCatalogObject type of this Catalog object.
- public TCatalogObjectType getCatalogObjectType();
+ TCatalogObjectType getCatalogObjectType();
// Returns the unqualified object name.
@Override
- public String getName();
+ String getName();
// Returns the unique name of this catalog object.
- public String getUniqueName();
+ String getUniqueName();
// Returns the version of this catalog object.
- public long getCatalogVersion();
+ long getCatalogVersion();
// Sets the version of this catalog object.
- public void setCatalogVersion(long newVersion);
+ void setCatalogVersion(long newVersion);
// Returns true if this CatalogObject has had its metadata loaded, false
otherwise.
- public boolean isLoaded();
+ boolean isLoaded();
+
+ // Returns the timestamp when the object of the catalog version is loaded.
+ long getLastLoadedTimeMs();
+
+ // Sets the timestamp of the above. Only used in ImpaladCatalog.
+ void setLastLoadedTimeMs(long timeMs);
}
diff --git a/fe/src/main/java/org/apache/impala/catalog/CatalogObjectImpl.java
b/fe/src/main/java/org/apache/impala/catalog/CatalogObjectImpl.java
index 89723c011..8234ef431 100644
--- a/fe/src/main/java/org/apache/impala/catalog/CatalogObjectImpl.java
+++ b/fe/src/main/java/org/apache/impala/catalog/CatalogObjectImpl.java
@@ -25,6 +25,8 @@ abstract public class CatalogObjectImpl implements
CatalogObject {
// Current catalog version of this object. Initialized to
// Catalog.INITIAL_CATALOG_VERSION.
private AtomicLong catalogVersion_ = new
AtomicLong(Catalog.INITIAL_CATALOG_VERSION);
+ // Timestamp when the object is recently loaded/reloaded.
+ private long mTimeMs_ = 0;
protected CatalogObjectImpl() {}
@@ -32,7 +34,18 @@ abstract public class CatalogObjectImpl implements
CatalogObject {
public long getCatalogVersion() { return catalogVersion_.get(); }
@Override
- public void setCatalogVersion(long newVersion) {
catalogVersion_.set(newVersion); }
+ public void setCatalogVersion(long newVersion) {
+ catalogVersion_.set(newVersion);
+ mTimeMs_ = System.currentTimeMillis();
+ }
+
+ @Override
+ public long getLastLoadedTimeMs() { return mTimeMs_; }
+
+ @Override
+ public void setLastLoadedTimeMs(long timeMs) {
+ mTimeMs_ = timeMs;
+ }
@Override
public boolean isLoaded() { return true; }
@@ -48,6 +61,7 @@ abstract public class CatalogObjectImpl implements
CatalogObject {
public final TCatalogObject toTCatalogObject() {
TCatalogObject catalogObject =
new TCatalogObject(getCatalogObjectType(), getCatalogVersion());
+ catalogObject.setLast_modified_time_ms(getLastLoadedTimeMs());
setTCatalogObject(catalogObject);
return catalogObject;
}
diff --git
a/fe/src/main/java/org/apache/impala/catalog/CatalogServiceCatalog.java
b/fe/src/main/java/org/apache/impala/catalog/CatalogServiceCatalog.java
index 55cc783ea..e2fb8fa8b 100644
--- a/fe/src/main/java/org/apache/impala/catalog/CatalogServiceCatalog.java
+++ b/fe/src/main/java/org/apache/impala/catalog/CatalogServiceCatalog.java
@@ -942,6 +942,7 @@ public class CatalogServiceCatalog extends Catalog {
Preconditions.checkState(topicMode_ == TopicMode.MINIMAL ||
topicMode_ == TopicMode.MIXED);
TCatalogObject min = new TCatalogObject(obj.type, obj.catalog_version);
+ min.setLast_modified_time_ms(obj.last_modified_time_ms);
switch (obj.type) {
case DATABASE:
min.setDb(new TDatabase(obj.db.db_name));
@@ -1436,6 +1437,7 @@ public class CatalogServiceCatalog extends Catalog {
if (dbVersion > ctx.fromVersion && dbVersion <= ctx.toVersion) {
TCatalogObject catalogDb =
new TCatalogObject(TCatalogObjectType.DATABASE, dbVersion);
+ catalogDb.setLast_modified_time_ms(db.getLastLoadedTimeMs());
catalogDb.setDb(db.toThrift());
ctx.addCatalogObject(catalogDb, false);
}
@@ -1770,6 +1772,7 @@ public class CatalogServiceCatalog extends Catalog {
return;
}
catalogTbl.setCatalog_version(tbl.getCatalogVersion());
+ catalogTbl.setLast_modified_time_ms(tbl.getLastLoadedTimeMs());
ctx.addCatalogObject(catalogTbl, false);
}
@@ -1812,6 +1815,7 @@ public class CatalogServiceCatalog extends Catalog {
if (ctx.versionNotInRange(fnVersion)) return;
TCatalogObject function =
new TCatalogObject(TCatalogObjectType.FUNCTION, fnVersion);
+ function.setLast_modified_time_ms(fn.getLastLoadedTimeMs());
function.setFn(fn.toThrift());
ctx.addCatalogObject(function, false);
}
@@ -1826,6 +1830,7 @@ public class CatalogServiceCatalog extends Catalog {
if (ctx.versionNotInRange(dsVersion)) return;
TCatalogObject catalogObj =
new TCatalogObject(TCatalogObjectType.DATA_SOURCE, dsVersion);
+ catalogObj.setLast_modified_time_ms(dataSource.getLastLoadedTimeMs());
catalogObj.setData_source(dataSource.toThrift());
ctx.addCatalogObject(catalogObj, false);
}
@@ -1840,6 +1845,7 @@ public class CatalogServiceCatalog extends Catalog {
if (ctx.versionNotInRange(cpVersion)) return;
TCatalogObject pool =
new TCatalogObject(TCatalogObjectType.HDFS_CACHE_POOL, cpVersion);
+ pool.setLast_modified_time_ms(cachePool.getLastLoadedTimeMs());
pool.setCache_pool(cachePool.toThrift());
ctx.addCatalogObject(pool, false);
}
diff --git a/fe/src/main/java/org/apache/impala/catalog/CtasTargetTable.java
b/fe/src/main/java/org/apache/impala/catalog/CtasTargetTable.java
index 189242f4e..d697e52f9 100644
--- a/fe/src/main/java/org/apache/impala/catalog/CtasTargetTable.java
+++ b/fe/src/main/java/org/apache/impala/catalog/CtasTargetTable.java
@@ -168,4 +168,10 @@ public abstract class CtasTargetTable implements FeTable {
public String getOwnerUser() {
return owner_;
}
+
+ @Override
+ public long getCatalogVersion() { return 0; }
+
+ @Override
+ public long getLastLoadedTimeMs() { return 0; }
}
diff --git a/fe/src/main/java/org/apache/impala/catalog/Db.java
b/fe/src/main/java/org/apache/impala/catalog/Db.java
index bfaa37895..67588cb04 100644
--- a/fe/src/main/java/org/apache/impala/catalog/Db.java
+++ b/fe/src/main/java/org/apache/impala/catalog/Db.java
@@ -529,6 +529,7 @@ public class Db extends CatalogObjectImpl implements FeDb {
public TCatalogObject toMinimalTCatalogObject() {
TCatalogObject min = new TCatalogObject(getCatalogObjectType(),
getCatalogVersion());
+ min.setLast_modified_time_ms(getLastLoadedTimeMs());
min.setDb(new TDatabase(getName()));
return min;
}
@@ -544,6 +545,7 @@ public class Db extends CatalogObjectImpl implements FeDb {
TGetPartialCatalogObjectResponse resp = new
TGetPartialCatalogObjectResponse();
resp.setObject_version_number(getCatalogVersion());
+ resp.setObject_loaded_time_ms(getLastLoadedTimeMs());
resp.db_info = new TPartialDbInfo();
if (selector.want_hms_database) {
// TODO(todd): we need to deep-copy here because 'addFunction' other DDLs
diff --git a/fe/src/main/java/org/apache/impala/catalog/FeTable.java
b/fe/src/main/java/org/apache/impala/catalog/FeTable.java
index 3453e4913..75cb667df 100644
--- a/fe/src/main/java/org/apache/impala/catalog/FeTable.java
+++ b/fe/src/main/java/org/apache/impala/catalog/FeTable.java
@@ -212,4 +212,13 @@ public interface FeTable {
*/
String getOwnerUser();
+ /**
+ * @return the catalog version of this table assigned in catalogd.
+ */
+ long getCatalogVersion();
+
+ /**
+ * @return the timestamp when the table is last loaded or reloaded in
catalogd.
+ */
+ long getLastLoadedTimeMs();
}
diff --git
a/fe/src/main/java/org/apache/impala/catalog/IcebergTimeTravelTable.java
b/fe/src/main/java/org/apache/impala/catalog/IcebergTimeTravelTable.java
index 7f4d3d011..14358f6d6 100644
--- a/fe/src/main/java/org/apache/impala/catalog/IcebergTimeTravelTable.java
+++ b/fe/src/main/java/org/apache/impala/catalog/IcebergTimeTravelTable.java
@@ -599,4 +599,10 @@ class ForwardingFeIcebergTable implements FeIcebergTable {
public String getOwnerUser() {
return base.getOwnerUser();
}
+
+ @Override
+ public long getCatalogVersion() { return 0; }
+
+ @Override
+ public long getLastLoadedTimeMs() { return 0; }
}
diff --git a/fe/src/main/java/org/apache/impala/catalog/ImpaladCatalog.java
b/fe/src/main/java/org/apache/impala/catalog/ImpaladCatalog.java
index a2ba7271e..a13f28f76 100644
--- a/fe/src/main/java/org/apache/impala/catalog/ImpaladCatalog.java
+++ b/fe/src/main/java/org/apache/impala/catalog/ImpaladCatalog.java
@@ -343,7 +343,7 @@ public class ImpaladCatalog extends Catalog implements
FeCatalog {
TableName tblName = new TableName(table.getDb_name(),
table.getTbl_name());
addTable(table,
newPartitions.getOrDefault(tblName, Collections.emptyList()),
- catalogObject.getCatalog_version());
+ catalogObject.getCatalog_version(),
catalogObject.getLast_modified_time_ms());
break;
case FUNCTION:
// Remove the function first, in case there is an existing function
with the same
@@ -474,7 +474,7 @@ public class ImpaladCatalog extends Catalog implements
FeCatalog {
}
private void addTable(TTable thriftTable, List<THdfsPartition> newPartitions,
- long catalogVersion) throws TableLoadingException {
+ long catalogVersion, long lastLoadedTime) throws TableLoadingException {
Db db = getDb(thriftTable.db_name);
if (db == null) {
if (LOG.isTraceEnabled()) {
@@ -488,6 +488,7 @@ public class ImpaladCatalog extends Catalog implements
FeCatalog {
Table existingTable = db.getTable(thriftTable.tbl_name);
Table newTable = Table.fromThrift(db, thriftTable);
newTable.setCatalogVersion(catalogVersion);
+ newTable.setLastLoadedTimeMs(lastLoadedTime);
if (existingTable != null && existingTable.getCatalogVersion() >=
catalogVersion) {
if (LOG.isTraceEnabled()) {
LOG.trace("Ignore stale update on table {}: currentVersion={},
updateVersion={}",
diff --git a/fe/src/main/java/org/apache/impala/catalog/Table.java
b/fe/src/main/java/org/apache/impala/catalog/Table.java
index 90f306508..d8866613d 100644
--- a/fe/src/main/java/org/apache/impala/catalog/Table.java
+++ b/fe/src/main/java/org/apache/impala/catalog/Table.java
@@ -738,6 +738,7 @@ public abstract class Table extends CatalogObjectImpl
implements FeTable {
private TCatalogObject toMinimalTCatalogObjectHelper() {
TCatalogObject catalogObject =
new TCatalogObject(getCatalogObjectType(), getCatalogVersion());
+ catalogObject.setLast_modified_time_ms(getLastLoadedTimeMs());
TTable table = new TTable(getDb().getName(), getName());
table.setTbl_comment(getTableComment());
catalogObject.setTable(table);
@@ -797,6 +798,7 @@ public abstract class Table extends CatalogObjectImpl
implements FeTable {
TGetPartialCatalogObjectResponse resp = new
TGetPartialCatalogObjectResponse();
resp.setObject_version_number(getCatalogVersion());
+ resp.setObject_loaded_time_ms(getLastLoadedTimeMs());
resp.table_info = new TPartialTableInfo();
resp.table_info.setStorage_metadata_load_time_ns(storageMetadataLoadTime_);
storageMetadataLoadTime_ = 0;
diff --git a/fe/src/main/java/org/apache/impala/catalog/VirtualTable.java
b/fe/src/main/java/org/apache/impala/catalog/VirtualTable.java
index 32ae98ede..01f3cc916 100644
--- a/fe/src/main/java/org/apache/impala/catalog/VirtualTable.java
+++ b/fe/src/main/java/org/apache/impala/catalog/VirtualTable.java
@@ -154,4 +154,10 @@ public abstract class VirtualTable implements FeTable {
@Override
public String getOwnerUser() { return owner_; }
+
+ @Override
+ public long getCatalogVersion() { return 0; }
+
+ @Override
+ public long getLastLoadedTimeMs() { return 0; }
}
diff --git
a/fe/src/main/java/org/apache/impala/catalog/local/CatalogdMetaProvider.java
b/fe/src/main/java/org/apache/impala/catalog/local/CatalogdMetaProvider.java
index 42204297d..a00b5be2a 100644
--- a/fe/src/main/java/org/apache/impala/catalog/local/CatalogdMetaProvider.java
+++ b/fe/src/main/java/org/apache/impala/catalog/local/CatalogdMetaProvider.java
@@ -804,7 +804,7 @@ public class CatalogdMetaProvider implements MetaProvider {
new ArrayList<>() :
resp.table_info.sql_constraints.getForeign_keys();
return new TableMetaRefImpl(
dbName, tableName, resp.table_info.hms_table,
resp.object_version_number,
- new SqlConstraints(primaryKeys, foreignKeys),
+ resp.object_loaded_time_ms, new SqlConstraints(primaryKeys,
foreignKeys),
resp.table_info.valid_write_ids,
resp.table_info.is_marked_cached,
resp.table_info.partition_prefixes,
resp.table_info.virtual_columns);
}
@@ -1912,6 +1912,12 @@ public class CatalogdMetaProvider implements
MetaProvider {
*/
private final long catalogVersion_;
+ /**
+ * The timestamp when the table is loaded in catalogd corresponding to the
catalog
+ * version.
+ */
+ private final long loadedTimeMs_;
+
/**
* Valid write id list of ACID tables.
*/
@@ -1926,13 +1932,14 @@ public class CatalogdMetaProvider implements
MetaProvider {
private final HdfsPartitionLocationCompressor partitionLocationCompressor_;
public TableMetaRefImpl(String dbName, String tableName,
- Table msTable, long catalogVersion, SqlConstraints sqlConstraints,
- TValidWriteIdList validWriteIds, boolean isMarkedCached,
- List<String> locationPrefixes, List<TColumn> tvirtCols) {
+ Table msTable, long catalogVersion, long loadedTimeMs,
+ SqlConstraints sqlConstraints, TValidWriteIdList validWriteIds,
+ boolean isMarkedCached, List<String> locationPrefixes, List<TColumn>
tvirtCols) {
this.dbName_ = dbName;
this.tableName_ = tableName;
this.msTable_ = msTable;
this.catalogVersion_ = catalogVersion;
+ this.loadedTimeMs_ = loadedTimeMs;
this.sqlConstraints_ = sqlConstraints;
this.validWriteIds_ = validWriteIds;
this.isMarkedCached_ = isMarkedCached;
@@ -1975,6 +1982,12 @@ public class CatalogdMetaProvider implements
MetaProvider {
public List<VirtualColumn> getVirtualColumns() {
return virtualColumns_;
}
+
+ @Override
+ public long getCatalogVersion() { return catalogVersion_; }
+
+ @Override
+ public long getLoadedTimeMs() { return loadedTimeMs_; }
}
/**
diff --git
a/fe/src/main/java/org/apache/impala/catalog/local/DirectMetaProvider.java
b/fe/src/main/java/org/apache/impala/catalog/local/DirectMetaProvider.java
index ffb6bb461..57efff547 100644
--- a/fe/src/main/java/org/apache/impala/catalog/local/DirectMetaProvider.java
+++ b/fe/src/main/java/org/apache/impala/catalog/local/DirectMetaProvider.java
@@ -548,6 +548,16 @@ class DirectMetaProvider implements MetaProvider {
throw new UnsupportedOperationException("Virtual columns are not
supported with " +
"DirectMetaProvider implementation");
}
+
+ @Override
+ public long getCatalogVersion() {
+ return 0;
+ }
+
+ @Override
+ public long getLoadedTimeMs() {
+ return 0;
+ }
}
@Override
diff --git a/fe/src/main/java/org/apache/impala/catalog/local/LocalTable.java
b/fe/src/main/java/org/apache/impala/catalog/local/LocalTable.java
index 91bfaa41d..d9e36443c 100644
--- a/fe/src/main/java/org/apache/impala/catalog/local/LocalTable.java
+++ b/fe/src/main/java/org/apache/impala/catalog/local/LocalTable.java
@@ -354,6 +354,18 @@ abstract class LocalTable implements FeTable {
return null;
}
+ @Override
+ public long getCatalogVersion() {
+ if (ref_ == null) return 0;
+ return ref_.getCatalogVersion();
+ }
+
+ @Override
+ public long getLastLoadedTimeMs() {
+ if (ref_ == null) return 0;
+ return ref_.getLoadedTimeMs();
+ }
+
protected void loadColumnStats() {
try {
List<ColumnStatisticsObj> stats = db_.getCatalog().getMetaProvider()
diff --git a/fe/src/main/java/org/apache/impala/catalog/local/MetaProvider.java
b/fe/src/main/java/org/apache/impala/catalog/local/MetaProvider.java
index 49286463f..d08b21b70 100644
--- a/fe/src/main/java/org/apache/impala/catalog/local/MetaProvider.java
+++ b/fe/src/main/java/org/apache/impala/catalog/local/MetaProvider.java
@@ -169,6 +169,8 @@ public interface MetaProvider {
boolean isPartitioned();
boolean isTransactional();
List<VirtualColumn> getVirtualColumns();
+ long getCatalogVersion();
+ long getLoadedTimeMs();
}
/**
diff --git a/fe/src/main/java/org/apache/impala/service/Frontend.java
b/fe/src/main/java/org/apache/impala/service/Frontend.java
index 199b6df70..a66d6fbc6 100644
--- a/fe/src/main/java/org/apache/impala/service/Frontend.java
+++ b/fe/src/main/java/org/apache/impala/service/Frontend.java
@@ -39,6 +39,7 @@ import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
+import java.util.Date;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
@@ -2581,6 +2582,15 @@ public class Frontend {
.map(TableName::toString)
.collect(Collectors.joining(", ")));
+ // Add the catalog versions and loaded timestamps.
+ FrontendProfile.getCurrent().addInfoString("Original Table Versions",
+ stmtTableCache.tables.values().stream()
+ .map(t -> String.join(", ", t.getFullName(),
+ Long.toString(t.getCatalogVersion()),
+ Long.toString(t.getLastLoadedTimeMs()),
+ new Date(t.getLastLoadedTimeMs()).toString()))
+ .collect(Collectors.joining("\n")));
+
// Analyze and authorize stmt
AnalysisContext analysisCtx = new AnalysisContext(queryCtx, authzFactory_,
timeline);
AnalysisResult analysisResult = analysisCtx.analyzeAndAuthorize(stmt,
stmtTableCache,
diff --git a/tests/query_test/test_observability.py
b/tests/query_test/test_observability.py
index fbfa86918..0a34028f4 100644
--- a/tests/query_test/test_observability.py
+++ b/tests/query_test/test_observability.py
@@ -309,6 +309,7 @@ class TestObservability(ImpalaTestSuite):
load_event_regexes = [
r'Frontend:',
r'Referenced Tables:',
+ r'Original Table Versions:',
r'CatalogFetch.ColumnStats.Hits',
r'CatalogFetch.ColumnStats.Misses',
r'CatalogFetch.ColumnStats.Requests',
diff --git a/tests/shell/test_shell_commandline.py
b/tests/shell/test_shell_commandline.py
index 401560473..a23cbdf68 100644
--- a/tests/shell/test_shell_commandline.py
+++ b/tests/shell/test_shell_commandline.py
@@ -414,11 +414,20 @@ class TestImpalaShell(ImpalaTestSuite):
args = base_args + ['create table %s.shell_profile_test (id int)' % db]
create = run_impala_shell_cmd(vector, args)
assert "Referenced Tables: \n" in create.stdout
+ assert "Original Table Versions: \n" in create.stdout
+ TABLE_VERSION = re.compile(
+ r"Original Table Versions: (\w+\.\w+), (\d+), (\d+), ([^\n]*)\n")
for statement in statements:
args = base_args + [statement]
result = run_impala_shell_cmd(vector, args)
assert "Referenced Tables: %s.shell_profile_test" % unique_database in
result.stdout
+ m = TABLE_VERSION.search(result.stdout)
+ assert m, "Original Table Versions not found in profile:\n" +
result.stdout
+ assert m.group(1) == unique_database + ".shell_profile_test"
+ assert int(m.group(2)) > 0, "Invalid catalog version in " + m.group(0) +
statement
+ assert int(m.group(3)) > 0, "Invalid loaded timestamp in " + m.group(0)
+ statement
+ assert len(m.group(4)) > 0, "Invalid timestamp string in " + m.group(0)
+ statement
def test_runtime_profile_multiple_referenced_tables(self, vector,
unique_database):
if vector.get_value('strict_hs2_protocol'):
@@ -454,6 +463,16 @@ class TestImpalaShell(ImpalaTestSuite):
for i in range(0, 2):
assert "{db}.shell_profile_test{index}".format(db=db, index=i) in
referenced_tables
+ TABLE_VERSIONS = re.compile(r"Original Table Versions: (\w+\.\w+), (\d+),
(\d+), "
+ r"([^\n]*)\n(\w+\.\w+), (\d+), (\d+),
([^\n]*)\n")
+ m = TABLE_VERSIONS.search(result.stdout)
+ assert m, "Original Table Versions not found in profile:\n" + result.stdout
+ for i in (0, 4):
+ assert db + ".shell_profile_test" in m.group(i + 1), "missing tables:" +
m.group(0)
+ assert int(m.group(i + 2)) > 0, "Invalid catalog version: " + m.group(0)
+ assert int(m.group(i + 3)) > 0, "Invalid timestamp: " + m.group(0)
+ assert len(m.group(i + 4)) > 0, "Invalid timestamp string: " + m.group(0)
+
def test_summary(self, vector):
if vector.get_value('strict_hs2_protocol'):
pytest.skip("Summary not supported in strict hs2 mode.")
