This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new b4c460023b6 branch-4.1: [fix](fe) Scope Hive file cache entries by
table (#68575)
b4c460023b6 is described below
commit b4c460023b694ba5c93bf1d67e8ff65e723aa31f
Author: 924060929 <[email protected]>
AuthorDate: Tue Sep 29 21:33:39 2026 +0800
branch-4.1: [fix](fe) Scope Hive file cache entries by table (#68575)
### What problem does this PR solve?
Issue Number: None
Related PR: None
Problem Summary: Two Hive partition tables can legitimately share a
partition location. On branch-4.1, the partition file-list cache key
uses the catalog, location, and partition values, but omits the table
ID. In contrast, `REFRESH TABLE` invalidates file-list entries by table
ID. If table a loads the old listing, table b can hit that entry after
writing a new file and refreshing b, so its scan omits the new file and
returns incomplete results. Include the table ID in the cache key's
equality and hash code. Each table then owns the listings that its
refresh invalidates.
The new FE unit test uses the same catalog, location, format, and
partition value for a and b. It failed on the original code because b
retrieved a's cached value. After the change, b misses a's value, both
entries coexist, and refreshing b removes only b's entry.
### Release note
Fix missing rows after refreshing a Hive table that shares a partition
location with another table.
### Check List (For Author)
- Test
- [ ] Regression test
- [x] Unit Test: `HiveMetaStoreCacheTest` (13 tests passed; new case
failed before the fix)
- [ ] Manual test
- [ ] No need to test or manual test
- FE Checkstyle: 0 violations.
- Local `./build.sh --fe` was not completed because this isolated
worktree lacked the complete thirdparty installation; `run buildall` is
requested on this PR.
- Behavior changed:
- [ ] No.
- [x] Yes. `REFRESH TABLE` reloads the current Hive table's file listing
even when another table shares its partition location.
- Does this need documentation?
- [x] No.
- [ ] Yes.
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label
---
.../datasource/hive/HiveExternalMetaCache.java | 4 ++-
.../datasource/hive/HiveMetaStoreCacheTest.java | 39 ++++++++++++++++++++++
2 files changed, 42 insertions(+), 1 deletion(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveExternalMetaCache.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveExternalMetaCache.java
index cdf58a9adc6..ce288bc37ae 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveExternalMetaCache.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveExternalMetaCache.java
@@ -1112,6 +1112,7 @@ public class HiveExternalMetaCache extends
AbstractExternalMetaCache {
private String inputFormat;
// The values of partitions.
protected List<String> partitionValues;
+ // Keep cache identity aligned with table-scoped invalidation.
private long id;
public FileCacheKey(long catalogId, long id, String location, String
inputFormat,
@@ -1142,6 +1143,7 @@ public class HiveExternalMetaCache extends
AbstractExternalMetaCache {
return dummyKey == ((FileCacheKey) obj).dummyKey;
}
return catalogId == ((FileCacheKey) obj).catalogId
+ && id == ((FileCacheKey) obj).id
&& location.equals(((FileCacheKey) obj).location)
&& Objects.equals(partitionValues, ((FileCacheKey)
obj).partitionValues);
}
@@ -1155,7 +1157,7 @@ public class HiveExternalMetaCache extends
AbstractExternalMetaCache {
if (dummyKey != 0) {
return Objects.hash(dummyKey);
}
- return Objects.hash(catalogId, location, partitionValues);
+ return Objects.hash(catalogId, id, location, partitionValues);
}
@Override
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetaStoreCacheTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetaStoreCacheTest.java
index 9cc96f58b7d..f067c879705 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetaStoreCacheTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetaStoreCacheTest.java
@@ -104,6 +104,45 @@ public class HiveMetaStoreCacheTest {
}
}
+ @Test
+ public void testRefreshTableWithSharedPartitionLocation() {
+ ThreadPoolExecutor executor =
ThreadPoolManager.newDaemonFixedThreadPool(
+ 1, 1, "refresh", 1, false);
+ ThreadPoolExecutor listExecutor =
ThreadPoolManager.newDaemonFixedThreadPool(
+ 1, 1, "file", 1, false);
+ try {
+ HiveExternalMetaCache cache = new HiveExternalMetaCache(executor,
listExecutor);
+ cache.initCatalog(0, new HashMap<>());
+ MetaCacheEntry<HiveExternalMetaCache.FileCacheKey,
HiveExternalMetaCache.FileCacheValue> fileCache =
+ cache.entry(0, HiveExternalMetaCache.ENTRY_FILE,
+ HiveExternalMetaCache.FileCacheKey.class,
+ HiveExternalMetaCache.FileCacheValue.class);
+
+ String location = "/warehouse/shared/p=p1";
+ List<String> values = Collections.singletonList("p1");
+ HiveExternalMetaCache.FileCacheKey keyA = new
HiveExternalMetaCache.FileCacheKey(
+ 0, Util.genIdByName("db", "a"), location, "parquet",
values);
+ HiveExternalMetaCache.FileCacheKey keyB = new
HiveExternalMetaCache.FileCacheKey(
+ 0, Util.genIdByName("db", "b"), location, "parquet",
values);
+ HiveExternalMetaCache.FileCacheValue oldFiles = new
HiveExternalMetaCache.FileCacheValue();
+ fileCache.put(keyA, oldFiles);
+
+ // b must list the directory after its write instead of reusing
a's old file list.
+ Assertions.assertNull(fileCache.getIfPresent(keyB));
+ HiveExternalMetaCache.FileCacheValue newFiles = new
HiveExternalMetaCache.FileCacheValue();
+ fileCache.put(keyB, newFiles);
+ Assertions.assertSame(oldFiles, fileCache.getIfPresent(keyA));
+ Assertions.assertSame(newFiles, fileCache.getIfPresent(keyB));
+
+ cache.invalidateTable(0, "db", "b");
+ Assertions.assertNull(fileCache.getIfPresent(keyB));
+ Assertions.assertSame(oldFiles, fileCache.getIfPresent(keyA));
+ } finally {
+ executor.shutdownNow();
+ listExecutor.shutdownNow();
+ }
+ }
+
@Test
public void testInvalidateTableCache() {
ThreadPoolExecutor executor =
ThreadPoolManager.newDaemonFixedThreadPool(
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
