This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.2 by this push:
new 8e1ab288b5e branch-4.2: [fix](compression) Preserve LZ4HC in tablet
metadata (#69574) (#69738)
8e1ab288b5e is described below
commit 8e1ab288b5e95e5fa4c1b537432e90bbfb4a6a8c
Author: Gavin Chou <[email protected]>
AuthorDate: Thu Oct 29 09:40:18 2026 +0800
branch-4.2: [fix](compression) Preserve LZ4HC in tablet metadata (#69574)
(#69738)
## Proposed changes
Backport #69574 to branch-4.2.
- Preserve `LZ4HC` when converting tablet metadata in the BE.
- Preserve `LZ4HC` in the cloud FE tablet metadata builder.
- Add regression coverage for both paths.
## Validation
- `sh format_code.sh be/src/storage/tablet/tablet_meta.cpp`
- `sh format_code.sh be/test/storage/tablet/tablet_meta_test.cpp`
- `sh run-fe-ut.sh --run
"org.apache.doris.cloud.datasource.CloudInternalCatalogTest"` (8 tests
passed)
- `sh run-be-ut.sh --run --filter
"TabletMetaTest.PreserveLz4hcCompressionType"` (blocked during link
because the prescribed `installed-doris-master` package does not contain
`libpaimon_c.a`)
- `sh build.sh --fe` was attempted; the prebuilt dependency check
requested a full third-party rebuild for the same missing artifact, so
the targeted FE unit-test build above was used for FE validation.
---
be/src/storage/tablet/tablet_meta.cpp | 3 +++
be/test/storage/tablet/tablet_meta_test.cpp | 7 +++++++
.../cloud/datasource/CloudInternalCatalog.java | 3 +++
.../cloud/datasource/CloudInternalCatalogTest.java | 24 ++++++++++++++++++----
4 files changed, 33 insertions(+), 4 deletions(-)
diff --git a/be/src/storage/tablet/tablet_meta.cpp
b/be/src/storage/tablet/tablet_meta.cpp
index b3cfb64a99e..f629e3b2874 100644
--- a/be/src/storage/tablet/tablet_meta.cpp
+++ b/be/src/storage/tablet/tablet_meta.cpp
@@ -228,6 +228,9 @@ TabletMeta::TabletMeta(int64_t table_id, int64_t
partition_id, int64_t tablet_id
case TCompressionType::LZ4F:
schema->set_compression_type(segment_v2::LZ4F);
break;
+ case TCompressionType::LZ4HC:
+ schema->set_compression_type(segment_v2::LZ4HC);
+ break;
case TCompressionType::ZLIB:
schema->set_compression_type(segment_v2::ZLIB);
break;
diff --git a/be/test/storage/tablet/tablet_meta_test.cpp
b/be/test/storage/tablet/tablet_meta_test.cpp
index 140f3dcff0b..38044b2f21a 100644
--- a/be/test/storage/tablet/tablet_meta_test.cpp
+++ b/be/test/storage/tablet/tablet_meta_test.cpp
@@ -83,6 +83,13 @@ TEST(TabletMetaTest, SaveAsBufferAndParse) {
}
}
+TEST(TabletMetaTest, PreserveLz4hcCompressionType) {
+ TabletMeta tablet_meta(1, 2, 3, 3, 4, 5, TTabletSchema(), 6, {{7, 8}},
UniqueId(9, 10),
+ TTabletType::TABLET_TYPE_DISK,
TCompressionType::LZ4HC);
+
+ EXPECT_EQ(segment_v2::LZ4HC,
tablet_meta.tablet_schema()->compression_type());
+}
+
TEST(TabletMetaTest, TopLevelInvertedIndexFormatOverridesSharedSchemaFormat) {
TabletMetaPB tablet_meta_pb;
tablet_meta_pb.set_table_id(1);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
index 8d1a5a68385..09323578134 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
@@ -323,6 +323,9 @@ public class CloudInternalCatalog extends InternalCatalog {
case LZ4F:
schemaBuilder.setCompressionType(SegmentV2.CompressionTypePB.LZ4F);
break;
+ case LZ4HC:
+
schemaBuilder.setCompressionType(SegmentV2.CompressionTypePB.LZ4HC);
+ break;
case ZLIB:
schemaBuilder.setCompressionType(SegmentV2.CompressionTypePB.ZLIB);
break;
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/cloud/datasource/CloudInternalCatalogTest.java
b/fe/fe-core/src/test/java/org/apache/doris/cloud/datasource/CloudInternalCatalogTest.java
index bcca81f90cf..b23e456b5db 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/cloud/datasource/CloudInternalCatalogTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/cloud/datasource/CloudInternalCatalogTest.java
@@ -44,6 +44,7 @@ import org.apache.doris.thrift.TSortType;
import org.apache.doris.thrift.TStorageFormat;
import org.apache.doris.thrift.TTabletType;
+import doris.segment_v2.SegmentV2;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
@@ -144,7 +145,7 @@ public class CloudInternalCatalogTest {
boolean original =
Config.enable_partition_inverted_index_storage_format_rollout;
try {
Config.enable_partition_inverted_index_storage_format_rollout =
false;
- OlapFile.TabletMetaCloudPB disabledTabletMeta =
createTabletMeta(tablet);
+ OlapFile.TabletMetaCloudPB disabledTabletMeta =
createTabletMeta(tablet, TCompressionType.LZ4F);
Assertions.assertEquals(17, disabledTabletMeta.getSchemaVersion());
Assertions.assertEquals(17,
disabledTabletMeta.getSchema().getSchemaVersion());
Assertions.assertEquals(OlapFile.InvertedIndexStorageFormatPB.SNII,
@@ -154,7 +155,7 @@ public class CloudInternalCatalogTest {
Assertions.assertFalse(disabledTabletMeta.getRsMetas(0).hasInvertedIndexStorageFormat());
Config.enable_partition_inverted_index_storage_format_rollout =
true;
- OlapFile.TabletMetaCloudPB enabledTabletMeta =
createTabletMeta(tablet);
+ OlapFile.TabletMetaCloudPB enabledTabletMeta =
createTabletMeta(tablet, TCompressionType.LZ4F);
Assertions.assertTrue(enabledTabletMeta.hasInvertedIndexStorageFormat());
Assertions.assertEquals(OlapFile.InvertedIndexStorageFormatPB.SNII,
enabledTabletMeta.getInvertedIndexStorageFormat());
@@ -166,6 +167,20 @@ public class CloudInternalCatalogTest {
}
}
+ @Test
+ public void testCreateTabletMetaPreservesLz4hcCompressionType() throws
Exception {
+ Tablet tablet = Mockito.mock(Tablet.class);
+ Replica replica = Mockito.mock(Replica.class);
+ Mockito.when(tablet.getId()).thenReturn(100L);
+
Mockito.when(tablet.getReplicas()).thenReturn(Collections.singletonList(replica));
+ Mockito.when(replica.getId()).thenReturn(200L);
+
+ OlapFile.TabletMetaCloudPB tabletMeta = createTabletMeta(tablet,
TCompressionType.LZ4HC);
+
+ Assertions.assertEquals(SegmentV2.CompressionTypePB.LZ4HC,
+ tabletMeta.getSchema().getCompressionType());
+ }
+
@Test
public void testReplaySingleReplicaDoesNotScanOtherRoutes() throws
Exception {
boolean savedClean = Config.enable_cloud_replica_stale_route_clean;
@@ -298,11 +313,12 @@ public class CloudInternalCatalogTest {
Mockito.when(systemInfoService.getBackend(backendId)).thenReturn(backend);
}
- private OlapFile.TabletMetaCloudPB createTabletMeta(Tablet tablet) throws
Exception {
+ private OlapFile.TabletMetaCloudPB createTabletMeta(Tablet tablet,
TCompressionType compressionType)
+ throws Exception {
return new CloudInternalCatalog().createTabletMetaBuilder(
1L, 2L, 3L, tablet, TTabletType.TABLET_TYPE_DISK, 23,
KeysType.DUP_KEYS, (short) 1,
Collections.emptySet(), 0.05, Collections.emptyList(),
Collections.emptyList(),
- new DataSortInfo(TSortType.LEXICAL, 0), TCompressionType.LZ4F,
TStorageFormat.DEFAULT,
+ new DataSortInfo(TSortType.LEXICAL, 0), compressionType,
TStorageFormat.DEFAULT,
"", false, false,
"table", 0L, false, false, 17, "size_based",
0L, 0L, 0L, 0L, 0L, false, Collections.emptyList(),
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
