[
https://issues.apache.org/jira/browse/HUDI-2902?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
sivabalan narayanan updated HUDI-2902:
--------------------------------------
Fix Version/s: 0.10.0
> Metadata table virtual keys are not effective in base files
> -----------------------------------------------------------
>
> Key: HUDI-2902
> URL: https://issues.apache.org/jira/browse/HUDI-2902
> Project: Apache Hudi
> Issue Type: Bug
> Components: Writer Core
> Affects Versions: 0.10.0
> Reporter: sivabalan narayanan
> Assignee: Manoj Govindassamy
> Priority: Major
> Fix For: 0.10.0
>
>
> I was inspecting a base hfile produced after compaction in metadata table and
> and looks like virtual keys did not kick in, even though default value should
> have enabled virtual keys.
>
> sample records from hfile
> {code:java}
> Rec 1970/01/01 => {"_hoodie_commit_time": "20211201085239959001",
> "_hoodie_commit_seqno": "20211201085239959001_0_100", "_hoodie_record_key":
> "1970/01/01", "_hoodie_partition_path": "files", "_hoodie_file_name":
> "files-0000_0-2363-6991_20211201085239959001.hfile", "key": "1970/01/01",
> "type": 2, "filesystemMetadata":
> {"44659a13-940a-4175-8666-42fd1d4c289d-0_4-1848-4325_20211201084520169.parquet":
> {"size": 6195400, "isDeleted": false},
> "44659a13-940a-4175-8666-42fd1d4c289d-0_4-2073-4842_20211201084722516.parquet":
> {"size": 6833466, "isDeleted": false},
> "44659a13-940a-4175-8666-42fd1d4c289d-0_14-2117-5043_20211201084748868.parquet":
> {"size": 8110687, "isDeleted": false},
> "44659a13-940a-4175-8666-42fd1d4c289d-0_14-2210-5726_20211201084932112.parquet":
> {"size": 11102072, "isDeleted": false},
> "44659a13-940a-4175-8666-42fd1d4c289d-0_14-2299-6470_20211201085132974.parquet":
> {"size": 13659745, "isDeleted": false},
> "44659a13-940a-4175-8666-42fd1d4c289d-0_14-2255-6066_20211201085029140.parquet":
> {"size": 12382272, "isDeleted": false},
> "44659a13-940a-4175-8666-42fd1d4c289d-0_4-2030-4747_20211201084658939.parquet":
> {"size": 6705717, "isDeleted": false},
> "44659a13-940a-4175-8666-42fd1d4c289d-0_4-1942-4555_20211201084613591.parquet":
> {"size": 6450598, "isDeleted": false},
> "44659a13-940a-4175-8666-42fd1d4c289d-0_14-2167-5388_20211201084839630.parquet":
> {"size": 9823267, "isDeleted": false},
> "44659a13-940a-4175-8666-42fd1d4c289d-0_4-1985-4650_20211201084634325.parquet":
> {"size": 6578112, "isDeleted": false},
> "44659a13-940a-4175-8666-42fd1d4c289d-0_4-1892-4453_20211201084546284.parquet":
> {"size": 6322913, "isDeleted": false},
> "44659a13-940a-4175-8666-42fd1d4c289d-0_14-2343-6841_20211201085239959.parquet":
> {"size": 14939682, "isDeleted": false}}}
> Rec 1970/01/02 => {"_hoodie_commit_time": "20211201085239959001",
> "_hoodie_commit_seqno": "20211201085239959001_0_101", "_hoodie_record_key":
> "1970/01/02", "_hoodie_partition_path": "files", "_hoodie_file_name":
> "files-0000_0-2363-6991_20211201085239959001.hfile", "key": "1970/01/02",
> "type": 2, "filesystemMetadata":
> {"78130c4f-45cf-480b-bb28-287f922f7dc3-0_8-2255-6060_20211201085029140.parquet":
> {"size": 12382043, "isDeleted": false},
> "78130c4f-45cf-480b-bb28-287f922f7dc3-0_8-2343-6835_20211201085239959.parquet":
> {"size": 14939570, "isDeleted": false},
> "78130c4f-45cf-480b-bb28-287f922f7dc3-0_2-1892-4451_20211201084546284.parquet":
> {"size": 6324655, "isDeleted": false},
> "78130c4f-45cf-480b-bb28-287f922f7dc3-0_8-2167-5382_20211201084839630.parquet":
> {"size": 9823518, "isDeleted": false},
> "78130c4f-45cf-480b-bb28-287f922f7dc3-0_8-2117-5037_20211201084748868.parquet":
> {"size": 8112193, "isDeleted": false},
> "78130c4f-45cf-480b-bb28-287f922f7dc3-0_2-2030-4745_20211201084658939.parquet":
> {"size": 6707347, "isDeleted": false},
> "78130c4f-45cf-480b-bb28-287f922f7dc3-0_2-1848-4323_20211201084520169.parquet":
> {"size": 6197274, "isDeleted": false},
> "78130c4f-45cf-480b-bb28-287f922f7dc3-0_2-1985-4648_20211201084634325.parquet":
> {"size": 6579727, "isDeleted": false},
> "78130c4f-45cf-480b-bb28-287f922f7dc3-0_2-2073-4840_20211201084722516.parquet":
> {"size": 6835314, "isDeleted": false},
> "78130c4f-45cf-480b-bb28-287f922f7dc3-0_8-2299-6464_20211201085132974.parquet":
> {"size": 13659572, "isDeleted": false},
> "78130c4f-45cf-480b-bb28-287f922f7dc3-0_2-1942-4553_20211201084613591.parquet":
> {"size": 6452356, "isDeleted": false},
> "78130c4f-45cf-480b-bb28-287f922f7dc3-0_8-2210-5720_20211201084932112.parquet":
> {"size": 11100792, "isDeleted": false}}}
> Rec 1970/01/03 => {"_hoodie_commit_time": "20211201085239959001",
> "_hoodie_commit_seqno": "20211201085239959001_0_102", "_hoodie_record_key":
> "1970/01/03", "_hoodie_partition_path": "files", "_hoodie_file_name":
> "files-0000_0-2363-6991_20211201085239959001.hfile", "key": "1970/01/03",
> "type": 2, "filesystemMetadata":
> {"c2037eb4-2a3c-4029-b43f-553879141587-0_3-1985-4649_20211201084634325.parquet":
> {"size": 6580071, "isDeleted": false},
> "c2037eb4-2a3c-4029-b43f-553879141587-0_11-2117-5040_20211201084748868.parquet":
> {"size": 8113910, "isDeleted": false},
> "c2037eb4-2a3c-4029-b43f-553879141587-0_3-2030-4746_20211201084658939.parquet":
> {"size": 6707747, "isDeleted": false},
> "c2037eb4-2a3c-4029-b43f-553879141587-0_11-2255-6063_20211201085029140.parquet":
> {"size": 12384338, "isDeleted": false},
> "c2037eb4-2a3c-4029-b43f-553879141587-0_11-2167-5385_20211201084839630.parquet":
> {"size": 9826663, "isDeleted": false},
> "c2037eb4-2a3c-4029-b43f-553879141587-0_11-2210-5723_20211201084932112.parquet":
> {"size": 11104039, "isDeleted": false},
> "c2037eb4-2a3c-4029-b43f-553879141587-0_3-1848-4324_20211201084520169.parquet":
> {"size": 6197292, "isDeleted": false},
> "c2037eb4-2a3c-4029-b43f-553879141587-0_11-2299-6467_20211201085132974.parquet":
> {"size": 13661858, "isDeleted": false},
> "c2037eb4-2a3c-4029-b43f-553879141587-0_11-2343-6838_20211201085239959.parquet":
> {"size": 14941707, "isDeleted": false},
> "c2037eb4-2a3c-4029-b43f-553879141587-0_3-2073-4841_20211201084722516.parquet":
> {"size": 6835421, "isDeleted": false},
> "c2037eb4-2a3c-4029-b43f-553879141587-0_3-1892-4452_20211201084546284.parquet":
> {"size": 6324866, "isDeleted": false},
> "c2037eb4-2a3c-4029-b43f-553879141587-0_3-1942-4554_20211201084613591.parquet":
> {"size": 6452600, "isDeleted": false}}}
> Rec 1970/01/04 => {"_hoodie_commit_time": "20211201085239959001",
> "_hoodie_commit_seqno": "20211201085239959001_0_103", "_hoodie_record_key":
> "1970/01/04", "_hoodie_partition_path": "files", "_hoodie_file_name":
> "files-0000_0-2363-6991_20211201085239959001.hfile", "key": "1970/01/04",
> "type": 2, "filesystemMetadata":
> {"30d15d17-43cc-46d0-9802-45633aadc832-0_0-1942-4551_20211201084613591.parquet":
> {"size": 6453250, "isDeleted": false},
> "30d15d17-43cc-46d0-9802-45633aadc832-0_0-1985-4646_20211201084634325.parquet":
> {"size": 6580820, "isDeleted": false},
> "30d15d17-43cc-46d0-9802-45633aadc832-0_4-2117-5033_20211201084748868.parquet":
> {"size": 8113305, "isDeleted": false},
> "30d15d17-43cc-46d0-9802-45633aadc832-0_4-2167-5378_20211201084839630.parquet":
> {"size": 9824513, "isDeleted": false},
> "30d15d17-43cc-46d0-9802-45633aadc832-0_4-2210-5716_20211201084932112.parquet":
> {"size": 11101478, "isDeleted": false},
> "30d15d17-43cc-46d0-9802-45633aadc832-0_4-2255-6056_20211201085029140.parquet":
> {"size": 12382664, "isDeleted": false},
> "30d15d17-43cc-46d0-9802-45633aadc832-0_0-1848-4321_20211201084520169.parquet":
> {"size": 6198035, "isDeleted": false},
> "30d15d17-43cc-46d0-9802-45633aadc832-0_4-2343-6831_20211201085239959.parquet":
> {"size": 14940122, "isDeleted": false},
> "30d15d17-43cc-46d0-9802-45633aadc832-0_0-1892-4449_20211201084546284.parquet":
> {"size": 6325539, "isDeleted": false},
> "30d15d17-43cc-46d0-9802-45633aadc832-0_4-2299-6460_20211201085132974.parquet":
> {"size": 13660273, "isDeleted": false},
> "30d15d17-43cc-46d0-9802-45633aadc832-0_0-2030-4743_20211201084658939.parquet":
> {"size": 6708514, "isDeleted": false},
> "30d15d17-43cc-46d0-9802-45633aadc832-0_0-2073-4838_20211201084722516.parquet":
> {"size": 6836137, "isDeleted": false}}}
> {code}
--
This message was sent by Atlassian Jira
(v8.20.1#820001)
