This is an automated email from the ASF dual-hosted git repository.
xiangfu0 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 2014abbdcea Copy the column set in the server table metadata endpoint
instead of narrowing a segment's own columns (#19665)
2014abbdcea is described below
commit 2014abbdceaa99abc17b05241d886b2a08e2ea24
Author: Xiang Fu <[email protected]>
AuthorDate: Sun Sep 27 03:20:53 2026 +0700
Copy the column set in the server table metadata endpoint instead of
narrowing a segment's own columns (#19665)
GET /tables/{table}/metadata?columns=* intersects the column sets of all
segments a server holds. It
took the first immutable segment's getAllColumns() as the running
intersection and retainAll'd every
later segment into it. That set is a live view of the segment's own Schema
(SegmentMetadata's default
returns Schema.getColumnNames(), the navigable key set of the field-spec
map), so the endpoint deleted
every column a later segment lacked from the first segment's metadata.
After schema evolution that is the
normal case: the newest segment loses its new columns, and SELECT * on it
silently drops them until the
segment is reloaded.
The endpoint now copies the first segment's column set before intersecting.
The regression test loads
three segments with pairwise different column sets, so the bug reproduces
on every iteration order, and
asserts that every segment's column names, metadata columns and schema are
unchanged afterwards.
---
.../pinot/server/api/resources/TablesResource.java | 4 +-
.../pinot/server/api/TablesResourceTest.java | 68 ++++++++++++++++++++++
2 files changed, 71 insertions(+), 1 deletion(-)
diff --git
a/pinot-server/src/main/java/org/apache/pinot/server/api/resources/TablesResource.java
b/pinot-server/src/main/java/org/apache/pinot/server/api/resources/TablesResource.java
index b4b4c35afee..1cd083fe7ce 100644
---
a/pinot-server/src/main/java/org/apache/pinot/server/api/resources/TablesResource.java
+++
b/pinot-server/src/main/java/org/apache/pinot/server/api/resources/TablesResource.java
@@ -252,7 +252,9 @@ public class TablesResource {
Set<String> allSegmentColumns = segmentMetadata.getAllColumns();
if (columnSet == null) {
- columnSet = allSegmentColumns;
+ // Copy: getAllColumns() is a view of the segment's own columns,
and retainAll below would otherwise
+ // narrow the first segment's metadata rather than the running
intersection.
+ columnSet = new HashSet<>(allSegmentColumns);
} else {
columnSet.retainAll(allSegmentColumns);
}
diff --git
a/pinot-server/src/test/java/org/apache/pinot/server/api/TablesResourceTest.java
b/pinot-server/src/test/java/org/apache/pinot/server/api/TablesResourceTest.java
index 3cff7dc2355..9b65b08225c 100644
---
a/pinot-server/src/test/java/org/apache/pinot/server/api/TablesResourceTest.java
+++
b/pinot-server/src/test/java/org/apache/pinot/server/api/TablesResourceTest.java
@@ -855,6 +855,74 @@ public class TablesResourceTest extends BaseResourceTest {
}
}
+ /// `GET /tables/{table}/metadata?columns=*` intersects the column sets of
all segments. The first segment's
+ /// `getAllColumns()` is a view of that segment's own metadata, so
intersecting in place would delete every column a
+ /// later segment lacks from the serving segment, and schema evolution makes
that the normal case. Three segments
+ /// with pairwise different column sets lose a column on every iteration
order if the view is narrowed in place.
+ @Test
+ public void testTableMetadataWithAllColumnsLeavesSegmentColumnsIntact()
+ throws Exception {
+ String tableName = "columnSetTable_OFFLINE";
+ List<ImmutableSegment> segments = new ArrayList<>();
+ addTable(tableName);
+ try {
+ segments.add(buildSegment(tableName, "allColumns", List.of("column1",
"column2", "column3")));
+ segments.add(buildSegment(tableName, "noColumn2", List.of("column1",
"column3")));
+ segments.add(buildSegment(tableName, "noColumn3", List.of("column1",
"column2")));
+ Map<String, Set<String>> columnsBefore = new HashMap<>();
+ for (ImmutableSegment segment : segments) {
+ _tableDataManagerMap.get(tableName).addSegment(segment);
+ columnsBefore.put(segment.getSegmentName(),
Set.copyOf(segment.getColumnNames()));
+ }
+
+ String response = _webTarget.path("/tables/" + tableName +
"/metadata").queryParam("columns", "*").request()
+ .get(String.class);
+ TableMetadataInfo metadata = JsonUtils.stringToObject(response,
TableMetadataInfo.class);
+
+ // The response covers the column every segment has ...
+ assertTrue(metadata.getColumnLengthMap().containsKey("column1"));
+ // ... and computing it left every segment's own column set untouched
+ for (ImmutableSegment segment : segments) {
+ String segmentName = segment.getSegmentName();
+ Set<String> expected = columnsBefore.get(segmentName);
+ assertEquals(Set.copyOf(segment.getColumnNames()), expected,
segmentName);
+ assertEquals(Set.copyOf(segment.getSegmentMetadata().getAllColumns()),
expected, segmentName);
+
assertEquals(Set.copyOf(segment.getSegmentMetadata().getSchema().getColumnNames()),
expected, segmentName);
+ }
+ } finally {
+ for (ImmutableSegment segment : segments) {
+ segment.offload();
+ segment.destroy();
+ }
+ _tableDataManagerMap.remove(tableName);
+ }
+ }
+
+ private ImmutableSegment buildSegment(String tableNameWithType, String
segmentName, List<String> columns)
+ throws Exception {
+ Schema.SchemaBuilder schemaBuilder =
+ new
Schema.SchemaBuilder().setSchemaName(TableNameBuilder.extractRawTableName(tableNameWithType));
+ for (String column : columns) {
+ schemaBuilder.addSingleValueDimension(column, DataType.INT);
+ }
+ List<GenericRow> rows = new ArrayList<>();
+ for (int i = 0; i < 10; i++) {
+ GenericRow row = new GenericRow();
+ for (String column : columns) {
+ row.putValue(column, i);
+ }
+ rows.add(row);
+ }
+ SegmentGeneratorConfig config = new SegmentGeneratorConfig(
+ new
TableConfigBuilder(TableType.OFFLINE).setTableName(tableNameWithType).build(),
schemaBuilder.build());
+ config.setOutDir(new File(_tempDir, tableNameWithType).getAbsolutePath());
+ config.setSegmentName(segmentName);
+ SegmentIndexCreationDriverImpl driver = new
SegmentIndexCreationDriverImpl();
+ driver.init(config, new GenericRowRecordReader(rows));
+ driver.build();
+ return ImmutableSegmentLoader.load(new File(config.getOutDir(),
driver.getSegmentName()), ReadMode.mmap);
+ }
+
// Override to use data with delete records
@Override
protected String getAvroFileName() {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
