mkaravel commented on code in PR #2971: URL: https://github.com/apache/parquet-java/pull/2971#discussion_r2011234930
########## parquet-cli/src/main/java/org/apache/parquet/cli/commands/ShowGeospatialStatisticsCommand.java: ########## @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.cli.commands; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; +import java.io.IOException; +import java.util.List; +import org.apache.commons.text.TextStringBuilder; +import org.apache.parquet.cli.BaseCommand; +import org.apache.parquet.column.statistics.geometry.GeospatialStatistics; +import org.apache.parquet.hadoop.ParquetFileReader; +import org.apache.parquet.hadoop.metadata.BlockMetaData; +import org.apache.parquet.hadoop.metadata.ColumnChunkMetaData; +import org.apache.parquet.hadoop.metadata.ParquetMetadata; +import org.apache.parquet.schema.MessageType; +import org.slf4j.Logger; + +@Parameters(commandDescription = "Print geospatial statistics for a Parquet file") +public class ShowGeospatialStatisticsCommand extends BaseCommand { + + public ShowGeospatialStatisticsCommand(Logger console) { + super(console); + } + + @Parameter(description = "<parquet path>") + List<String> targets; + + @Override + @SuppressWarnings("unchecked") + public int run() throws IOException { + Preconditions.checkArgument(targets != null && !targets.isEmpty(), "A Parquet file is required."); + Preconditions.checkArgument(targets.size() == 1, "Cannot process multiple Parquet files."); + + String source = targets.get(0); + try (ParquetFileReader reader = ParquetFileReader.open(getConf(), qualifiedPath(source))) { + ParquetMetadata footer = reader.getFooter(); + MessageType schema = footer.getFileMetaData().getSchema(); + + console.info("\nFile path: {}", source); + + List<BlockMetaData> rowGroups = footer.getBlocks(); + for (int index = 0, n = rowGroups.size(); index < n; index++) { + printRowGroupGeospatialStats(console, index, rowGroups.get(index), schema); + console.info(""); + } + } + + return 0; + } + + private void printRowGroupGeospatialStats(Logger console, int index, BlockMetaData rowGroup, MessageType schema) { + int maxColumnWidth = Math.max( + "column".length(), + rowGroup.getColumns().stream() + .map(col -> col.getPath().toString().length()) + .max(Integer::compare) + .orElse(0)); + + console.info(String.format("\nRow group %d\n%s", index, new TextStringBuilder(80).appendPadding(80, '-'))); + + String formatString = String.format("%%-%ds %%-15s %%-40s", maxColumnWidth); + console.info(String.format(formatString, "column", "bounding box", "geospatial types")); + + for (ColumnChunkMetaData column : rowGroup.getColumns()) { + printColumnGeospatialStats(console, column, schema, maxColumnWidth); + } + } + + private void printColumnGeospatialStats( Review Comment: What is the purpose of this method? Is it for display purposes only or does it serve some other purpose? The reason I am asking is because below we are printing bounding boxes in text format, and this can incur some rounding errors when converting a double value to string. ########## parquet-column/src/main/java/org/apache/parquet/column/statistics/geometry/GeospatialUtils.java: ########## @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.parquet.column.statistics.geometry; + +import org.locationtech.jts.geom.CoordinateSequence; +import org.locationtech.jts.geom.CoordinateSequenceFilter; +import org.locationtech.jts.geom.Geometry; + +class GeospatialUtils { + + public static void normalizeLongitude(Geometry geometry) { + if (geometry == null || geometry.isEmpty()) { + return; + } + + geometry.apply(new CoordinateSequenceFilter() { + @Override + public void filter(CoordinateSequence seq, int i) { + double x = seq.getX(i); + x = normalizeLongitude(x); + seq.setOrdinate(i, CoordinateSequence.X, x); + } + + @Override + public boolean isDone() { + return false; // Continue processing until all coordinates are processed + } + + @Override + public boolean isGeometryChanged() { + return true; // The geometry is changed as we are modifying the coordinates + } + }); + + geometry.geometryChanged(); // Notify the geometry that its coordinates have been changed + } Review Comment: IIUC this is modifying input data when writing. Is this a typical choice? Is the parquet reader/writer really allowed to modify data? ########## parquet-cli/src/main/java/org/apache/parquet/cli/commands/ShowGeospatialStatisticsCommand.java: ########## @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.cli.commands; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; +import java.io.IOException; +import java.util.List; +import org.apache.commons.text.TextStringBuilder; +import org.apache.parquet.cli.BaseCommand; +import org.apache.parquet.column.statistics.geometry.GeospatialStatistics; +import org.apache.parquet.hadoop.ParquetFileReader; +import org.apache.parquet.hadoop.metadata.BlockMetaData; +import org.apache.parquet.hadoop.metadata.ColumnChunkMetaData; +import org.apache.parquet.hadoop.metadata.ParquetMetadata; +import org.apache.parquet.schema.MessageType; +import org.slf4j.Logger; + +@Parameters(commandDescription = "Print geospatial statistics for a Parquet file") +public class ShowGeospatialStatisticsCommand extends BaseCommand { + + public ShowGeospatialStatisticsCommand(Logger console) { + super(console); + } + + @Parameter(description = "<parquet path>") + List<String> targets; + + @Override + @SuppressWarnings("unchecked") + public int run() throws IOException { + Preconditions.checkArgument(targets != null && !targets.isEmpty(), "A Parquet file is required."); + Preconditions.checkArgument(targets.size() == 1, "Cannot process multiple Parquet files."); + + String source = targets.get(0); + try (ParquetFileReader reader = ParquetFileReader.open(getConf(), qualifiedPath(source))) { + ParquetMetadata footer = reader.getFooter(); + MessageType schema = footer.getFileMetaData().getSchema(); + + console.info("\nFile path: {}", source); + + List<BlockMetaData> rowGroups = footer.getBlocks(); + for (int index = 0, n = rowGroups.size(); index < n; index++) { + printRowGroupGeospatialStats(console, index, rowGroups.get(index), schema); + console.info(""); + } + } + + return 0; + } + + private void printRowGroupGeospatialStats(Logger console, int index, BlockMetaData rowGroup, MessageType schema) { + int maxColumnWidth = Math.max( + "column".length(), + rowGroup.getColumns().stream() + .map(col -> col.getPath().toString().length()) + .max(Integer::compare) + .orElse(0)); + + console.info(String.format("\nRow group %d\n%s", index, new TextStringBuilder(80).appendPadding(80, '-'))); + + String formatString = String.format("%%-%ds %%-15s %%-40s", maxColumnWidth); + console.info(String.format(formatString, "column", "bounding box", "geospatial types")); + + for (ColumnChunkMetaData column : rowGroup.getColumns()) { + printColumnGeospatialStats(console, column, schema, maxColumnWidth); + } + } + + private void printColumnGeospatialStats( + Logger console, ColumnChunkMetaData column, MessageType schema, int columnWidth) { + GeospatialStatistics stats = column.getGeospatialStatistics(); + + if (stats != null && stats.isValid()) { + String boundingBox = + stats.getBoundingBox() != null ? stats.getBoundingBox().toString() : "-"; Review Comment: What is the intended behavior here if the column consists of empty geometries only? ########## parquet-column/src/test/java/org/apache/parquet/column/statistics/geometry/TestBoundingBox.java: ########## @@ -0,0 +1,192 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.column.statistics.geometry; + +import org.junit.Assert; +import org.junit.Test; +import org.locationtech.jts.geom.Coordinate; +import org.locationtech.jts.geom.GeometryFactory; +import org.locationtech.jts.geom.Point; +import org.locationtech.jts.geom.Polygon; + +public class TestBoundingBox { + + @Test + public void testUpdate() { + GeometryFactory geometryFactory = new GeometryFactory(); + BoundingBox boundingBox = new BoundingBox(); + + // Create a 2D point + Point point2D = geometryFactory.createPoint(new Coordinate(10, 20)); + boundingBox.update(point2D, "EPSG:4326"); + Assert.assertEquals(10.0, boundingBox.getXMin(), 0.0); + Assert.assertEquals(10.0, boundingBox.getXMax(), 0.0); + Assert.assertEquals(20.0, boundingBox.getYMin(), 0.0); + Assert.assertEquals(20.0, boundingBox.getYMax(), 0.0); + } + + @Test + public void testWraparound() { + GeometryFactory geometryFactory = new GeometryFactory(); + BoundingBox boundingBox = new BoundingBox(); + + // Create a polygon near the antimeridian line + Coordinate[] coords1 = new Coordinate[] { + new Coordinate(170, 10), new Coordinate(175, 15), new Coordinate(170, 15), new Coordinate(170, 10) + }; + Polygon polygon1 = geometryFactory.createPolygon(coords1); + boundingBox.update(polygon1, "EPSG:4326"); + // Check if the wraparound is handled correctly + Assert.assertEquals(170.0, boundingBox.getXMin(), 0.0); + Assert.assertEquals(175.0, boundingBox.getXMax(), 0.0); + Assert.assertEquals(10.0, boundingBox.getYMin(), 0.0); + Assert.assertEquals(15.0, boundingBox.getYMax(), 0.0); + + // Create an additional polygon crossing the antimeridian line + Coordinate[] coords2 = new Coordinate[] { + new Coordinate(175, -10), new Coordinate(-175, -5), new Coordinate(175, -5), new Coordinate(175, -10) + }; + Polygon polygon2 = geometryFactory.createPolygon(coords2); + + boundingBox.update(polygon2, "EPSG:4326"); + // Check if the wraparound is handled correctly + Assert.assertEquals(175.0, boundingBox.getXMin(), 0.0); + Assert.assertEquals(-175.0, boundingBox.getXMax(), 0.0); + Assert.assertEquals(-10.0, boundingBox.getYMin(), 0.0); + Assert.assertEquals(15.0, boundingBox.getYMax(), 0.0); + + // Create another polygon on the other side of the antimeridian line + Coordinate[] coords3 = new Coordinate[] { + new Coordinate(-170, 20), new Coordinate(-165, 25), new Coordinate(-170, 25), new Coordinate(-170, 20) + }; + // longitude range: [-170, -165] + Polygon polygon3 = geometryFactory.createPolygon(coords3); + boundingBox.update(polygon3, "EPSG:4326"); + + // Check if the wraparound is handled correctly + Assert.assertEquals(175.0, boundingBox.getXMin(), 0.0); + Assert.assertEquals(-175.0, boundingBox.getXMax(), 0.0); + Assert.assertEquals(-10.0, boundingBox.getYMin(), 0.0); + Assert.assertEquals(25.0, boundingBox.getYMax(), 0.0); + } + + @Test + public void testEmptyGeometry() { + GeometryFactory geometryFactory = new GeometryFactory(); + BoundingBox boundingBox = new BoundingBox(); + + // Create an empty point + Point emptyPoint = geometryFactory.createPoint(); + boundingBox.update(emptyPoint, "EPSG:4326"); + + // Empty geometry should retain the initial -Inf/Inf state + Assert.assertEquals(Double.POSITIVE_INFINITY, boundingBox.getXMin(), 0.0); + Assert.assertEquals(Double.NEGATIVE_INFINITY, boundingBox.getXMax(), 0.0); + Assert.assertEquals(Double.POSITIVE_INFINITY, boundingBox.getYMin(), 0.0); + Assert.assertEquals(Double.NEGATIVE_INFINITY, boundingBox.getYMax(), 0.0); Review Comment: Why do we return such a box in this case? There is nothing in the spec that specifies behavior for a column of empty geometries. Why not return NaN values? ########## parquet-column/src/main/java/org/apache/parquet/column/statistics/geometry/BoundingBox.java: ########## @@ -0,0 +1,264 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.column.statistics.geometry; + +import org.apache.parquet.Preconditions; +import org.locationtech.jts.geom.Coordinate; +import org.locationtech.jts.geom.Envelope; +import org.locationtech.jts.geom.Geometry; + +public class BoundingBox { + + private boolean allowWraparound = true; + + private double xMin = Double.POSITIVE_INFINITY; + private double xMax = Double.NEGATIVE_INFINITY; + private double yMin = Double.POSITIVE_INFINITY; + private double yMax = Double.NEGATIVE_INFINITY; + private double zMin = Double.POSITIVE_INFINITY; + private double zMax = Double.NEGATIVE_INFINITY; + private double mMin = Double.POSITIVE_INFINITY; + private double mMax = Double.NEGATIVE_INFINITY; + + public BoundingBox( + double xMin, double xMax, double yMin, double yMax, double zMin, double zMax, double mMin, double mMax) { + this.xMin = xMin; + this.xMax = xMax; + this.yMin = yMin; + this.yMax = yMax; + this.zMin = zMin; + this.zMax = zMax; + this.mMin = mMin; + this.mMax = mMax; + } + + public BoundingBox() {} + + void enableWraparound(boolean enable) { + allowWraparound = enable; + } + + public double getXMin() { + return xMin; + } + + public double getXMax() { + return xMax; + } + + public double getYMin() { + return yMin; + } + + public double getYMax() { + return yMax; + } + + public double getZMin() { + return zMin; + } + + public double getZMax() { + return zMax; + } + + public double getMMin() { + return mMin; + } + + public double getMMax() { + return mMax; + } + + // Method to update the bounding box with the coordinates of a Geometry object + // geometry can be changed by this method + void update(Geometry geometry, String crs) { + if (geometry == null) { + // If geometry is null, abort Review Comment: Maybe add here in the comment that in this case the returned box contains NaN values? ########## parquet-column/src/main/java/org/apache/parquet/schema/PrimitiveType.java: ########## @@ -271,6 +271,18 @@ public Optional<PrimitiveComparator> visit( LogicalTypeAnnotation.BsonLogicalTypeAnnotation bsonLogicalType) { return of(PrimitiveComparator.UNSIGNED_LEXICOGRAPHICAL_BINARY_COMPARATOR); } + + @Override + public Optional<PrimitiveComparator> visit( + LogicalTypeAnnotation.GeometryLogicalTypeAnnotation geometryLogicalType) { + return of(PrimitiveComparator.UNSIGNED_LEXICOGRAPHICAL_BINARY_COMPARATOR); + } + + @Override + public Optional<PrimitiveComparator> visit( + LogicalTypeAnnotation.GeographyLogicalTypeAnnotation geographyLogicalType) { + return of(PrimitiveComparator.UNSIGNED_LEXICOGRAPHICAL_BINARY_COMPARATOR); + } Review Comment: What are these meant for? ########## parquet-column/src/main/java/org/apache/parquet/column/statistics/geometry/BoundingBox.java: ########## @@ -0,0 +1,264 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.column.statistics.geometry; + +import org.apache.parquet.Preconditions; +import org.locationtech.jts.geom.Coordinate; +import org.locationtech.jts.geom.Envelope; +import org.locationtech.jts.geom.Geometry; + +public class BoundingBox { + + private boolean allowWraparound = true; + + private double xMin = Double.POSITIVE_INFINITY; + private double xMax = Double.NEGATIVE_INFINITY; + private double yMin = Double.POSITIVE_INFINITY; + private double yMax = Double.NEGATIVE_INFINITY; + private double zMin = Double.POSITIVE_INFINITY; + private double zMax = Double.NEGATIVE_INFINITY; + private double mMin = Double.POSITIVE_INFINITY; + private double mMax = Double.NEGATIVE_INFINITY; + + public BoundingBox( + double xMin, double xMax, double yMin, double yMax, double zMin, double zMax, double mMin, double mMax) { + this.xMin = xMin; + this.xMax = xMax; + this.yMin = yMin; + this.yMax = yMax; + this.zMin = zMin; + this.zMax = zMax; + this.mMin = mMin; + this.mMax = mMax; + } + + public BoundingBox() {} + + void enableWraparound(boolean enable) { + allowWraparound = enable; + } + + public double getXMin() { + return xMin; + } + + public double getXMax() { + return xMax; + } + + public double getYMin() { + return yMin; + } + + public double getYMax() { + return yMax; + } + + public double getZMin() { + return zMin; + } + + public double getZMax() { + return zMax; + } + + public double getMMin() { + return mMin; + } + + public double getMMax() { + return mMax; + } + + // Method to update the bounding box with the coordinates of a Geometry object + // geometry can be changed by this method + void update(Geometry geometry, String crs) { + if (geometry == null) { + // If geometry is null, abort + abort(); + return; + } + if (geometry.isEmpty()) { + // For empty geometries, keep track of the geometry type but set bounds to empty interval + return; // Keep the initial -Inf/Inf state for bounds + } + + if (shouldNormalizeLongitude(crs)) { + GeospatialUtils.normalizeLongitude(geometry); + } Review Comment: What is the reasoning behind normalization? We are operating on the Cartesian plane (for geometries) and the spec explicitly talks about linear/planar interpolation. ########## parquet-column/src/test/java/org/apache/parquet/column/statistics/geometry/TestBoundingBox.java: ########## @@ -0,0 +1,192 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.column.statistics.geometry; + +import org.junit.Assert; +import org.junit.Test; +import org.locationtech.jts.geom.Coordinate; +import org.locationtech.jts.geom.GeometryFactory; +import org.locationtech.jts.geom.Point; +import org.locationtech.jts.geom.Polygon; + +public class TestBoundingBox { + + @Test + public void testUpdate() { + GeometryFactory geometryFactory = new GeometryFactory(); + BoundingBox boundingBox = new BoundingBox(); + + // Create a 2D point + Point point2D = geometryFactory.createPoint(new Coordinate(10, 20)); + boundingBox.update(point2D, "EPSG:4326"); + Assert.assertEquals(10.0, boundingBox.getXMin(), 0.0); + Assert.assertEquals(10.0, boundingBox.getXMax(), 0.0); + Assert.assertEquals(20.0, boundingBox.getYMin(), 0.0); + Assert.assertEquals(20.0, boundingBox.getYMax(), 0.0); + } + + @Test + public void testWraparound() { + GeometryFactory geometryFactory = new GeometryFactory(); + BoundingBox boundingBox = new BoundingBox(); + + // Create a polygon near the antimeridian line + Coordinate[] coords1 = new Coordinate[] { + new Coordinate(170, 10), new Coordinate(175, 15), new Coordinate(170, 15), new Coordinate(170, 10) + }; + Polygon polygon1 = geometryFactory.createPolygon(coords1); + boundingBox.update(polygon1, "EPSG:4326"); + // Check if the wraparound is handled correctly + Assert.assertEquals(170.0, boundingBox.getXMin(), 0.0); + Assert.assertEquals(175.0, boundingBox.getXMax(), 0.0); + Assert.assertEquals(10.0, boundingBox.getYMin(), 0.0); + Assert.assertEquals(15.0, boundingBox.getYMax(), 0.0); + + // Create an additional polygon crossing the antimeridian line + Coordinate[] coords2 = new Coordinate[] { + new Coordinate(175, -10), new Coordinate(-175, -5), new Coordinate(175, -5), new Coordinate(175, -10) + }; + Polygon polygon2 = geometryFactory.createPolygon(coords2); + + boundingBox.update(polygon2, "EPSG:4326"); + // Check if the wraparound is handled correctly + Assert.assertEquals(175.0, boundingBox.getXMin(), 0.0); + Assert.assertEquals(-175.0, boundingBox.getXMax(), 0.0); + Assert.assertEquals(-10.0, boundingBox.getYMin(), 0.0); + Assert.assertEquals(15.0, boundingBox.getYMax(), 0.0); + + // Create another polygon on the other side of the antimeridian line + Coordinate[] coords3 = new Coordinate[] { + new Coordinate(-170, 20), new Coordinate(-165, 25), new Coordinate(-170, 25), new Coordinate(-170, 20) + }; + // longitude range: [-170, -165] + Polygon polygon3 = geometryFactory.createPolygon(coords3); + boundingBox.update(polygon3, "EPSG:4326"); + + // Check if the wraparound is handled correctly + Assert.assertEquals(175.0, boundingBox.getXMin(), 0.0); + Assert.assertEquals(-175.0, boundingBox.getXMax(), 0.0); + Assert.assertEquals(-10.0, boundingBox.getYMin(), 0.0); + Assert.assertEquals(25.0, boundingBox.getYMax(), 0.0); + } + + @Test + public void testEmptyGeometry() { + GeometryFactory geometryFactory = new GeometryFactory(); + BoundingBox boundingBox = new BoundingBox(); + + // Create an empty point + Point emptyPoint = geometryFactory.createPoint(); + boundingBox.update(emptyPoint, "EPSG:4326"); + + // Empty geometry should retain the initial -Inf/Inf state + Assert.assertEquals(Double.POSITIVE_INFINITY, boundingBox.getXMin(), 0.0); + Assert.assertEquals(Double.NEGATIVE_INFINITY, boundingBox.getXMax(), 0.0); + Assert.assertEquals(Double.POSITIVE_INFINITY, boundingBox.getYMin(), 0.0); + Assert.assertEquals(Double.NEGATIVE_INFINITY, boundingBox.getYMax(), 0.0); + + // Test that after adding a non-empty geometry, values are updated correctly + Point point = geometryFactory.createPoint(new Coordinate(10, 20)); + boundingBox.update(point, "EPSG:4326"); + Assert.assertEquals(10.0, boundingBox.getXMin(), 0.0); + Assert.assertEquals(10.0, boundingBox.getXMax(), 0.0); + Assert.assertEquals(20.0, boundingBox.getYMin(), 0.0); + Assert.assertEquals(20.0, boundingBox.getYMax(), 0.0); + + // Update with another empty geometry, should not change the bounds + boundingBox.update(emptyPoint, "EPSG:4326"); + Assert.assertEquals(10.0, boundingBox.getXMin(), 0.0); + Assert.assertEquals(10.0, boundingBox.getXMax(), 0.0); + Assert.assertEquals(20.0, boundingBox.getYMin(), 0.0); + Assert.assertEquals(20.0, boundingBox.getYMax(), 0.0); + } + + @Test + public void testNaNCoordinates() { + GeometryFactory geometryFactory = new GeometryFactory(); + BoundingBox boundingBox = new BoundingBox(); + + // Create a point with NaN coordinates + Point nanPoint = geometryFactory.createPoint(new Coordinate(Double.NaN, Double.NaN)); + boundingBox.update(nanPoint, "EPSG:4326"); + + // NaN values should be ignored, maintaining the initial -Inf/Inf state + Assert.assertEquals(Double.POSITIVE_INFINITY, boundingBox.getXMin(), 0.0); + Assert.assertEquals(Double.NEGATIVE_INFINITY, boundingBox.getXMax(), 0.0); + Assert.assertEquals(Double.POSITIVE_INFINITY, boundingBox.getYMin(), 0.0); + Assert.assertEquals(Double.NEGATIVE_INFINITY, boundingBox.getYMax(), 0.0); Review Comment: Same question here. ########## parquet-column/src/main/java/org/apache/parquet/column/statistics/geometry/BoundingBox.java: ########## @@ -0,0 +1,264 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.column.statistics.geometry; + +import org.apache.parquet.Preconditions; +import org.locationtech.jts.geom.Coordinate; +import org.locationtech.jts.geom.Envelope; +import org.locationtech.jts.geom.Geometry; + +public class BoundingBox { + + private boolean allowWraparound = true; + + private double xMin = Double.POSITIVE_INFINITY; + private double xMax = Double.NEGATIVE_INFINITY; + private double yMin = Double.POSITIVE_INFINITY; + private double yMax = Double.NEGATIVE_INFINITY; + private double zMin = Double.POSITIVE_INFINITY; + private double zMax = Double.NEGATIVE_INFINITY; + private double mMin = Double.POSITIVE_INFINITY; + private double mMax = Double.NEGATIVE_INFINITY; + + public BoundingBox( + double xMin, double xMax, double yMin, double yMax, double zMin, double zMax, double mMin, double mMax) { + this.xMin = xMin; + this.xMax = xMax; + this.yMin = yMin; + this.yMax = yMax; + this.zMin = zMin; + this.zMax = zMax; + this.mMin = mMin; + this.mMax = mMax; + } + + public BoundingBox() {} + + void enableWraparound(boolean enable) { + allowWraparound = enable; + } + + public double getXMin() { + return xMin; + } + + public double getXMax() { + return xMax; + } + + public double getYMin() { + return yMin; + } + + public double getYMax() { + return yMax; + } + + public double getZMin() { + return zMin; + } + + public double getZMax() { + return zMax; + } + + public double getMMin() { + return mMin; + } + + public double getMMax() { + return mMax; + } + + // Method to update the bounding box with the coordinates of a Geometry object + // geometry can be changed by this method + void update(Geometry geometry, String crs) { + if (geometry == null) { + // If geometry is null, abort + abort(); + return; + } + if (geometry.isEmpty()) { + // For empty geometries, keep track of the geometry type but set bounds to empty interval + return; // Keep the initial -Inf/Inf state for bounds + } + + if (shouldNormalizeLongitude(crs)) { + GeospatialUtils.normalizeLongitude(geometry); + } + + Envelope envelope = geometry.getEnvelopeInternal(); + double minX = envelope.getMinX(); + double minY = envelope.getMinY(); + double maxX = envelope.getMaxX(); + double maxY = envelope.getMaxY(); + + // Initialize Z and M values + double minZ = Double.POSITIVE_INFINITY; + double maxZ = Double.NEGATIVE_INFINITY; + double minM = Double.POSITIVE_INFINITY; + double maxM = Double.NEGATIVE_INFINITY; + + Coordinate[] coordinates = geometry.getCoordinates(); + for (Coordinate coord : coordinates) { + // Skip NaN values + if (!Double.isNaN(coord.x) && !Double.isNaN(coord.y)) { + if (!Double.isNaN(coord.getZ())) { + minZ = Math.min(minZ, coord.getZ()); + maxZ = Math.max(maxZ, coord.getZ()); + } + if (!Double.isNaN(coord.getM())) { + minM = Math.min(minM, coord.getM()); + maxM = Math.max(maxM, coord.getM()); + } + } + } + // Only update bounds if we have valid coordinates + if (!Double.isNaN(minX) && !Double.isNaN(maxX)) { + updateXBounds(minX, maxX, allowWraparound); + } + + if (!Double.isNaN(minY)) { + yMin = Math.min(yMin, minY); + } + if (!Double.isNaN(maxY)) { + yMax = Math.max(yMax, maxY); + } + + if (minZ != Double.POSITIVE_INFINITY) { + zMin = Math.min(zMin, minZ); + } + if (maxZ != Double.NEGATIVE_INFINITY) { + zMax = Math.max(zMax, maxZ); + } + + if (minM != Double.POSITIVE_INFINITY) { + mMin = Math.min(mMin, minM); + } + if (maxM != Double.NEGATIVE_INFINITY) { + mMax = Math.max(mMax, maxM); + } Review Comment: Based on the logic above, `minZ` (and all other similar) is initialized to `Double.POSITIVE_INFINITY` and gets updated only if `minZ` is not a NaN. At the same time `zMin` is initialized to `Double.POSITIVE_INFINITY`. What is the purpose of the `if` statement here? Wouldn't it be enough to just write: ```java zMin = Math.min(zMin, minZ); ``` Similarly for the other coordinates. ########## parquet-column/src/main/java/org/apache/parquet/column/statistics/geometry/BoundingBox.java: ########## @@ -0,0 +1,264 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.parquet.column.statistics.geometry; + +import org.apache.parquet.Preconditions; +import org.locationtech.jts.geom.Coordinate; +import org.locationtech.jts.geom.Envelope; +import org.locationtech.jts.geom.Geometry; + +public class BoundingBox { + + private boolean allowWraparound = true; + + private double xMin = Double.POSITIVE_INFINITY; + private double xMax = Double.NEGATIVE_INFINITY; + private double yMin = Double.POSITIVE_INFINITY; + private double yMax = Double.NEGATIVE_INFINITY; + private double zMin = Double.POSITIVE_INFINITY; + private double zMax = Double.NEGATIVE_INFINITY; + private double mMin = Double.POSITIVE_INFINITY; + private double mMax = Double.NEGATIVE_INFINITY; + + public BoundingBox( + double xMin, double xMax, double yMin, double yMax, double zMin, double zMax, double mMin, double mMax) { + this.xMin = xMin; + this.xMax = xMax; + this.yMin = yMin; + this.yMax = yMax; + this.zMin = zMin; + this.zMax = zMax; + this.mMin = mMin; + this.mMax = mMax; + } + + public BoundingBox() {} + + void enableWraparound(boolean enable) { + allowWraparound = enable; + } + + public double getXMin() { + return xMin; + } + + public double getXMax() { + return xMax; + } + + public double getYMin() { + return yMin; + } + + public double getYMax() { + return yMax; + } + + public double getZMin() { + return zMin; + } + + public double getZMax() { + return zMax; + } + + public double getMMin() { + return mMin; + } + + public double getMMax() { + return mMax; + } + + // Method to update the bounding box with the coordinates of a Geometry object + // geometry can be changed by this method + void update(Geometry geometry, String crs) { + if (geometry == null) { + // If geometry is null, abort + abort(); + return; + } + if (geometry.isEmpty()) { + // For empty geometries, keep track of the geometry type but set bounds to empty interval + return; // Keep the initial -Inf/Inf state for bounds + } + + if (shouldNormalizeLongitude(crs)) { + GeospatialUtils.normalizeLongitude(geometry); + } + + Envelope envelope = geometry.getEnvelopeInternal(); + double minX = envelope.getMinX(); + double minY = envelope.getMinY(); + double maxX = envelope.getMaxX(); + double maxY = envelope.getMaxY(); + + // Initialize Z and M values + double minZ = Double.POSITIVE_INFINITY; + double maxZ = Double.NEGATIVE_INFINITY; + double minM = Double.POSITIVE_INFINITY; + double maxM = Double.NEGATIVE_INFINITY; + + Coordinate[] coordinates = geometry.getCoordinates(); + for (Coordinate coord : coordinates) { + // Skip NaN values + if (!Double.isNaN(coord.x) && !Double.isNaN(coord.y)) { + if (!Double.isNaN(coord.getZ())) { + minZ = Math.min(minZ, coord.getZ()); + maxZ = Math.max(maxZ, coord.getZ()); + } + if (!Double.isNaN(coord.getM())) { + minM = Math.min(minM, coord.getM()); + maxM = Math.max(maxM, coord.getM()); + } + } + } + // Only update bounds if we have valid coordinates + if (!Double.isNaN(minX) && !Double.isNaN(maxX)) { + updateXBounds(minX, maxX, allowWraparound); + } + + if (!Double.isNaN(minY)) { + yMin = Math.min(yMin, minY); + } + if (!Double.isNaN(maxY)) { + yMax = Math.max(yMax, maxY); + } + + if (minZ != Double.POSITIVE_INFINITY) { + zMin = Math.min(zMin, minZ); + } + if (maxZ != Double.NEGATIVE_INFINITY) { + zMax = Math.max(zMax, maxZ); + } + + if (minM != Double.POSITIVE_INFINITY) { + mMin = Math.min(mMin, minM); + } + if (maxM != Double.NEGATIVE_INFINITY) { + mMax = Math.max(mMax, maxM); + } + } + + void merge(BoundingBox other) { + Preconditions.checkArgument(other != null, "Cannot merge with null bounding box"); + double minX = other.xMin; + double maxX = other.xMax; + + updateXBounds(minX, maxX, allowWraparound); + + yMin = Math.min(yMin, other.yMin); + yMax = Math.max(yMax, other.yMax); + zMin = Math.min(zMin, other.zMin); + zMax = Math.max(zMax, other.zMax); + mMin = Math.min(mMin, other.mMin); + mMax = Math.max(mMax, other.mMax); + } + + public void reset() { + xMin = Double.POSITIVE_INFINITY; + xMax = Double.NEGATIVE_INFINITY; + yMin = Double.POSITIVE_INFINITY; + yMax = Double.NEGATIVE_INFINITY; + zMin = Double.POSITIVE_INFINITY; + zMax = Double.NEGATIVE_INFINITY; + mMin = Double.POSITIVE_INFINITY; + mMax = Double.NEGATIVE_INFINITY; + } + + public void abort() { + xMin = Double.NaN; + xMax = Double.NaN; + yMin = Double.NaN; + yMax = Double.NaN; + zMin = Double.NaN; + zMax = Double.NaN; + mMin = Double.NaN; + mMax = Double.NaN; + } + + private boolean isCrossingAntiMeridian(double x1, double x2) { + return Math.abs(x1 - x2) > 180; + } + + private void updateXBounds(double minX, double maxX, boolean allowWraparound) { + if (!allowWraparound) { + xMin = Math.min(xMin, minX); + xMax = Math.max(xMax, maxX); + } else { + if (xMin == Double.POSITIVE_INFINITY || xMax == Double.NEGATIVE_INFINITY) { + xMin = minX; + xMax = maxX; + } else { + if (!isCrossingAntiMeridian(xMax, xMin)) { + if (!isCrossingAntiMeridian(maxX, minX)) { + xMin = Math.min(xMin, minX); + xMax = Math.max(xMax, maxX); + } else { + xMin = Math.max(xMin, maxX); + xMax = Math.min(xMax, minX); + } + } else { + if (!isCrossingAntiMeridian(maxX, minX)) { + xMin = Math.max(xMin, minX); + xMax = Math.min(xMax, maxX); + } else { + xMin = Math.max(xMin, maxX); + xMax = Math.min(xMax, minX); + } + } + } + } + } + + /** + * Determines if the longitude should be normalized based on the given CRS (Coordinate Reference System). + * Normalization is required only when the CRS is set to OGC:CRS84, EPSG:4326, or SRID:4326 (case insensitive). + * + * @param crs the Coordinate Reference System string + * @return true if the longitude should be normalized, false otherwise + */ + private boolean shouldNormalizeLongitude(String crs) { + if (crs == null) { + return false; + } + String normalizedCrs = crs.trim().toUpperCase(); + return "OGC:CRS84".equals(normalizedCrs) + || "EPSG:4326".equals(normalizedCrs) + || "SRID:4326".equals(normalizedCrs); Review Comment: This is the part that puzzles me the most in this PR: why do we normalize for CRS84 only? What about all other coordinate reference systems that are geographic? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: issues-unsubscr...@parquet.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org --------------------------------------------------------------------- To unsubscribe, e-mail: issues-unsubscr...@parquet.apache.org For additional commands, e-mail: issues-h...@parquet.apache.org
