This is an automated email from the ASF dual-hosted git repository. imbruced pushed a commit to branch feature/geopackage-reader in repository https://gitbox.apache.org/repos/asf/sedona.git
commit c411456f7be4220b009e1f6dff33e5d7e786b7d3 Author: pawelkocinski <[email protected]> AuthorDate: Mon Sep 23 23:21:42 2024 +0200 Add other missing data types. --- .../connection/GeoPackageConnectionManager.scala | 0 .../geopackage/errors/GeopackageException.scala | 0 .../datasources/geopackage/model/Envelope.scala | 0 .../geopackage/model/GeoPackageField.scala | 0 .../geopackage/model/GeoPackageType.scala | 0 .../geopackage/model/ImageFileFormat.scala | 0 .../geopackage/model/PartitionOptions.scala | 0 .../datasources/geopackage/model/TableType.scala | 0 .../datasources/geopackage/model/TileMatrix.scala | 0 .../geopackage/model/TileMetadata.scala | 0 .../geopackage/model/TileRowMetadata.scala | 0 .../transform/DataTypesTransformations.scala | 0 .../geopackage/transform/GeometryReader.scala | 0 .../datasources/geopackage/transform/Image.scala | 0 .../geopackage/transform/ValuesMapper.scala | 0 .../datasources/geopackage/GeoPackageScan.scala | 1 - .../geopackage/GeoPackageScanBuilder.scala | 3 - .../datasources/geopackage/GeoPackageTable.scala | 4 +- ...org.apache.spark.sql.sources.DataSourceRegister | 1 + .../geopackage/GeoPackageDataSource.scala | 88 ++++++++++++++++++++++ .../geopackage/GeoPackageLoadOptions.scala} | 15 ++-- .../geopackage/GeoPackageMetadataReader.scala} | 4 +- .../geopackage/GeoPackagePartitionReader.scala | 76 +++++++++++++++++++ .../GeoPackagePartitionReaderFactory.scala | 46 +++++++++++ .../datasources/geopackage/GeoPackageScan.scala | 3 +- .../geopackage/GeoPackageScanBuilder.scala | 5 +- .../datasources/geopackage/GeoPackageTable.scala | 6 +- .../apache/sedona/sql/GeoPackageReaderTest.scala} | 21 +++++- ...org.apache.spark.sql.sources.DataSourceRegister | 1 + .../geopackage/GeoPackageDataSource.scala | 88 ++++++++++++++++++++++ .../geopackage/GeoPackageLoadOptions.scala} | 15 ++-- .../geopackage/GeoPackageMetadataReader.scala} | 4 +- .../geopackage/GeoPackagePartitionReader.scala | 76 +++++++++++++++++++ .../GeoPackagePartitionReaderFactory.scala | 46 +++++++++++ .../datasources/geopackage/GeoPackageScan.scala | 3 +- .../geopackage/GeoPackageScanBuilder.scala | 5 +- .../datasources/geopackage/GeoPackageTable.scala | 6 +- .../apache/sedona/sql/GeoPackageReaderTest.scala} | 22 +++++- 38 files changed, 490 insertions(+), 49 deletions(-) diff --git a/spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/geopackage/connection/GeoPackageConnectionManager.scala b/spark/common/src/main/scala/org/apache/sedona/sql/datasources/geopackage/connection/GeoPackageConnectionManager.scala similarity index 100% rename from spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/geopackage/connection/GeoPackageConnectionManager.scala rename to spark/common/src/main/scala/org/apache/sedona/sql/datasources/geopackage/connection/GeoPackageConnectionManager.scala diff --git a/spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/geopackage/errors/GeopackageException.scala b/spark/common/src/main/scala/org/apache/sedona/sql/datasources/geopackage/errors/GeopackageException.scala similarity index 100% copy from spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/geopackage/errors/GeopackageException.scala copy to spark/common/src/main/scala/org/apache/sedona/sql/datasources/geopackage/errors/GeopackageException.scala diff --git a/spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/geopackage/model/Envelope.scala b/spark/common/src/main/scala/org/apache/sedona/sql/datasources/geopackage/model/Envelope.scala similarity index 100% rename from spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/geopackage/model/Envelope.scala rename to spark/common/src/main/scala/org/apache/sedona/sql/datasources/geopackage/model/Envelope.scala diff --git a/spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/geopackage/model/GeoPackageField.scala b/spark/common/src/main/scala/org/apache/sedona/sql/datasources/geopackage/model/GeoPackageField.scala similarity index 100% rename from spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/geopackage/model/GeoPackageField.scala rename to spark/common/src/main/scala/org/apache/sedona/sql/datasources/geopackage/model/GeoPackageField.scala diff --git a/spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/geopackage/model/GeoPackageType.scala b/spark/common/src/main/scala/org/apache/sedona/sql/datasources/geopackage/model/GeoPackageType.scala similarity index 100% rename from spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/geopackage/model/GeoPackageType.scala rename to spark/common/src/main/scala/org/apache/sedona/sql/datasources/geopackage/model/GeoPackageType.scala diff --git a/spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/geopackage/model/ImageFileFormat.scala b/spark/common/src/main/scala/org/apache/sedona/sql/datasources/geopackage/model/ImageFileFormat.scala similarity index 100% rename from spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/geopackage/model/ImageFileFormat.scala rename to spark/common/src/main/scala/org/apache/sedona/sql/datasources/geopackage/model/ImageFileFormat.scala diff --git a/spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/geopackage/model/PartitionOptions.scala b/spark/common/src/main/scala/org/apache/sedona/sql/datasources/geopackage/model/PartitionOptions.scala similarity index 100% rename from spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/geopackage/model/PartitionOptions.scala rename to spark/common/src/main/scala/org/apache/sedona/sql/datasources/geopackage/model/PartitionOptions.scala diff --git a/spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/geopackage/model/TableType.scala b/spark/common/src/main/scala/org/apache/sedona/sql/datasources/geopackage/model/TableType.scala similarity index 100% copy from spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/geopackage/model/TableType.scala copy to spark/common/src/main/scala/org/apache/sedona/sql/datasources/geopackage/model/TableType.scala diff --git a/spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/geopackage/model/TileMatrix.scala b/spark/common/src/main/scala/org/apache/sedona/sql/datasources/geopackage/model/TileMatrix.scala similarity index 100% rename from spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/geopackage/model/TileMatrix.scala rename to spark/common/src/main/scala/org/apache/sedona/sql/datasources/geopackage/model/TileMatrix.scala diff --git a/spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/geopackage/model/TileMetadata.scala b/spark/common/src/main/scala/org/apache/sedona/sql/datasources/geopackage/model/TileMetadata.scala similarity index 100% rename from spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/geopackage/model/TileMetadata.scala rename to spark/common/src/main/scala/org/apache/sedona/sql/datasources/geopackage/model/TileMetadata.scala diff --git a/spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/geopackage/model/TileRowMetadata.scala b/spark/common/src/main/scala/org/apache/sedona/sql/datasources/geopackage/model/TileRowMetadata.scala similarity index 100% copy from spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/geopackage/model/TileRowMetadata.scala copy to spark/common/src/main/scala/org/apache/sedona/sql/datasources/geopackage/model/TileRowMetadata.scala diff --git a/spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/geopackage/transform/DataTypesTransformations.scala b/spark/common/src/main/scala/org/apache/sedona/sql/datasources/geopackage/transform/DataTypesTransformations.scala similarity index 100% rename from spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/geopackage/transform/DataTypesTransformations.scala rename to spark/common/src/main/scala/org/apache/sedona/sql/datasources/geopackage/transform/DataTypesTransformations.scala diff --git a/spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/geopackage/transform/GeometryReader.scala b/spark/common/src/main/scala/org/apache/sedona/sql/datasources/geopackage/transform/GeometryReader.scala similarity index 100% rename from spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/geopackage/transform/GeometryReader.scala rename to spark/common/src/main/scala/org/apache/sedona/sql/datasources/geopackage/transform/GeometryReader.scala diff --git a/spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/geopackage/transform/Image.scala b/spark/common/src/main/scala/org/apache/sedona/sql/datasources/geopackage/transform/Image.scala similarity index 100% rename from spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/geopackage/transform/Image.scala rename to spark/common/src/main/scala/org/apache/sedona/sql/datasources/geopackage/transform/Image.scala diff --git a/spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/geopackage/transform/ValuesMapper.scala b/spark/common/src/main/scala/org/apache/sedona/sql/datasources/geopackage/transform/ValuesMapper.scala similarity index 100% rename from spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/geopackage/transform/ValuesMapper.scala rename to spark/common/src/main/scala/org/apache/sedona/sql/datasources/geopackage/transform/ValuesMapper.scala diff --git a/spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/geopackage/GeoPackageScan.scala b/spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/geopackage/GeoPackageScan.scala index 289ad87e5..1f63813ce 100644 --- a/spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/geopackage/GeoPackageScan.scala +++ b/spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/geopackage/GeoPackageScan.scala @@ -18,7 +18,6 @@ */ package org.apache.sedona.sql.datasources.geopackage -import org.apache.sedona.sql.datasources.geopackage.model.TableType.TableType import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.expressions.Expression import org.apache.spark.sql.connector.read.PartitionReaderFactory diff --git a/spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/geopackage/GeoPackageScanBuilder.scala b/spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/geopackage/GeoPackageScanBuilder.scala index ada2f91a6..529e5cd22 100644 --- a/spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/geopackage/GeoPackageScanBuilder.scala +++ b/spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/geopackage/GeoPackageScanBuilder.scala @@ -18,14 +18,11 @@ */ package org.apache.sedona.sql.datasources.geopackage -import org.apache.sedona.sql.datasources.geopackage.connection.GeoPackageConnectionManager -import org.apache.sedona.sql.datasources.geopackage.model.TableType import org.apache.spark.sql.SparkSession import org.apache.spark.sql.connector.read.Scan import org.apache.spark.sql.execution.datasources.PartitioningAwareFileIndex import org.apache.spark.sql.execution.datasources.v2.FileScanBuilder import org.apache.spark.sql.types.StructType -import org.apache.spark.sql.util.CaseInsensitiveStringMap class GeoPackageScanBuilder( sparkSession: SparkSession, diff --git a/spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/geopackage/GeoPackageTable.scala b/spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/geopackage/GeoPackageTable.scala index 8cb059b7b..43136fd9f 100644 --- a/spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/geopackage/GeoPackageTable.scala +++ b/spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/geopackage/GeoPackageTable.scala @@ -19,14 +19,12 @@ package org.apache.sedona.sql.datasources.geopackage import org.apache.hadoop.fs.FileStatus -import org.apache.sedona.sql.datasources.geopackage.connection.GeoPackageConnectionManager -import org.apache.sedona.sql.datasources.geopackage.model.TableType import org.apache.spark.sql.SparkSession import org.apache.spark.sql.connector.read.ScanBuilder import org.apache.spark.sql.connector.write.{LogicalWriteInfo, WriteBuilder} import org.apache.spark.sql.execution.datasources.FileFormat import org.apache.spark.sql.execution.datasources.v2.FileTable -import org.apache.spark.sql.types.{StringType, StructField, StructType} +import org.apache.spark.sql.types.StructType import org.apache.spark.sql.util.CaseInsensitiveStringMap case class GeoPackageTable( diff --git a/spark/spark-3.4/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister b/spark/spark-3.4/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister index d2f1d0340..39b7d446c 100644 --- a/spark/spark-3.4/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister +++ b/spark/spark-3.4/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister @@ -1,3 +1,4 @@ org.apache.spark.sql.execution.datasources.parquet.GeoParquetFileFormat org.apache.spark.sql.execution.datasources.v2.geoparquet.metadata.GeoParquetMetadataDataSource org.apache.sedona.sql.datasources.shapefile.ShapefileDataSource +org.apache.sedona.sql.datasources.geopackage.GeoPackageDataSource diff --git a/spark/spark-3.4/src/main/scala/org/apache/spark/sql/execution/datasources/geopackage/GeoPackageDataSource.scala b/spark/spark-3.4/src/main/scala/org/apache/spark/sql/execution/datasources/geopackage/GeoPackageDataSource.scala new file mode 100644 index 000000000..08c52ecdb --- /dev/null +++ b/spark/spark-3.4/src/main/scala/org/apache/spark/sql/execution/datasources/geopackage/GeoPackageDataSource.scala @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.spark.sql.execution.datasources.geopackage + +import org.apache.sedona.sql.datasources.geopackage.connection.GeoPackageConnectionManager +import org.apache.sedona.sql.datasources.geopackage.model.TableType +import org.apache.sedona.sql.datasources.geopackage.model.TableType.TableType +import org.apache.spark.sql.connector.catalog.Table +import org.apache.spark.sql.execution.datasources.FileFormat +import org.apache.spark.sql.execution.datasources.v2.FileDataSourceV2 +import org.apache.spark.sql.sources.DataSourceRegister +import org.apache.spark.sql.util.CaseInsensitiveStringMap + +class GeoPackageDataSource extends FileDataSourceV2 with DataSourceRegister { + + override def fallbackFileFormat: Class[_ <: FileFormat] = { + null + } + + override protected def getTable(options: CaseInsensitiveStringMap): Table = { + val loadOptions = getLoadOptions(options) + + GeoPackageTable( + "", + sparkSession, + options, + Seq(loadOptions.path), + None, + fallbackFileFormat, + loadOptions) + } + + private def getLoadOptions(options: CaseInsensitiveStringMap): GeoPackageLoadOptions = { + val path = options.get("path") + if (path.isEmpty) { + throw new IllegalArgumentException("GeoPackage path is not specified") + } + + val showMetadata = options.getBoolean("showMetadata", false) + val maybeTableName = options.get("tableName") + + if (!showMetadata && maybeTableName == null) { + throw new IllegalArgumentException("Table name is not specified") + } + + val tableName = if (showMetadata) { + "gpkg_contents" + } else { + maybeTableName + } + + val fields = GeoPackageConnectionManager + .getSchema(path, tableName) + + GeoPackageLoadOptions( + path = path, + showMetadata = showMetadata, + tableName = tableName, + tableType = getTableType(showMetadata = showMetadata, path = path, tableName = tableName), + fields = fields) + } + + private def getTableType(showMetadata: Boolean, path: String, tableName: String): TableType = { + if (showMetadata) { + TableType.METADATA + } else { + GeoPackageConnectionManager.findFeatureMetadata(path, tableName) + } + } + + override def shortName(): String = "geopackage" +} diff --git a/spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/geopackage/model/TableType.scala b/spark/spark-3.4/src/main/scala/org/apache/spark/sql/execution/datasources/geopackage/GeoPackageLoadOptions.scala similarity index 68% copy from spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/geopackage/model/TableType.scala copy to spark/spark-3.4/src/main/scala/org/apache/spark/sql/execution/datasources/geopackage/GeoPackageLoadOptions.scala index 774419232..1c0090ae3 100644 --- a/spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/geopackage/model/TableType.scala +++ b/spark/spark-3.4/src/main/scala/org/apache/spark/sql/execution/datasources/geopackage/GeoPackageLoadOptions.scala @@ -16,9 +16,14 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.sedona.sql.datasources.geopackage.model +package org.apache.spark.sql.execution.datasources.geopackage -object TableType extends Enumeration { - type TableType = Value - val FEATURES, TILES, METADATA, UNKNOWN = Value -} +import org.apache.sedona.sql.datasources.geopackage.model.GeoPackageField +import org.apache.sedona.sql.datasources.geopackage.model.TableType.TableType + +case class GeoPackageLoadOptions( + path: String, + showMetadata: Boolean, + tableName: String, + tableType: TableType, + fields: Seq[GeoPackageField]) diff --git a/spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/geopackage/model/TileRowMetadata.scala b/spark/spark-3.4/src/main/scala/org/apache/spark/sql/execution/datasources/geopackage/GeoPackageMetadataReader.scala similarity index 85% copy from spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/geopackage/model/TileRowMetadata.scala copy to spark/spark-3.4/src/main/scala/org/apache/spark/sql/execution/datasources/geopackage/GeoPackageMetadataReader.scala index e3fcb4012..315fb58ae 100644 --- a/spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/geopackage/model/TileRowMetadata.scala +++ b/spark/spark-3.4/src/main/scala/org/apache/spark/sql/execution/datasources/geopackage/GeoPackageMetadataReader.scala @@ -16,6 +16,6 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.sedona.sql.datasources.geopackage.model +package org.apache.spark.sql.execution.datasources.geopackage -case class TileRowMetadata(tileColumn: Int, tileRow: Int, zoomLevel: Int) +class GeoPackageMetadataReader {} diff --git a/spark/spark-3.4/src/main/scala/org/apache/spark/sql/execution/datasources/geopackage/GeoPackagePartitionReader.scala b/spark/spark-3.4/src/main/scala/org/apache/spark/sql/execution/datasources/geopackage/GeoPackagePartitionReader.scala new file mode 100644 index 000000000..94262388b --- /dev/null +++ b/spark/spark-3.4/src/main/scala/org/apache/spark/sql/execution/datasources/geopackage/GeoPackagePartitionReader.scala @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.spark.sql.execution.datasources.geopackage + +import org.apache.sedona.sql.datasources.geopackage.connection.GeoPackageConnectionManager +import org.apache.sedona.sql.datasources.geopackage.model.TableType.{FEATURES, METADATA, TILES, UNKNOWN} +import org.apache.sedona.sql.datasources.geopackage.model.{PartitionOptions, TileRowMetadata} +import org.apache.sedona.sql.datasources.geopackage.transform.ValuesMapper +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.connector.read.PartitionReader + +import java.sql.ResultSet + +case class GeoPackagePartitionReader( + var values: Seq[Any], + rs: ResultSet, + options: PartitionOptions) + extends PartitionReader[InternalRow] { + + def this(partitionOptions: PartitionOptions) = { + this( + Seq.empty, + GeoPackageConnectionManager.getTableCursor(partitionOptions), + partitionOptions) + } + + override def next(): Boolean = { + val hasNext = rs.next() + if (!hasNext) { + return false + } + + values = ValuesMapper.mapValues(adjustPartitionOptions, rs) + + hasNext + } + + private def adjustPartitionOptions: PartitionOptions = { + options.loadOptions.tableType match { + case FEATURES | METADATA => options + case TILES => + val tileRowMetadata = TileRowMetadata( + zoomLevel = rs.getInt("zoom_level"), + tileColumn = rs.getInt("tile_column"), + tileRow = rs.getInt("tile_row")) + + options.withTileRowMetadata(tileRowMetadata) + case UNKNOWN => options + } + + } + + override def get(): InternalRow = { + InternalRow.fromSeq(values) + } + + override def close(): Unit = { + rs.close() + } +} diff --git a/spark/spark-3.4/src/main/scala/org/apache/spark/sql/execution/datasources/geopackage/GeoPackagePartitionReaderFactory.scala b/spark/spark-3.4/src/main/scala/org/apache/spark/sql/execution/datasources/geopackage/GeoPackagePartitionReaderFactory.scala new file mode 100644 index 000000000..62ebea5eb --- /dev/null +++ b/spark/spark-3.4/src/main/scala/org/apache/spark/sql/execution/datasources/geopackage/GeoPackagePartitionReaderFactory.scala @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.spark.sql.execution.datasources.geopackage + +import org.apache.sedona.sql.datasources.geopackage.connection.GeoPackageConnectionManager +import org.apache.sedona.sql.datasources.geopackage.model.{PartitionOptions, TableType} +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.connector.read.{InputPartition, PartitionReader, PartitionReaderFactory} + +class GeoPackagePartitionReaderFactory(loadOptions: GeoPackageLoadOptions) + extends PartitionReaderFactory { + + override def createReader(partition: InputPartition): PartitionReader[InternalRow] = { + if (loadOptions.showMetadata) { + return new GeoPackagePartitionReader(PartitionOptions.fromLoadOptions(loadOptions)) + } + + loadOptions.tableType match { + case TableType.FEATURES => + new GeoPackagePartitionReader(PartitionOptions.fromLoadOptions(loadOptions)) + + case TableType.TILES => + val tileMetadata = + GeoPackageConnectionManager.findTilesMetadata(loadOptions.path, loadOptions.tableName) + + new GeoPackagePartitionReader( + PartitionOptions.fromLoadOptions(loadOptions).withTile(tileMetadata)) + } + } +} diff --git a/spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/geopackage/GeoPackageScan.scala b/spark/spark-3.4/src/main/scala/org/apache/spark/sql/execution/datasources/geopackage/GeoPackageScan.scala similarity index 92% copy from spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/geopackage/GeoPackageScan.scala copy to spark/spark-3.4/src/main/scala/org/apache/spark/sql/execution/datasources/geopackage/GeoPackageScan.scala index 289ad87e5..625c979ca 100644 --- a/spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/geopackage/GeoPackageScan.scala +++ b/spark/spark-3.4/src/main/scala/org/apache/spark/sql/execution/datasources/geopackage/GeoPackageScan.scala @@ -16,9 +16,8 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.sedona.sql.datasources.geopackage +package org.apache.spark.sql.execution.datasources.geopackage -import org.apache.sedona.sql.datasources.geopackage.model.TableType.TableType import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.expressions.Expression import org.apache.spark.sql.connector.read.PartitionReaderFactory diff --git a/spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/geopackage/GeoPackageScanBuilder.scala b/spark/spark-3.4/src/main/scala/org/apache/spark/sql/execution/datasources/geopackage/GeoPackageScanBuilder.scala similarity index 84% copy from spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/geopackage/GeoPackageScanBuilder.scala copy to spark/spark-3.4/src/main/scala/org/apache/spark/sql/execution/datasources/geopackage/GeoPackageScanBuilder.scala index ada2f91a6..9d878c47e 100644 --- a/spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/geopackage/GeoPackageScanBuilder.scala +++ b/spark/spark-3.4/src/main/scala/org/apache/spark/sql/execution/datasources/geopackage/GeoPackageScanBuilder.scala @@ -16,16 +16,13 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.sedona.sql.datasources.geopackage +package org.apache.spark.sql.execution.datasources.geopackage -import org.apache.sedona.sql.datasources.geopackage.connection.GeoPackageConnectionManager -import org.apache.sedona.sql.datasources.geopackage.model.TableType import org.apache.spark.sql.SparkSession import org.apache.spark.sql.connector.read.Scan import org.apache.spark.sql.execution.datasources.PartitioningAwareFileIndex import org.apache.spark.sql.execution.datasources.v2.FileScanBuilder import org.apache.spark.sql.types.StructType -import org.apache.spark.sql.util.CaseInsensitiveStringMap class GeoPackageScanBuilder( sparkSession: SparkSession, diff --git a/spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/geopackage/GeoPackageTable.scala b/spark/spark-3.4/src/main/scala/org/apache/spark/sql/execution/datasources/geopackage/GeoPackageTable.scala similarity index 88% copy from spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/geopackage/GeoPackageTable.scala copy to spark/spark-3.4/src/main/scala/org/apache/spark/sql/execution/datasources/geopackage/GeoPackageTable.scala index 8cb059b7b..808e8d2f6 100644 --- a/spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/geopackage/GeoPackageTable.scala +++ b/spark/spark-3.4/src/main/scala/org/apache/spark/sql/execution/datasources/geopackage/GeoPackageTable.scala @@ -16,17 +16,15 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.sedona.sql.datasources.geopackage +package org.apache.spark.sql.execution.datasources.geopackage import org.apache.hadoop.fs.FileStatus -import org.apache.sedona.sql.datasources.geopackage.connection.GeoPackageConnectionManager -import org.apache.sedona.sql.datasources.geopackage.model.TableType import org.apache.spark.sql.SparkSession import org.apache.spark.sql.connector.read.ScanBuilder import org.apache.spark.sql.connector.write.{LogicalWriteInfo, WriteBuilder} import org.apache.spark.sql.execution.datasources.FileFormat import org.apache.spark.sql.execution.datasources.v2.FileTable -import org.apache.spark.sql.types.{StringType, StructField, StructType} +import org.apache.spark.sql.types.StructType import org.apache.spark.sql.util.CaseInsensitiveStringMap case class GeoPackageTable( diff --git a/spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/geopackage/errors/GeopackageException.scala b/spark/spark-3.4/src/test/scala/org/apache/sedona/sql/GeoPackageReaderTest.scala similarity index 61% copy from spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/geopackage/errors/GeopackageException.scala copy to spark/spark-3.4/src/test/scala/org/apache/sedona/sql/GeoPackageReaderTest.scala index ae89baca7..20a4b1c08 100644 --- a/spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/geopackage/errors/GeopackageException.scala +++ b/spark/spark-3.4/src/test/scala/org/apache/sedona/sql/GeoPackageReaderTest.scala @@ -16,10 +16,23 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.sedona.sql.datasources.geopackage.errors +package org.apache.sedona.sql -class GeopackageException extends Exception { - def this(message: String) { - this() +import org.scalatest.matchers.should.Matchers +class GeoPackageReaderTest extends TestBaseScala with Matchers { + + val path: String = resourceFolder + "geopackage/example.gpkg" + + describe("Reading geopackage metadata") { + it("should read GeoPackage metadata") { + val df = sparkSession.read + .format("geopackage") + .option("showMetadata", "true") + .load(path) + + df.where("data_type = 'tiles'").show(false) + + df.count shouldEqual 34 + } } } diff --git a/spark/spark-3.5/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister b/spark/spark-3.5/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister index d2f1d0340..39b7d446c 100644 --- a/spark/spark-3.5/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister +++ b/spark/spark-3.5/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister @@ -1,3 +1,4 @@ org.apache.spark.sql.execution.datasources.parquet.GeoParquetFileFormat org.apache.spark.sql.execution.datasources.v2.geoparquet.metadata.GeoParquetMetadataDataSource org.apache.sedona.sql.datasources.shapefile.ShapefileDataSource +org.apache.sedona.sql.datasources.geopackage.GeoPackageDataSource diff --git a/spark/spark-3.5/src/main/scala/org/apache/spark/sql/execution/datasources/geopackage/GeoPackageDataSource.scala b/spark/spark-3.5/src/main/scala/org/apache/spark/sql/execution/datasources/geopackage/GeoPackageDataSource.scala new file mode 100644 index 000000000..08c52ecdb --- /dev/null +++ b/spark/spark-3.5/src/main/scala/org/apache/spark/sql/execution/datasources/geopackage/GeoPackageDataSource.scala @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.spark.sql.execution.datasources.geopackage + +import org.apache.sedona.sql.datasources.geopackage.connection.GeoPackageConnectionManager +import org.apache.sedona.sql.datasources.geopackage.model.TableType +import org.apache.sedona.sql.datasources.geopackage.model.TableType.TableType +import org.apache.spark.sql.connector.catalog.Table +import org.apache.spark.sql.execution.datasources.FileFormat +import org.apache.spark.sql.execution.datasources.v2.FileDataSourceV2 +import org.apache.spark.sql.sources.DataSourceRegister +import org.apache.spark.sql.util.CaseInsensitiveStringMap + +class GeoPackageDataSource extends FileDataSourceV2 with DataSourceRegister { + + override def fallbackFileFormat: Class[_ <: FileFormat] = { + null + } + + override protected def getTable(options: CaseInsensitiveStringMap): Table = { + val loadOptions = getLoadOptions(options) + + GeoPackageTable( + "", + sparkSession, + options, + Seq(loadOptions.path), + None, + fallbackFileFormat, + loadOptions) + } + + private def getLoadOptions(options: CaseInsensitiveStringMap): GeoPackageLoadOptions = { + val path = options.get("path") + if (path.isEmpty) { + throw new IllegalArgumentException("GeoPackage path is not specified") + } + + val showMetadata = options.getBoolean("showMetadata", false) + val maybeTableName = options.get("tableName") + + if (!showMetadata && maybeTableName == null) { + throw new IllegalArgumentException("Table name is not specified") + } + + val tableName = if (showMetadata) { + "gpkg_contents" + } else { + maybeTableName + } + + val fields = GeoPackageConnectionManager + .getSchema(path, tableName) + + GeoPackageLoadOptions( + path = path, + showMetadata = showMetadata, + tableName = tableName, + tableType = getTableType(showMetadata = showMetadata, path = path, tableName = tableName), + fields = fields) + } + + private def getTableType(showMetadata: Boolean, path: String, tableName: String): TableType = { + if (showMetadata) { + TableType.METADATA + } else { + GeoPackageConnectionManager.findFeatureMetadata(path, tableName) + } + } + + override def shortName(): String = "geopackage" +} diff --git a/spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/geopackage/model/TableType.scala b/spark/spark-3.5/src/main/scala/org/apache/spark/sql/execution/datasources/geopackage/GeoPackageLoadOptions.scala similarity index 68% rename from spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/geopackage/model/TableType.scala rename to spark/spark-3.5/src/main/scala/org/apache/spark/sql/execution/datasources/geopackage/GeoPackageLoadOptions.scala index 774419232..1c0090ae3 100644 --- a/spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/geopackage/model/TableType.scala +++ b/spark/spark-3.5/src/main/scala/org/apache/spark/sql/execution/datasources/geopackage/GeoPackageLoadOptions.scala @@ -16,9 +16,14 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.sedona.sql.datasources.geopackage.model +package org.apache.spark.sql.execution.datasources.geopackage -object TableType extends Enumeration { - type TableType = Value - val FEATURES, TILES, METADATA, UNKNOWN = Value -} +import org.apache.sedona.sql.datasources.geopackage.model.GeoPackageField +import org.apache.sedona.sql.datasources.geopackage.model.TableType.TableType + +case class GeoPackageLoadOptions( + path: String, + showMetadata: Boolean, + tableName: String, + tableType: TableType, + fields: Seq[GeoPackageField]) diff --git a/spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/geopackage/model/TileRowMetadata.scala b/spark/spark-3.5/src/main/scala/org/apache/spark/sql/execution/datasources/geopackage/GeoPackageMetadataReader.scala similarity index 85% rename from spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/geopackage/model/TileRowMetadata.scala rename to spark/spark-3.5/src/main/scala/org/apache/spark/sql/execution/datasources/geopackage/GeoPackageMetadataReader.scala index e3fcb4012..315fb58ae 100644 --- a/spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/geopackage/model/TileRowMetadata.scala +++ b/spark/spark-3.5/src/main/scala/org/apache/spark/sql/execution/datasources/geopackage/GeoPackageMetadataReader.scala @@ -16,6 +16,6 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.sedona.sql.datasources.geopackage.model +package org.apache.spark.sql.execution.datasources.geopackage -case class TileRowMetadata(tileColumn: Int, tileRow: Int, zoomLevel: Int) +class GeoPackageMetadataReader {} diff --git a/spark/spark-3.5/src/main/scala/org/apache/spark/sql/execution/datasources/geopackage/GeoPackagePartitionReader.scala b/spark/spark-3.5/src/main/scala/org/apache/spark/sql/execution/datasources/geopackage/GeoPackagePartitionReader.scala new file mode 100644 index 000000000..94262388b --- /dev/null +++ b/spark/spark-3.5/src/main/scala/org/apache/spark/sql/execution/datasources/geopackage/GeoPackagePartitionReader.scala @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.spark.sql.execution.datasources.geopackage + +import org.apache.sedona.sql.datasources.geopackage.connection.GeoPackageConnectionManager +import org.apache.sedona.sql.datasources.geopackage.model.TableType.{FEATURES, METADATA, TILES, UNKNOWN} +import org.apache.sedona.sql.datasources.geopackage.model.{PartitionOptions, TileRowMetadata} +import org.apache.sedona.sql.datasources.geopackage.transform.ValuesMapper +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.connector.read.PartitionReader + +import java.sql.ResultSet + +case class GeoPackagePartitionReader( + var values: Seq[Any], + rs: ResultSet, + options: PartitionOptions) + extends PartitionReader[InternalRow] { + + def this(partitionOptions: PartitionOptions) = { + this( + Seq.empty, + GeoPackageConnectionManager.getTableCursor(partitionOptions), + partitionOptions) + } + + override def next(): Boolean = { + val hasNext = rs.next() + if (!hasNext) { + return false + } + + values = ValuesMapper.mapValues(adjustPartitionOptions, rs) + + hasNext + } + + private def adjustPartitionOptions: PartitionOptions = { + options.loadOptions.tableType match { + case FEATURES | METADATA => options + case TILES => + val tileRowMetadata = TileRowMetadata( + zoomLevel = rs.getInt("zoom_level"), + tileColumn = rs.getInt("tile_column"), + tileRow = rs.getInt("tile_row")) + + options.withTileRowMetadata(tileRowMetadata) + case UNKNOWN => options + } + + } + + override def get(): InternalRow = { + InternalRow.fromSeq(values) + } + + override def close(): Unit = { + rs.close() + } +} diff --git a/spark/spark-3.5/src/main/scala/org/apache/spark/sql/execution/datasources/geopackage/GeoPackagePartitionReaderFactory.scala b/spark/spark-3.5/src/main/scala/org/apache/spark/sql/execution/datasources/geopackage/GeoPackagePartitionReaderFactory.scala new file mode 100644 index 000000000..62ebea5eb --- /dev/null +++ b/spark/spark-3.5/src/main/scala/org/apache/spark/sql/execution/datasources/geopackage/GeoPackagePartitionReaderFactory.scala @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.spark.sql.execution.datasources.geopackage + +import org.apache.sedona.sql.datasources.geopackage.connection.GeoPackageConnectionManager +import org.apache.sedona.sql.datasources.geopackage.model.{PartitionOptions, TableType} +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.connector.read.{InputPartition, PartitionReader, PartitionReaderFactory} + +class GeoPackagePartitionReaderFactory(loadOptions: GeoPackageLoadOptions) + extends PartitionReaderFactory { + + override def createReader(partition: InputPartition): PartitionReader[InternalRow] = { + if (loadOptions.showMetadata) { + return new GeoPackagePartitionReader(PartitionOptions.fromLoadOptions(loadOptions)) + } + + loadOptions.tableType match { + case TableType.FEATURES => + new GeoPackagePartitionReader(PartitionOptions.fromLoadOptions(loadOptions)) + + case TableType.TILES => + val tileMetadata = + GeoPackageConnectionManager.findTilesMetadata(loadOptions.path, loadOptions.tableName) + + new GeoPackagePartitionReader( + PartitionOptions.fromLoadOptions(loadOptions).withTile(tileMetadata)) + } + } +} diff --git a/spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/geopackage/GeoPackageScan.scala b/spark/spark-3.5/src/main/scala/org/apache/spark/sql/execution/datasources/geopackage/GeoPackageScan.scala similarity index 92% copy from spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/geopackage/GeoPackageScan.scala copy to spark/spark-3.5/src/main/scala/org/apache/spark/sql/execution/datasources/geopackage/GeoPackageScan.scala index 289ad87e5..625c979ca 100644 --- a/spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/geopackage/GeoPackageScan.scala +++ b/spark/spark-3.5/src/main/scala/org/apache/spark/sql/execution/datasources/geopackage/GeoPackageScan.scala @@ -16,9 +16,8 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.sedona.sql.datasources.geopackage +package org.apache.spark.sql.execution.datasources.geopackage -import org.apache.sedona.sql.datasources.geopackage.model.TableType.TableType import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.expressions.Expression import org.apache.spark.sql.connector.read.PartitionReaderFactory diff --git a/spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/geopackage/GeoPackageScanBuilder.scala b/spark/spark-3.5/src/main/scala/org/apache/spark/sql/execution/datasources/geopackage/GeoPackageScanBuilder.scala similarity index 84% copy from spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/geopackage/GeoPackageScanBuilder.scala copy to spark/spark-3.5/src/main/scala/org/apache/spark/sql/execution/datasources/geopackage/GeoPackageScanBuilder.scala index ada2f91a6..9d878c47e 100644 --- a/spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/geopackage/GeoPackageScanBuilder.scala +++ b/spark/spark-3.5/src/main/scala/org/apache/spark/sql/execution/datasources/geopackage/GeoPackageScanBuilder.scala @@ -16,16 +16,13 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.sedona.sql.datasources.geopackage +package org.apache.spark.sql.execution.datasources.geopackage -import org.apache.sedona.sql.datasources.geopackage.connection.GeoPackageConnectionManager -import org.apache.sedona.sql.datasources.geopackage.model.TableType import org.apache.spark.sql.SparkSession import org.apache.spark.sql.connector.read.Scan import org.apache.spark.sql.execution.datasources.PartitioningAwareFileIndex import org.apache.spark.sql.execution.datasources.v2.FileScanBuilder import org.apache.spark.sql.types.StructType -import org.apache.spark.sql.util.CaseInsensitiveStringMap class GeoPackageScanBuilder( sparkSession: SparkSession, diff --git a/spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/geopackage/GeoPackageTable.scala b/spark/spark-3.5/src/main/scala/org/apache/spark/sql/execution/datasources/geopackage/GeoPackageTable.scala similarity index 88% copy from spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/geopackage/GeoPackageTable.scala copy to spark/spark-3.5/src/main/scala/org/apache/spark/sql/execution/datasources/geopackage/GeoPackageTable.scala index 8cb059b7b..808e8d2f6 100644 --- a/spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/geopackage/GeoPackageTable.scala +++ b/spark/spark-3.5/src/main/scala/org/apache/spark/sql/execution/datasources/geopackage/GeoPackageTable.scala @@ -16,17 +16,15 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.sedona.sql.datasources.geopackage +package org.apache.spark.sql.execution.datasources.geopackage import org.apache.hadoop.fs.FileStatus -import org.apache.sedona.sql.datasources.geopackage.connection.GeoPackageConnectionManager -import org.apache.sedona.sql.datasources.geopackage.model.TableType import org.apache.spark.sql.SparkSession import org.apache.spark.sql.connector.read.ScanBuilder import org.apache.spark.sql.connector.write.{LogicalWriteInfo, WriteBuilder} import org.apache.spark.sql.execution.datasources.FileFormat import org.apache.spark.sql.execution.datasources.v2.FileTable -import org.apache.spark.sql.types.{StringType, StructField, StructType} +import org.apache.spark.sql.types.StructType import org.apache.spark.sql.util.CaseInsensitiveStringMap case class GeoPackageTable( diff --git a/spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/geopackage/errors/GeopackageException.scala b/spark/spark-3.5/src/test/scala/org/apache/sedona/sql/GeoPackageReaderTest.scala similarity index 61% rename from spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/geopackage/errors/GeopackageException.scala rename to spark/spark-3.5/src/test/scala/org/apache/sedona/sql/GeoPackageReaderTest.scala index ae89baca7..bb77bc1c8 100644 --- a/spark/spark-3.3/src/main/scala/org/apache/sedona/sql/datasources/geopackage/errors/GeopackageException.scala +++ b/spark/spark-3.5/src/test/scala/org/apache/sedona/sql/GeoPackageReaderTest.scala @@ -16,10 +16,24 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.sedona.sql.datasources.geopackage.errors +package org.apache.sedona.sql -class GeopackageException extends Exception { - def this(message: String) { - this() +import org.scalatest.matchers.should.Matchers + +class GeoPackageReaderTest extends TestBaseScala with Matchers { + + val path: String = resourceFolder + "geopackage/example.gpkg" + + describe("Reading geopackage metadata") { + it("should read GeoPackage metadata") { + val df = sparkSession.read + .format("geopackage") + .option("showMetadata", "true") + .load(path) + + df.where("data_type = 'tiles'").show(false) + + df.count shouldEqual 34 + } } }
