This is an automated email from the ASF dual-hosted git repository.
jiayu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/sedona.git
The following commit(s) were added to refs/heads/master by this push:
new 6a8280836b [SEDONA-712] add toSpatialRdd method for RDD[InternalRow]
(#1816)
6a8280836b is described below
commit 6a8280836b2e1a493a0cbff8b22a05989866cae3
Author: James Willis <[email protected]>
AuthorDate: Fri Feb 14 13:19:11 2025 -0800
[SEDONA-712] add toSpatialRdd method for RDD[InternalRow] (#1816)
---
.../sedona_sql/adapters/StructuredAdapter.scala | 23 +++++++++++++++++-----
1 file changed, 18 insertions(+), 5 deletions(-)
diff --git
a/spark/common/src/main/scala/org/apache/spark/sql/sedona_sql/adapters/StructuredAdapter.scala
b/spark/common/src/main/scala/org/apache/spark/sql/sedona_sql/adapters/StructuredAdapter.scala
index 70cef8d783..8a8e7463de 100644
---
a/spark/common/src/main/scala/org/apache/spark/sql/sedona_sql/adapters/StructuredAdapter.scala
+++
b/spark/common/src/main/scala/org/apache/spark/sql/sedona_sql/adapters/StructuredAdapter.scala
@@ -84,17 +84,30 @@ object StructuredAdapter {
}
/**
- * Convert DataFrame to SpatialRDD. It puts InternalRow as user data of
Geometry. It allows only
- * one geometry column.
+ * Convert DataFrame to SpatialRDD. It puts InternalRow as user data of
Geometry.
*
* @param dataFrame
* @param geometryFieldName
*/
- def toSpatialRdd(dataFrame: DataFrame, geometryFieldName: String):
SpatialRDD[Geometry] = {
+ def toSpatialRdd(dataFrame: DataFrame, geometryFieldName: String):
SpatialRDD[Geometry] =
+ toSpatialRdd(dataFrame.queryExecution.toRdd, dataFrame.schema,
geometryFieldName)
+
+ /**
+ * Convert RDD[InternalRow] to SpatialRDD. It puts InternalRow as user data
of Geometry.
+ *
+ * @param rdd
+ * @param schema
+ * @param geometryFieldName
+ * @return
+ */
+ def toSpatialRdd(
+ rdd: RDD[InternalRow],
+ schema: StructType,
+ geometryFieldName: String): SpatialRDD[Geometry] = {
val spatialRDD = new SpatialRDD[Geometry]
- spatialRDD.schema = dataFrame.schema
+ spatialRDD.schema = schema
val ordinal = spatialRDD.schema.fieldIndex(geometryFieldName)
- spatialRDD.rawSpatialRDD = dataFrame.queryExecution.toRdd
+ spatialRDD.rawSpatialRDD = rdd
.map(row => {
val geom = GeometrySerializer.deserialize(row.getBinary(ordinal))
geom.setUserData(row.copy())
