This is an automated email from the ASF dual-hosted git repository.
jiayu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/sedona.git
The following commit(s) were added to refs/heads/master by this push:
new 44afedd80d [GH-2166] More pathing fixes for Python package refactor
(#2167)
44afedd80d is described below
commit 44afedd80d1b02bd53c0620dbf07d09afb6bbc62
Author: James Willis <[email protected]>
AuthorDate: Tue Jul 29 11:19:07 2025 -0700
[GH-2166] More pathing fixes for Python package refactor (#2167)
* SedonaUtils fix
* maintain backwards compat from the restructuring in #1930
mostly these are just adding "redirects" to the new location.
the geoarrow I moved out of the util folder. unless we had never released
this at the time, this seemed to just be a mistake that failed to maintain
backwards compatability.
I skipped some things because I thought maybe theyre not supported APIS:
* everything in core
* sedona.register outside of what is available from the init file
I'm sure I missed some things here but hopefull this makes things a lot
better for existing users of sedona.
* PR Comments + remove SedonaMapUtils.py, its not a public interface.
* revert test style changes
* clean up imports
* remove print statements
* fix tests
* Fix raster UDT import order issue
---------
Co-authored-by: Kristin Cowalcijk <[email protected]>
---
.../{utils/geoarrow.py => geoarrow/__init__.py} | 5 +--
.../{utils/__init__.py => maps/SedonaKepler.py} | 4 +++
.../{utils/__init__.py => maps/SedonaPyDeck.py} | 3 ++
python/sedona/{utils => maps}/__init__.py | 8 +++++
python/sedona/{utils => raster}/__init__.py | 7 ++++
.../{utils/__init__.py => raster/awt_raster.py} | 4 +++
.../{utils/__init__.py => raster/data_buffer.py} | 4 +++
.../sedona/{utils/__init__.py => raster/meta.py} | 4 +++
.../{utils/__init__.py => raster/raster_serde.py} | 4 +++
.../{utils/geoarrow.py => raster/sample_model.py} | 21 ++++++-----
.../{utils/__init__.py => raster/sedona_raster.py} | 8 +++++
.../__init__.py => raster_utils/SedonaUtils.py} | 4 +++
python/sedona/{sql => raster_utils}/__init__.py | 0
.../{utils/geoarrow => register}/__init__.py | 7 ++--
python/sedona/spark/__init__.py | 3 ++
python/sedona/spark/sql/types.py | 4 ---
python/sedona/spark/utils/prep.py | 15 ++++++++
python/sedona/sql/__init__.py | 6 ++++
python/sedona/utils/__init__.py | 10 ++++++
python/tests/test_path_compatibility.py | 42 ++++++++++++++++++++--
python/tests/test_path_compatibility_all.py | 23 ++++++++++++
21 files changed, 166 insertions(+), 20 deletions(-)
diff --git a/python/sedona/utils/geoarrow.py
b/python/sedona/geoarrow/__init__.py
similarity index 87%
copy from python/sedona/utils/geoarrow.py
copy to python/sedona/geoarrow/__init__.py
index bbcb5aa608..9c9ce56140 100644
--- a/python/sedona/utils/geoarrow.py
+++ b/python/sedona/geoarrow/__init__.py
@@ -16,7 +16,8 @@
# under the License.
import warnings
-from sedona.spark.geoarrow import create_spatial_dataframe, dataframe_to_arrow
+from sedona.spark.geoarrow import dataframe_to_arrow, create_spatial_dataframe
+from sedona.spark import geoarrow
warnings.warn(
"The 'sedona.geoarrow' module is deprecated and will be removed in future
versions. Please use 'sedona.spark.geoarrow' instead.",
@@ -24,4 +25,4 @@ warnings.warn(
stacklevel=2,
)
-__all__ = ["create_spatial_dataframe", "dataframe_to_arrow"]
+__all__ = geoarrow.__all__
diff --git a/python/sedona/utils/__init__.py
b/python/sedona/maps/SedonaKepler.py
similarity index 90%
copy from python/sedona/utils/__init__.py
copy to python/sedona/maps/SedonaKepler.py
index 13a83393a9..c415c5a7f3 100644
--- a/python/sedona/utils/__init__.py
+++ b/python/sedona/maps/SedonaKepler.py
@@ -14,3 +14,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+
+from sedona.spark.maps.SedonaKepler import SedonaKepler
+
+__all__ = ["SedonaKepler"]
diff --git a/python/sedona/utils/__init__.py
b/python/sedona/maps/SedonaPyDeck.py
similarity index 90%
copy from python/sedona/utils/__init__.py
copy to python/sedona/maps/SedonaPyDeck.py
index 13a83393a9..850bf0ed88 100644
--- a/python/sedona/utils/__init__.py
+++ b/python/sedona/maps/SedonaPyDeck.py
@@ -14,3 +14,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+from sedona.spark.maps.SedonaPyDeck import SedonaPyDeck
+
+__all__ = ["SedonaPyDeck"]
diff --git a/python/sedona/utils/__init__.py b/python/sedona/maps/__init__.py
similarity index 79%
copy from python/sedona/utils/__init__.py
copy to python/sedona/maps/__init__.py
index 13a83393a9..0fdb2acac8 100644
--- a/python/sedona/utils/__init__.py
+++ b/python/sedona/maps/__init__.py
@@ -14,3 +14,11 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+
+import warnings
+
+warnings.warn(
+ "The 'sedona.maps' module is deprecated and will be removed in future
versions. Please use 'sedona.spark.maps' instead.",
+ DeprecationWarning,
+ stacklevel=2,
+)
diff --git a/python/sedona/utils/__init__.py b/python/sedona/raster/__init__.py
similarity index 78%
copy from python/sedona/utils/__init__.py
copy to python/sedona/raster/__init__.py
index 13a83393a9..a17f23effa 100644
--- a/python/sedona/utils/__init__.py
+++ b/python/sedona/raster/__init__.py
@@ -14,3 +14,10 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+import warnings
+
+warnings.warn(
+ "The 'sedona.raster' package structure has been reorganized. Please update
your imports to use 'sedona.spark.raster' prefix instead.",
+ DeprecationWarning,
+ stacklevel=2,
+)
diff --git a/python/sedona/utils/__init__.py
b/python/sedona/raster/awt_raster.py
similarity index 90%
copy from python/sedona/utils/__init__.py
copy to python/sedona/raster/awt_raster.py
index 13a83393a9..3b1aac9113 100644
--- a/python/sedona/utils/__init__.py
+++ b/python/sedona/raster/awt_raster.py
@@ -14,3 +14,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+
+from sedona.spark.raster.awt_raster import AWTRaster
+
+__all__ = ["AWTRaster"]
diff --git a/python/sedona/utils/__init__.py
b/python/sedona/raster/data_buffer.py
similarity index 90%
copy from python/sedona/utils/__init__.py
copy to python/sedona/raster/data_buffer.py
index 13a83393a9..26f8b8a243 100644
--- a/python/sedona/utils/__init__.py
+++ b/python/sedona/raster/data_buffer.py
@@ -14,3 +14,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+
+from sedona.spark.raster.data_buffer import DataBuffer
+
+__all__ = ["DataBuffer"]
diff --git a/python/sedona/utils/__init__.py b/python/sedona/raster/meta.py
similarity index 84%
copy from python/sedona/utils/__init__.py
copy to python/sedona/raster/meta.py
index 13a83393a9..36e9497826 100644
--- a/python/sedona/utils/__init__.py
+++ b/python/sedona/raster/meta.py
@@ -14,3 +14,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+
+from sedona.spark.raster.meta import PixelAnchor, AffineTransform,
SampleDimension
+
+__all__ = ["PixelAnchor", "AffineTransform", "SampleDimension"]
diff --git a/python/sedona/utils/__init__.py
b/python/sedona/raster/raster_serde.py
similarity index 87%
copy from python/sedona/utils/__init__.py
copy to python/sedona/raster/raster_serde.py
index 13a83393a9..5df780b048 100644
--- a/python/sedona/utils/__init__.py
+++ b/python/sedona/raster/raster_serde.py
@@ -14,3 +14,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+
+from sedona.spark.raster.raster_serde import RasterTypes, deserialize
+
+__all__ = ["RasterTypes", "deserialize"]
diff --git a/python/sedona/utils/geoarrow.py
b/python/sedona/raster/sample_model.py
similarity index 68%
rename from python/sedona/utils/geoarrow.py
rename to python/sedona/raster/sample_model.py
index bbcb5aa608..cf8a87d8d2 100644
--- a/python/sedona/utils/geoarrow.py
+++ b/python/sedona/raster/sample_model.py
@@ -15,13 +15,18 @@
# specific language governing permissions and limitations
# under the License.
-import warnings
-from sedona.spark.geoarrow import create_spatial_dataframe, dataframe_to_arrow
-
-warnings.warn(
- "The 'sedona.geoarrow' module is deprecated and will be removed in future
versions. Please use 'sedona.spark.geoarrow' instead.",
- DeprecationWarning,
- stacklevel=2,
+from sedona.spark.raster.sample_model import (
+ SampleModel,
+ ComponentSampleModel,
+ PixelInterleavedSampleModel,
+ SinglePixelPackedSampleModel,
+ MultiPixelPackedSampleModel,
)
-__all__ = ["create_spatial_dataframe", "dataframe_to_arrow"]
+__all__ = [
+ "SampleModel",
+ "ComponentSampleModel",
+ "PixelInterleavedSampleModel",
+ "SinglePixelPackedSampleModel",
+ "MultiPixelPackedSampleModel",
+]
diff --git a/python/sedona/utils/__init__.py
b/python/sedona/raster/sedona_raster.py
similarity index 81%
copy from python/sedona/utils/__init__.py
copy to python/sedona/raster/sedona_raster.py
index 13a83393a9..854d8fcac2 100644
--- a/python/sedona/utils/__init__.py
+++ b/python/sedona/raster/sedona_raster.py
@@ -14,3 +14,11 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+
+from sedona.spark.raster.sedona_raster import (
+ GDAL_VERSION,
+ SedonaRaster,
+ InDbSedonaRaster,
+)
+
+__all__ = ["GDAL_VERSION", "SedonaRaster", "InDbSedonaRaster"]
diff --git a/python/sedona/utils/__init__.py
b/python/sedona/raster_utils/SedonaUtils.py
similarity index 89%
copy from python/sedona/utils/__init__.py
copy to python/sedona/raster_utils/SedonaUtils.py
index 13a83393a9..af94c6b03b 100644
--- a/python/sedona/utils/__init__.py
+++ b/python/sedona/raster_utils/SedonaUtils.py
@@ -14,3 +14,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+
+from sedona.spark.raster_utils.SedonaUtils import SedonaUtils
+
+__all__ = ["SedonaUtils"]
diff --git a/python/sedona/sql/__init__.py
b/python/sedona/raster_utils/__init__.py
similarity index 100%
copy from python/sedona/sql/__init__.py
copy to python/sedona/raster_utils/__init__.py
diff --git a/python/sedona/utils/geoarrow/__init__.py
b/python/sedona/register/__init__.py
similarity index 75%
rename from python/sedona/utils/geoarrow/__init__.py
rename to python/sedona/register/__init__.py
index 07b2bb12f1..cc91f680fa 100644
--- a/python/sedona/utils/geoarrow/__init__.py
+++ b/python/sedona/register/__init__.py
@@ -15,14 +15,13 @@
# specific language governing permissions and limitations
# under the License.
+from sedona.spark.register.geo_registrator import SedonaRegistrator
import warnings
-from sedona.spark.geoarrow.geoarrow import create_spatial_dataframe,
dataframe_to_arrow
-
warnings.warn(
- "The 'sedona.geoarrow' module is deprecated and will be removed in future
versions. Please use 'sedona.spark.geoarrow' instead.",
+ "Importing from 'sedona.register' is deprecated. Please use
'sedona.spark.register.geo_registrator' instead.",
DeprecationWarning,
stacklevel=2,
)
-__all__ = ["create_spatial_dataframe", "dataframe_to_arrow"]
+__all__ = ["SedonaRegistrator"]
diff --git a/python/sedona/spark/__init__.py b/python/sedona/spark/__init__.py
index eeb7fadc50..5859bb315a 100644
--- a/python/sedona/spark/__init__.py
+++ b/python/sedona/spark/__init__.py
@@ -64,6 +64,9 @@ from sedona.spark.stats.weighting import (
add_distance_band_column,
add_binary_distance_band_column,
)
+from sedona.spark.raster.awt_raster import AWTRaster
+from sedona.spark.raster.data_buffer import DataBuffer
+from sedona.spark.raster.meta import SampleDimension
from sedona.spark.utils import KryoSerializer, SedonaKryoRegistrator
from sedona.spark.utils.adapter import Adapter
from sedona.spark.utils.spatial_rdd_parser import GeoData
diff --git a/python/sedona/spark/sql/types.py b/python/sedona/spark/sql/types.py
index 301c54f55d..63267a1d5c 100644
--- a/python/sedona/spark/sql/types.py
+++ b/python/sedona/spark/sql/types.py
@@ -113,7 +113,3 @@ class RasterType(UserDefinedType):
@classmethod
def scalaUDT(cls):
return "org.apache.spark.sql.sedona_sql.UDT.RasterUDT"
-
-
-if SedonaRaster is not None:
- SedonaRaster.__UDT__ = RasterType()
diff --git a/python/sedona/spark/utils/prep.py
b/python/sedona/spark/utils/prep.py
index 54ec6f298f..4469f6796f 100644
--- a/python/sedona/spark/utils/prep.py
+++ b/python/sedona/spark/utils/prep.py
@@ -44,6 +44,7 @@ def assign_all() -> bool:
assign_udt_shapely_objects(geoms=geoms)
assign_user_data_to_shapely_objects(geoms=geoms)
assign_udt_geography()
+ assign_udt_raster()
return True
@@ -64,3 +65,17 @@ def assign_udt_geography():
from sedona.spark.sql.types import GeographyType
Geography.__UDT__ = GeographyType()
+
+
+def assign_udt_raster():
+ from sedona.spark.sql.types import RasterType
+
+ try:
+ from sedona.spark.raster.sedona_raster import SedonaRaster
+
+ SedonaRaster.__UDT__ = RasterType()
+ except ImportError:
+ # Some dependencies for SedonaRaster to work is not available (for
+ # instance, rasterio), Raster support is an opt-in feature, so we
+ # ignore this error and skip registering UDT for it.
+ pass
diff --git a/python/sedona/sql/__init__.py b/python/sedona/sql/__init__.py
index 13a83393a9..2f41765817 100644
--- a/python/sedona/sql/__init__.py
+++ b/python/sedona/sql/__init__.py
@@ -14,3 +14,9 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+
+# These allow use to access the __all__
+from sedona.spark.sql import *
+from sedona.spark import sql
+
+__all__ = sql.__all__
diff --git a/python/sedona/utils/__init__.py b/python/sedona/utils/__init__.py
index 13a83393a9..4cef83cd70 100644
--- a/python/sedona/utils/__init__.py
+++ b/python/sedona/utils/__init__.py
@@ -14,3 +14,13 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+import warnings
+from sedona.spark.utils.serde import KryoSerializer, SedonaKryoRegistrator
+
+warnings.warn(
+ "The 'sedona.utils' package structure has been reorganized. Please update
your imports to use 'sedona.spark.utils' prefix instead.",
+ DeprecationWarning,
+ stacklevel=2,
+)
+
+__all__ = ["KryoSerializer", "SedonaKryoRegistrator"]
diff --git a/python/tests/test_path_compatibility.py
b/python/tests/test_path_compatibility.py
index f5c7048eaa..2ea74ff26e 100644
--- a/python/tests/test_path_compatibility.py
+++ b/python/tests/test_path_compatibility.py
@@ -27,12 +27,12 @@ from sedona.core.geom.envelope import Envelope
from sedona.core.geom.geography import Geography
from sedona.core.spatialOperator import JoinQuery
from sedona.core.spatialOperator import JoinQueryRaw, KNNQuery, RangeQuery
+
from sedona.sql import st_aggregates as sta
from sedona.sql import st_constructors as stc
from sedona.sql import st_functions as stf
from sedona.sql import st_predicates as stp
from sedona.sql.st_aggregates import ST_Union_Aggr
-from sedona.sql.st_constructors import ST_MakePoint
from sedona.sql.st_functions import ST_X
from sedona.sql.st_predicates import ST_Intersects
from sedona.sql.types import GeographyType, GeometryType, RasterType
@@ -45,9 +45,17 @@ from sedona.stats.weighting import (
add_binary_distance_band_column,
)
from sedona.utils.adapter import Adapter
-from sedona.utils.geoarrow import create_spatial_dataframe
from sedona.utils.spatial_rdd_parser import GeoData
from tests.test_base import TestBase
+from sedona.raster_utils.SedonaUtils import SedonaUtils
+from sedona.sql import ST_MakePoint, ST_Y, ST_Touches, ST_Envelope_Aggr
+from sedona.geoarrow import create_spatial_dataframe, dataframe_to_arrow
+from sedona.utils import KryoSerializer, SedonaKryoRegistrator
+from sedona.maps import SedonaKepler, SedonaPyDeck
+
+from sedona.raster.awt_raster import AWTRaster
+from sedona.raster.data_buffer import DataBuffer
+from sedona.raster.meta import SampleDimension
class TestPathCompatibility(TestBase):
@@ -119,3 +127,33 @@ class TestPathCompatibility(TestBase):
def test_geoarrow_import(self):
# Test create_spatial_dataframe import
assert create_spatial_dataframe is not None
+ assert dataframe_to_arrow is not None
+
+ def test_raster_utils_imports(self):
+ # Test raster utils imports
+ assert SedonaUtils is not None
+
+ def test_import_df_functions_from_sedona_sql(self):
+ # one from each module
+ assert ST_MakePoint is not None
+ assert ST_Y is not None
+ assert ST_Touches is not None
+ assert ST_Envelope_Aggr is not None
+
+ def test_geoarrow_imports(self):
+ assert create_spatial_dataframe is not None
+ assert dataframe_to_arrow is not None
+
+ def test_sedona_util_imports(self):
+ assert KryoSerializer is not None
+ assert SedonaKryoRegistrator is not None
+
+ def test_maps_imports(self):
+ # Test Map imports
+ assert SedonaKepler is not None
+ assert SedonaPyDeck is not None
+
+ def test_raster_imports(self):
+ assert AWTRaster is not None
+ assert DataBuffer is not None
+ assert SampleDimension is not None
diff --git a/python/tests/test_path_compatibility_all.py
b/python/tests/test_path_compatibility_all.py
index 19305cbda0..8cd8c0cd70 100644
--- a/python/tests/test_path_compatibility_all.py
+++ b/python/tests/test_path_compatibility_all.py
@@ -87,3 +87,26 @@ class TestPathCompatibilityAll(TestBase):
def test_geoarrow_import(self):
# Test create_spatial_dataframe import
assert create_spatial_dataframe is not None
+ assert dataframe_to_arrow is not None
+
+ def test_raster_utils_imports(self):
+ # Test raster utils imports
+ assert SedonaUtils is not None
+
+ def test_geoarrow_imports(self):
+ assert create_spatial_dataframe is not None
+ assert dataframe_to_arrow is not None
+
+ def test_sedona_util_imports(self):
+ assert KryoSerializer is not None
+ assert SedonaKryoRegistrator is not None
+
+ def test_maps_imports(self):
+ # Test Map imports
+ assert SedonaKepler is not None
+ assert SedonaPyDeck is not None
+
+ def test_raster_imports(self):
+ assert AWTRaster is not None
+ assert DataBuffer is not None
+ assert SampleDimension is not None
