This is an automated email from the ASF dual-hosted git repository.
jiayu pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/sedona-db.git
The following commit(s) were added to refs/heads/main by this push:
new 9f36d8b chore(dev/release): Add release verification script (#107)
9f36d8b is described below
commit 9f36d8b114f75560b1f7fc6b23116c118c822be7
Author: Dewey Dunnington <[email protected]>
AuthorDate: Thu Sep 18 13:29:15 2025 -0500
chore(dev/release): Add release verification script (#107)
Co-authored-by: Copilot <[email protected]>
---
.github/workflows/packaging.yml | 1 +
c/sedona-geoarrow-c/src/geoarrow_c.rs | 2 +-
c/sedona-proj/src/proj.rs | 8 +-
ci/scripts/build-source.sh | 15 ++
dev/release/README.md | 51 +++++
dev/release/verify-release-candidate.sh | 351 ++++++++++++++++++++++++++++++++
sedona-cli/Cargo.toml | 2 +-
7 files changed, 421 insertions(+), 9 deletions(-)
diff --git a/.github/workflows/packaging.yml b/.github/workflows/packaging.yml
index a1c88a4..c4c116e 100644
--- a/.github/workflows/packaging.yml
+++ b/.github/workflows/packaging.yml
@@ -42,6 +42,7 @@ jobs:
- uses: actions/checkout@v4
with:
fetch-depth: 0
+ submodules: 'true'
- uses: actions/setup-python@v5
with:
python-version: "3.x"
diff --git a/c/sedona-geoarrow-c/src/geoarrow_c.rs
b/c/sedona-geoarrow-c/src/geoarrow_c.rs
index e3c5d9f..0956172 100644
--- a/c/sedona-geoarrow-c/src/geoarrow_c.rs
+++ b/c/sedona-geoarrow-c/src/geoarrow_c.rs
@@ -268,7 +268,7 @@ impl Default for Visitor {
impl From<&str> for GeoArrowStringView {
fn from(value: &str) -> Self {
Self {
- data: value.as_ptr() as *const i8,
+ data: value.as_ptr() as *const _,
size_bytes: value.len() as i64,
}
}
diff --git a/c/sedona-proj/src/proj.rs b/c/sedona-proj/src/proj.rs
index 31dfbb5..c597ec0 100644
--- a/c/sedona-proj/src/proj.rs
+++ b/c/sedona-proj/src/proj.rs
@@ -552,13 +552,7 @@ impl ProjApi {
));
inner.proj_create_crs_to_crs_from_pj = Some(std::mem::transmute(
proj_create_crs_to_crs_from_pj
- as unsafe extern "C" fn(
- *mut _,
- *const _,
- *const _,
- *mut _,
- *const *const i8,
- ) -> _,
+ as unsafe extern "C" fn(*mut _, *const _, *const _, *mut
_, *const _) -> _,
));
inner.proj_create = Some(std::mem::transmute(
proj_create as unsafe extern "C" fn(*mut _, *const _) -> _,
diff --git a/ci/scripts/build-source.sh b/ci/scripts/build-source.sh
index c2e7e05..31c7192 100755
--- a/ci/scripts/build-source.sh
+++ b/ci/scripts/build-source.sh
@@ -40,6 +40,21 @@ main() {
rm -rf "${base_name}/"
git archive "${revision}" --prefix "${base_name}/" | tar xf -
+ # Resolve all submodules for sedona-s2geography. In the future we probably
+ # want to improve the packaging of sedona-s2geography such that we don't
need
+ # this step:
+ # https://github.com/apache/sedona-db/issues/109
+ while read SUBMODULE; do
+ SUBMODULE_REV=$(echo "${SUBMODULE}" | awk '{print $1}')
+ SUBMODULE_PATH=$(echo "${SUBMODULE}" | awk '{print $2}')
+ # Check if submodule path starts with "submodules/"
+ if [[ "${SUBMODULE_PATH}" == submodules/* ]]; then
+ echo "Skipping testing submodule ${SUBMODULE}"
+ else
+ git -C "${SUBMODULE_PATH}" archive
--prefix="${base_name}/${SUBMODULE_PATH}/" "${SUBMODULE_REV}" | tar xf - -C
"${source_top_dir}"
+ fi
+ done < <(git submodule status)
+
# Create new tarball
tar czf "${tar_ball}" "${base_name}/"
rm -rf "${base_name}/"
diff --git a/dev/release/README.md b/dev/release/README.md
index f6eda14..f468146 100644
--- a/dev/release/README.md
+++ b/dev/release/README.md
@@ -19,6 +19,50 @@
# Releasing SedonaDB
+## Verifying a release candidate
+
+Release candidates are verified using the script `verify-release-candidate.sh
<version> <rc_num>`.
+For example, to verify SedonaDB 0.1.0 RC0, run:
+
+```shell
+# git clone https://github.com/apache/sedona-db.git && cd sedona-db
+# or
+# cd existing/sedona-db && git fetch upstream && git switch main && git pull
upstream main
+dev/release/verify-release-candidate.sh 0.1.0 0
+```
+
+Release verification requires a recent Rust toolchain. This toolchain can be
installed
+by following instructions from <https://rustup.rs/>.
+
+MacOS users can use [Homebrew](https://brew.sh) to install the required
dependencies.
+
+```shell
+brew install geos proj openssl abseil
+```
+
+Linux users (e.g., `docker run --rm -it condaforge/mambaforge`) can use
`conda` to
+install the required dependencies:
+
+```shell
+conda create -y --name verify-sedona-db
+conda activate verify-sedona-db
+conda install -y curl gnupg geos proj openssl libabseil cmake pkg-config
+export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$CONDA_PREFIX/lib"
+```
+
+Currently system libclang is required to generate C bindings at build time:
+
+```shell
+apt-get update && apt-get install -y libclang-dev
+```
+
+When verifying via Docker or on a smaller machine it may be necessary to limit
the
+number of parallel jobs to avoid running out of memory:
+
+```shell
+export CARGO_BUILD_JOBS=4
+```
+
## Creating a release
Create a release branch on the corresponding remote pointing to the official
Apache
@@ -43,6 +87,13 @@ are considered a "packaging" step (i.e., the artifacts
aren't uploaded to the
release or voted on), although those CI jobs are important to ensuring
the release is ready for a vote.
+Before creating a tag, download the tarball from the latest packaging run and
+check it locally:
+
+```shell
+dev/release/verify-release-candidate.sh path/to/tarball.tar.gz
+```
+
When the state of the `branch-x.x.x` branch is clean and checks are complete,
the release candidate tag can be created:
diff --git a/dev/release/verify-release-candidate.sh
b/dev/release/verify-release-candidate.sh
new file mode 100755
index 0000000..003f737
--- /dev/null
+++ b/dev/release/verify-release-candidate.sh
@@ -0,0 +1,351 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -e
+set -o pipefail
+
+if [ ${VERBOSE:-0} -gt 0 ]; then
+ set -x
+fi
+
+# Check that required dependencies are installed
+check_dependencies() {
+ local missing_deps=0
+
+ local required_deps=("curl" "git" "gpg" "cc" "cargo" "cmake")
+ for dep in "${required_deps[@]}"; do
+ if ! command -v $dep &> /dev/null; then
+ echo "Error: $dep is not installed or not in PATH"
+ missing_deps=1
+ fi
+ done
+
+ # pkg-config doesn't work with the above check
+ if ! pkg-config --version &> /dev/null; then
+ echo "Error: pkg-config is not installed or not in PATH"
+ missing_deps=1
+ fi
+
+ # Check for either shasum or sha256sum/sha512sum
+ local has_sha_tools=0
+ if command -v shasum &> /dev/null; then
+ has_sha_tools=1
+ elif command -v sha256sum &> /dev/null && command -v sha512sum &> /dev/null;
then
+ has_sha_tools=1
+ else
+ echo "Error: Neither shasum nor sha256sum/sha512sum are installed or in
PATH"
+ missing_deps=1
+ fi
+
+ if [ $missing_deps -ne 0 ]; then
+ echo "Please install missing dependencies and try again"
+ exit 1
+ fi
+}
+
+
+# Check that required native dependencies are installed. For the purposes of
this
+# script we require geos, proj, absl_base, and openssl via pkg-config, even
though
+# technically absl_base and openssl are resolved via CMake for
sedona-s2geography.
+check_pkg_config_dependencies() {
+ local missing_deps=0
+ local required_deps=(geos proj openssl absl_base)
+ for dep in "${required_deps[@]}"; do
+ if ! pkg-config --modversion $dep &> /dev/null; then
+ echo "Error: $dep is not installed or not in PKG_CONFIG_PATH"
+ missing_deps=1
+ fi
+ done
+
+ local absl_version=$(pkg-config --modversion absl_base)
+ # Check if Abseil version is sufficient (need at least version 20230802)
+ if [ "$absl_version" -lt "20230802" ]; then
+ echo "Error: Abseil version $absl_version is too old, need at least
20230802"
+ echo "On Linux, this typically requires verification within a conda
environment"
+ echo "or by installing Abseil via vcpkg and ensuring PKG_CONFIG_PATH and"
+ echo "CMAKE_TOOLCHAIN_FILE are updated appropriately."
+ missing_deps=1
+ fi
+
+ if [ $missing_deps -ne 0 ]; then
+ echo "Please install or update missing dependencies and try again"
+ echo "Using Homebrew: brew install geos proj openssl abseil"
+ echo "Using conda: conda install geos proj openssl libabseil"
+ exit 1
+ fi
+}
+
+case $# in
+ 0) VERSION="HEAD"
+ SOURCE_KIND="local"
+ ;;
+ 1) VERSION="TARBALL"
+ SOURCE_KIND="local_tarball"
+ LOCAL_TARBALL="$(realpath $1)"
+ ;;
+ 2) VERSION="$1"
+ RC_NUMBER="$2"
+ SOURCE_KIND="tarball"
+ ;;
+ *) echo "Usage:"
+ echo " Verify release candidate:"
+ echo " $0 X.Y.Z RC_NUMBER"
+ echo ""
+ echo " Run the source verification tasks on this sedona-db checkout:"
+ echo " $0"
+ exit 1
+ ;;
+esac
+
+# Check dependencies early
+check_dependencies
+check_pkg_config_dependencies
+
+# Note that these point to the current verify-release-candidate.sh directories
+# which is different from the SEDONADB_SOURCE_DIR set in
ensure_source_directory()
+SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)"
+SEDONADB_DIR="$(cd "${SOURCE_DIR}/../.." && pwd)"
+
+show_header() {
+ if [ -z "$GITHUB_ACTIONS" ]; then
+ echo ""
+ printf '=%.0s' $(seq ${#1}); printf '\n'
+ echo "${1}"
+ printf '=%.0s' $(seq ${#1}); printf '\n'
+ else
+ echo "::group::${1}"; printf '\n'
+ fi
+}
+
+show_info() {
+ echo "└ ${1}"
+}
+
+SEDONADB_DIST_URL='https://dist.apache.org/repos/dist/dev/sedona'
+
+download_dist_file() {
+ curl \
+ --silent \
+ --show-error \
+ --fail \
+ --location \
+ --remote-name $SEDONADB_DIST_URL/$1
+}
+
+download_rc_file() {
+ download_dist_file apache-sedona-db-${VERSION}-rc${RC_NUMBER}/$1
+}
+
+import_gpg_keys() {
+ if [ "${GPGKEYS_ALREADY_IMPORTED:-0}" -gt 0 ]; then
+ return 0
+ fi
+ download_dist_file KEYS
+
+ if [ "${SEDONADB_ACCEPT_IMPORT_GPG_KEYS_ERROR:-0}" -gt 0 ]; then
+ gpg --import KEYS || true
+ else
+ gpg --import KEYS
+ fi
+
+ GPGKEYS_ALREADY_IMPORTED=1
+}
+
+if type shasum >/dev/null 2>&1; then
+ sha512_verify="shasum -a 512 -c"
+else
+ sha512_verify="sha512sum -c"
+fi
+
+fetch_archive() {
+ import_gpg_keys
+
+ local dist_name=$1
+ download_rc_file ${dist_name}.tar.gz
+ download_rc_file ${dist_name}.tar.gz.asc
+ download_rc_file ${dist_name}.tar.gz.sha512
+ gpg --verify ${dist_name}.tar.gz.asc ${dist_name}.tar.gz
+ ${sha512_verify} ${dist_name}.tar.gz.sha512
+}
+
+verify_dir_artifact_signatures() {
+ import_gpg_keys
+
+ # verify the signature and the checksums of each artifact
+ find $1 -name '*.asc' | while read sigfile; do
+ artifact=${sigfile/.asc/}
+ gpg --verify $sigfile $artifact || exit 1
+
+ # go into the directory because the checksum files contain only the
+ # basename of the artifact
+ pushd $(dirname $artifact)
+ base_artifact=$(basename $artifact)
+ if [ -f $base_artifact.sha512 ]; then
+ ${sha512_verify} $base_artifact.sha512 || exit 1
+ fi
+ popd
+ done
+}
+
+setup_tempdir() {
+ cleanup() {
+ if [ "${TEST_SUCCESS}" = "yes" ]; then
+ rm -fr "${SEDONADB_TMPDIR}"
+ else
+ echo "Failed to verify release candidate. See ${SEDONADB_TMPDIR} for
details."
+ fi
+ }
+
+ show_header "Creating temporary directory"
+
+ if [ -z "${SEDONADB_TMPDIR}" ]; then
+ # clean up automatically if SEDONADB_TMPDIR is not defined
+ SEDONADB_TMPDIR=$(mktemp -d -t "sedonadb-${VERSION}.XXXXXX")
+ trap cleanup EXIT
+ else
+ # don't clean up automatically
+ mkdir -p "${SEDONADB_TMPDIR}"
+ fi
+
+ echo "Working in sandbox ${SEDONADB_TMPDIR}"
+}
+
+test_rust() {
+ show_header "Build and test Rust libraries"
+
+ pushd "${SEDONADB_SOURCE_DIR}"
+ cargo test --workspace --exclude sedona-s2geography
+ popd
+}
+
+activate_or_create_venv() {
+ if [ ! -z "${SEDONADB_PYTHON_VENV}" ]; then
+ show_info "Activating virtual environment at ${SEDONADB_PYTHON_VENV}"
+ source "${SEDONADB_PYTHON_VENV}/bin/activate"
+ else
+ # Try python3 first, then try regular python (e.g., already in a venv)
+ if [ -z "${PYTHON_BIN}" ] && python3 --version >/dev/null; then
+ PYTHON_BIN=python3
+ elif [ -z "${PYTHON_BIN}" ]; then
+ PYTHON_BIN=python
+ fi
+
+ show_info "Creating temporary virtual environment using ${PYTHON_BIN}..."
+ "${PYTHON_BIN}" -m venv "${SEDONADB_TMPDIR}/venv"
+ source "${SEDONADB_TMPDIR}/venv/bin/activate"
+ python -m pip install --upgrade pip
+ fi
+}
+
+test_python() {
+ show_header "Build and test Python package"
+ activate_or_create_venv
+
+ pushd "${SEDONADB_SOURCE_DIR}/python"
+
+ show_info "Installing Python package"
+ rm -rf "${SEDONADB_TMPDIR}/python"
+ pip install "sedonadb/[test]" -v
+
+ show_info "Testing Python package"
+ python -m pytest -vv
+
+ popd
+}
+
+ensure_source_directory() {
+ show_header "Ensuring source directory"
+
+ dist_name="apache-sedona-db-${VERSION}"
+
+ if [ "${SOURCE_KIND}" = "local" ]; then
+ # Local repository
+ if [ -z "$SEDONADB_SOURCE_DIR" ]; then
+ export SEDONADB_SOURCE_DIR="${SEDONADB_DIR}"
+ fi
+ echo "Verifying local sedona-db checkout at ${SEDONADB_SOURCE_DIR}"
+ elif [ "${SOURCE_KIND}" = "local_tarball" ]; then
+ # Local tarball
+ pushd $SEDONADB_TMPDIR
+ tar xf "$LOCAL_TARBALL"
+ dist_name=$(ls)
+ export SEDONADB_SOURCE_DIR="${SEDONADB_TMPDIR}/${dist_name}"
+
+ # Ensure submodules are where tests expect them to be
+ pushd "$SEDONADB_SOURCE_DIR/submodules"
+ git clone https://github.com/apache/sedona-testing.git
+ git clone https://github.com/geoarrow/geoarrow-data.git
+ popd
+
+ popd
+
+ echo "Verifying local tarball at $LOCAL_TARBALL"
+ else
+ # Release tarball
+ echo "Verifying official SedonaDB release candidate
${VERSION}-rc${RC_NUMBER}"
+ export SEDONADB_SOURCE_DIR="${SEDONADB_TMPDIR}/${dist_name}"
+ if [ ! -d "${SEDONADB_SOURCE_DIR}" ]; then
+ pushd $SEDONADB_TMPDIR
+ fetch_archive ${dist_name}
+ tar xf ${dist_name}.tar.gz
+
+ # Ensure submodules are where tests expect them to be
+ pushd "${SEDONADB_SOURCE_DIR}/submodules"
+ git clone https://github.com/apache/sedona-testing.git
+ git clone https://github.com/geoarrow/geoarrow-data.git
+ popd
+
+ popd
+ fi
+ fi
+}
+
+test_source_distribution() {
+ pushd $SEDONADB_SOURCE_DIR
+
+ if [ ${TEST_RUST} -gt 0 ]; then
+ test_rust
+ fi
+
+ if [ ${TEST_PYTHON} -gt 0 ]; then
+ test_python
+ fi
+
+ popd
+}
+
+# By default test all functionalities.
+# To deactivate one test, deactivate the test and all of its dependents
+# To explicitly select one test, set TEST_DEFAULT=0 TEST_X=1
+: ${TEST_DEFAULT:=1}
+
+: ${TEST_SOURCE:=${TEST_DEFAULT}}
+: ${TEST_RUST:=${TEST_SOURCE}}
+: ${TEST_PYTHON:=${TEST_SOURCE}}
+
+TEST_SUCCESS=no
+
+setup_tempdir
+ensure_source_directory
+test_source_distribution
+
+TEST_SUCCESS=yes
+
+echo "Release candidate ${VERSION}-RC${RC_NUMBER} looks good!"
+exit 0
diff --git a/sedona-cli/Cargo.toml b/sedona-cli/Cargo.toml
index 3a48b53..08ac0bb 100644
--- a/sedona-cli/Cargo.toml
+++ b/sedona-cli/Cargo.toml
@@ -60,6 +60,6 @@ mimalloc = { workspace = true, optional = true }
libmimalloc-sys = { workspace = true, optional = true }
regex = { workspace = true }
rustyline = "15.0"
-sedona = { path = "../rust/sedona", features = ["aws", "gcp", "http", "proj",
"s2geography"] }
+sedona = { path = "../rust/sedona", features = ["aws", "gcp", "http", "proj"] }
sedona-tg = { path = "../c/sedona-tg" }
tokio = { workspace = true, features = ["macros", "rt", "rt-multi-thread",
"sync", "parking_lot", "signal"] }
