commit: 69b7e2d62be6532978740e3e18b72b8fa8d0788d Author: Fabian Groffen <grobian <AT> gentoo <DOT> org> AuthorDate: Sun Sep 3 07:31:57 2023 +0000 Commit: Fabian Groffen <grobian <AT> gentoo <DOT> org> CommitDate: Sun Sep 3 07:31:57 2023 +0000 URL: https://gitweb.gentoo.org/repo/proj/prefix.git/commit/?id=69b7e2d6
scripts: make bootstrap snapshot available on rsync slaves In bug #913415 a bunch of confusing and wrong things came out, and in an attempt to fix this, the historical "prefix-overlay" fetched from dev was replaced with ordinary "portage" from rsync snapshots dir. However, this relies on the prefix mirror to provide the file, which can easily be overridden by the user. So retrieve the snapshot in use and retain it on the rsync slaves to make this in the very least less confusing for those following the locations, and ensure it works for people with custom mirror setups. Bug: https://bugs.gentoo.org/913415 Signed-off-by: Fabian Groffen <grobian <AT> gentoo.org> scripts/bootstrap-prefix.sh | 3 +++ scripts/rsync-generation/mksnapshot.sh | 36 ++++++++++++++++++++++------------ 2 files changed, 27 insertions(+), 12 deletions(-) diff --git a/scripts/bootstrap-prefix.sh b/scripts/bootstrap-prefix.sh index 142ef8213e..735e896dd0 100755 --- a/scripts/bootstrap-prefix.sh +++ b/scripts/bootstrap-prefix.sh @@ -562,6 +562,9 @@ do_tree() { } bootstrap_tree() { + # retain this comment and the line below to + # keep this snapshot around in the snapshots + # MKSNAPSHOT-ANCHOR -- directory of rsync slaves local PV="20230709" # RAP uses the latest gentoo main repo snapshot to bootstrap. diff --git a/scripts/rsync-generation/mksnapshot.sh b/scripts/rsync-generation/mksnapshot.sh index 5c13154cf2..d53d135693 100755 --- a/scripts/rsync-generation/mksnapshot.sh +++ b/scripts/rsync-generation/mksnapshot.sh @@ -4,39 +4,49 @@ SCRIPTLOC="$(readlink -f "${BASH_SOURCE[0]%/*}")" # for gpg's keys export HOME="${SCRIPTLOC}/misc" -cd /export/scratch/home/gentoo/snapshots +cd /export/gentoo-rsync/rsync0-prefix-tree/snapshots || exit 1 TODAY=$(date +%s) YESTERDAY=$((TODAY - 86400)) -RSYNCTREE=${PWD%/*}/prefix-rsync-tree +RSYNCTREE=${PWD%/*} SNAME=${PWD}/portage-$(date +%Y%m%d -d @${YESTERDAY}).tar -TMPDIR=${PWD%/*}/tmp-prefix-snapshot +TMPDIR=${PWD}/tmp-prefix-snapshot # clean up find . -maxdepth 2 -daystart -ctime +4 -type f | xargs --no-run-if-empty rm +# pull in active snapshot +BOOTSTRAP_SNAPSHOT=$( \ + grep -A1 MKSNAPSHOT-ANCHOR "${RSYNCTREE}"/scripts/bootstrap-prefix.sh | \ + sed -n 's/^.*PV="\([0-9]\+\)"\s*$/portage-\1.tar.bz2/p' \ +) +curl -s -L "https://distfiles.prefix.bitzolder.nl/prefix/distfiles/${BOOTSTRAP_SNAPSHOT}"; > "${BOOTSTRAP_SNAPSHOT}" + rm -Rf ${TMPDIR} mkdir -p ${TMPDIR} # quickly take a snapshot, such that we get a consistent image pushd ${RSYNCTREE} > /dev/null -tar -cf ${SNAME} * +tar -cf ${SNAME} --exclude=snapshots * || exit 1 popd > /dev/null # now revamp it such that it's in a directory "portage" +rm -Rf ${TMPDIR} +mkdir -p ${TMPDIR} pushd ${TMPDIR} > /dev/null mkdir portage tar -xf ${SNAME} -C portage/ -tar -cf ${SNAME} portage/ +tar --numeric-owner --format=posix --hard-dereference -cf ${SNAME} portage/ popd > /dev/null rm -Rf ${TMPDIR} -# be nice, and this machine only has one CPU (so can't parallelise) -nice -n19 bzip2 -c -9 ${SNAME} > ${SNAME}.bz2 -nice -n19 xz -c -9 ${SNAME} > ${SNAME}.xz -nice -n19 gzip -c -9 ${SNAME} > ${SNAME}.gz +# be nice +nice -n19 bzip2 -c -9 ${SNAME} > ${SNAME}.bz2 & +nice -n19 xz -c -9 ${SNAME} > ${SNAME}.xz & +nice -n19 gzip -c -9 ${SNAME} > ${SNAME}.gz & +wait # generate accompanying meta files md5sum ${SNAME##*/} > ${SNAME}.xz.umd5sum @@ -46,9 +56,11 @@ md5sum ${SNAME##*/}.bz2 > ${SNAME}.bz2.md5sum md5sum ${SNAME##*/} > ${SNAME}.gz.umd5sum md5sum ${SNAME##*/}.bz2 > ${SNAME}.gz.md5sum # gpg is really stupid, or I am too stupid to find the right option -gpg --batch --no-tty --passphrase-fd 0 --default-key C6317B3C --detach-sign --armor -o ${SNAME}.xz.gpgsig ${SNAME}.xz < ${SCRIPTLOC}/autosigner.pwd -gpg --batch --no-tty --passphrase-fd 0 --default-key C6317B3C --detach-sign --armor -o ${SNAME}.bz2.gpgsig ${SNAME}.bz2 < ${SCRIPTLOC}/autosigner.pwd -gpg --batch --no-tty --passphrase-fd 0 --default-key C6317B3C --detach-sign --armor -o ${SNAME}.gz.gpgsig ${SNAME}.gz < ${SCRIPTLOC}/autosigner.pwd +gpgopts="--quiet --batch --no-tty --passphrase-fd 0 --pinentry-mode loopback" +gpgopts+=" --default-key C6317B3C --detach-sign --armor" +gpg ${gpgopts} -o ${SNAME}.xz.gpgsig ${SNAME}.xz < ${SCRIPTLOC}/autosigner.pwd +gpg ${gpgopts} -o ${SNAME}.bz2.gpgsig ${SNAME}.bz2 < ${SCRIPTLOC}/autosigner.pwd +gpg ${gpgopts} -o ${SNAME}.gz.gpgsig ${SNAME}.gz < ${SCRIPTLOC}/autosigner.pwd # we no longer need the tar rm ${SNAME}
