Package: debootstrap X-Debbugs-Cc: m...@debian.org Version: 1.0.128+nmu2 Severity: normal Tags: patch
Dear Maintainer, currently debootstrap fails to work with APT repositories that have arch:all split out exclusively into a separate Packages file, instead of having arch:all merged into every arch:any Packages file in archive metadata. APT has already been supporting the new scheme for multiple releases, so it would be nice if debootstrap supported it too (so far, it is the only tool I found that doesn't do that). This would enable us in PureOS to use the new APT repository style, as well as potentially other derivatives and Debian itself in future as well. The APT repository format (for both styles and the current "compatibility mode" that the Debian archive runs on) is described here: https://wiki.debian.org/DebianRepository/Format#Architectures I attached a patch which implements support for this feature in debootstrap. So far it has been working find for bootstrapping both Debian (with the "compatibility" archive metadata mode) as well as a PureOS scratch repository which uses the newer-style APT repository mode. You can also review this patch as MR @ Salsa: https://salsa.debian.org/installer-team/debootstrap/-/merge_requests/92 Thank you for considering! I've set this bug as "normal", since I do consider bringing debootstrap on-par with what features APT supports for a while as important, but feel free to reduce it to "whishlist" priority if you think that is more appropriate for a feature request like this. With kind regards, Matthias Klumpp -- I welcome VSRE emails. See http://vsre.info/
From 5dd4cef1a5a03dc180eb2ec1d2bfc97023f74d2b Mon Sep 17 00:00:00 2001 From: Matthias Klumpp <m...@debian.org> Date: Sun, 23 Apr 2023 22:30:02 +0200 Subject: [PATCH] Implement support for repos with modern-style arch:all support This implements support for modern APT repositories which have arch:any and arch:all packages in separate Packages files, as outlined in the Debian repository documentation: https://wiki.debian.org/DebianRepository/Format#Architectures --- functions | 274 +++++++++++++++++++++++++++++++++--------------------- 1 file changed, 168 insertions(+), 106 deletions(-) diff --git a/functions b/functions index 0ff5379..8bffbc6 100644 --- a/functions +++ b/functions @@ -557,6 +557,24 @@ extract_release_components () { fi } +repo_supports_arch_all () { + local a no_arch_all_support + local reldest="$1"; shift + TMPARCHS="$(sed -n 's/Architectures: *//p' "$reldest")" + ARCH_ALL_SUPPORTED=0 + for a in $TMPARCHS ; do + if [ "$a" = "all" ]; then + ARCH_ALL_SUPPORTED=1 + break + fi + done + + no_arch_all_support=$(grep "^No-Support-for-Architecture-all: Packages$" "$reldest" || true) + if [ "$no_arch_all_support" != "" ]; then + ARCH_ALL_SUPPORTED=0 + fi +} + CODENAME="" validate_suite () { local reldest suite s @@ -667,7 +685,7 @@ download_release_sig () { download_release_indices () { local m1 inreldest reldest relsigdest totalpkgs \ subpath xzi bz2i gzi normi i ext \ - donepkgs pkgdest acquirebyhash s c m + donepkgs pkgdest acquirebyhash archs s c a m m1="${MIRRORS%% *}" for s in $SUITE $EXTRA_SUITES; do inreldest="$TARGET/$($DLDEST rel "$s" "$m1" "dists/$s/InRelease")" @@ -680,70 +698,80 @@ download_release_indices () { extract_release_components "$reldest" + repo_supports_arch_all "$reldest" + + archs="$ARCH" + if [ $ARCH_ALL_SUPPORTED -eq 1 ]; then + archs="all $ARCH" + fi + acquirebyhash=$(grep "^Acquire-By-Hash: yes$" "$reldest" || true) - totalpkgs=0 - for c in $COMPONENTS; do - subpath="$c/binary-$ARCH/Packages" - xzi="$(get_release_checksum "$reldest" "$subpath.xz")" - bz2i="$(get_release_checksum "$reldest" "$subpath.bz2")" - gzi="$(get_release_checksum "$reldest" "$subpath.gz")" - normi="$(get_release_checksum "$reldest" "$subpath")" - if [ "$normi" != "" ]; then - i="$normi" - elif in_path bunzip2 && [ "$bz2i" != "" ]; then - i="$bz2i" - elif in_path unxz && [ "$xzi" != "" ]; then - i="$xzi" - elif in_path gunzip && [ "$gzi" != "" ]; then - i="$gzi" - fi - if [ "$i" != "" ]; then - totalpkgs=$(( $totalpkgs + ${i#* } )) - else - mv "$reldest" "$reldest.malformed" - error 1 MISSINGRELENTRY "Invalid Release file, no entry for %s" "$subpath" - fi - done - donepkgs=0 - progress 0 $totalpkgs DOWNPKGS "Downloading Packages files" - for c in $COMPONENTS; do - subpath="$c/binary-$ARCH/Packages" - path="dists/$s/$subpath" - xzi="$(get_release_checksum "$reldest" "$subpath.xz")" - bz2i="$(get_release_checksum "$reldest" "$subpath.bz2")" - gzi="$(get_release_checksum "$reldest" "$subpath.gz")" - normi="$(get_release_checksum "$reldest" "$subpath")" - ext="" - if [ "$acquirebyhash" != "" ]; then - ext="$ext byhash" - fi - if [ "$normi" != "" ]; then - ext="$ext $normi ." - i="$normi" - fi - if in_path unxz && [ "$xzi" != "" ]; then - ext="$ext $xzi xz" - i="${i:-$xzi}" - fi - if in_path bunzip2 && [ "$bz2i" != "" ]; then - ext="$ext $bz2i bz2" - i="${i:-$bz2i}" - fi - if in_path gunzip && [ "$gzi" != "" ]; then - ext="$ext $gzi gz" - i="${i:-$gzi}" - fi - progress_next $(($donepkgs + ${i#* })) - for m in $MIRRORS; do - pkgdest="$TARGET/$($DLDEST pkg "$s" "$c" "$ARCH" "$m" "$path")" - if get "$m/$path" "$pkgdest" $ext; then break; fi + for a in $archs; do + totalpkgs=0 + for c in $COMPONENTS; do + subpath="$c/binary-$a/Packages" + xzi="$(get_release_checksum "$reldest" "$subpath.xz")" + bz2i="$(get_release_checksum "$reldest" "$subpath.bz2")" + gzi="$(get_release_checksum "$reldest" "$subpath.gz")" + normi="$(get_release_checksum "$reldest" "$subpath")" + if [ "$normi" != "" ]; then + i="$normi" + elif in_path bunzip2 && [ "$bz2i" != "" ]; then + i="$bz2i" + elif in_path unxz && [ "$xzi" != "" ]; then + i="$xzi" + elif in_path gunzip && [ "$gzi" != "" ]; then + i="$gzi" + fi + if [ "$i" != "" ]; then + totalpkgs=$(( $totalpkgs + ${i#* } )) + else + mv "$reldest" "$reldest.malformed" + error 1 MISSINGRELENTRY "Invalid Release file, no entry for %s" "$subpath" + fi + done + + donepkgs=0 + progress 0 $totalpkgs DOWNPKGS "Downloading Packages files for $a" + for c in $COMPONENTS; do + subpath="$c/binary-$a/Packages" + path="dists/$s/$subpath" + xzi="$(get_release_checksum "$reldest" "$subpath.xz")" + bz2i="$(get_release_checksum "$reldest" "$subpath.bz2")" + gzi="$(get_release_checksum "$reldest" "$subpath.gz")" + normi="$(get_release_checksum "$reldest" "$subpath")" + ext="" + if [ "$acquirebyhash" != "" ]; then + ext="$ext byhash" + fi + if [ "$normi" != "" ]; then + ext="$ext $normi ." + i="$normi" + fi + if in_path unxz && [ "$xzi" != "" ]; then + ext="$ext $xzi xz" + i="${i:-$xzi}" + fi + if in_path bunzip2 && [ "$bz2i" != "" ]; then + ext="$ext $bz2i bz2" + i="${i:-$bz2i}" + fi + if in_path gunzip && [ "$gzi" != "" ]; then + ext="$ext $gzi gz" + i="${i:-$gzi}" + fi + progress_next $(($donepkgs + ${i#* })) + for m in $MIRRORS; do + pkgdest="$TARGET/$($DLDEST pkg "$s" "$c" "$a" "$m" "$path")" + if get "$m/$path" "$pkgdest" $ext; then break; fi + done + if [ ! -f "$pkgdest" ]; then + error 1 COULDNTDL "Couldn't download %s" "$m/$path" + fi + donepkgs=$(($donepkgs + ${i#* })) + progress $donepkgs $totalpkgs DOWNPKGS "Downloading Packages files" done - if [ ! -f "$pkgdest" ]; then - error 1 COULDNTDL "Couldn't download %s" "$m/$path" - fi - donepkgs=$(($donepkgs + ${i#* })) - progress $donepkgs $totalpkgs DOWNPKGS "Downloading Packages files" done done } @@ -807,7 +835,7 @@ download_debs () { download_release () { local m1 numdebs countdebs totaldebs leftoverdebs path pkgdest \ - dloaddebs s c m + dloaddebs archs s c a m m1="${MIRRORS%% *}" numdebs="$#" @@ -822,25 +850,32 @@ download_release () { leftoverdebs=$(printf "$leftoverdebs"|tr ' ' '\n'|sort -u|tr '\n' ' ') numdebs=$(printf "$leftoverdebs"|wc -w) + archs="$ARCH" + if [ $ARCH_ALL_SUPPORTED -eq 1 ]; then + archs="all $ARCH" + fi + for s in $SUITE $EXTRA_SUITES; do - for c in $COMPONENTS; do - if [ "$countdebs" -ge "$numdebs" ]; then break; fi + for a in $archs; do + for c in $COMPONENTS; do + if [ "$countdebs" -ge "$numdebs" ]; then break; fi - path="dists/$s/$c/binary-$ARCH/Packages" - pkgdest="$TARGET/$($DLDEST pkg "$s" "$c" "$ARCH" "$m1" "$path")" - if [ ! -e "$pkgdest" ]; then continue; fi + path="dists/$s/$c/binary-$a/Packages" + pkgdest="$TARGET/$($DLDEST pkg "$s" "$c" "$a" "$m1" "$path")" + if [ ! -e "$pkgdest" ]; then continue; fi - info CHECKINGSIZES "Checking component %s on %s..." "$c" "$m1" + info CHECKINGSIZES "Checking component %s on %s..." "$c" "$m1" - leftoverdebs="$(get_package_sizes "$m1" "$pkgdest" $leftoverdebs)" + leftoverdebs="$(get_package_sizes "$m1" "$pkgdest" $leftoverdebs)" - countdebs=$(($countdebs + ${leftoverdebs%% *})) - leftoverdebs=${leftoverdebs#* } + countdebs=$(($countdebs + ${leftoverdebs%% *})) + leftoverdebs=${leftoverdebs#* } - totaldebs=${leftoverdebs%% *} - leftoverdebs=${leftoverdebs#* } + totaldebs=${leftoverdebs%% *} + leftoverdebs=${leftoverdebs#* } - progress "$countdebs" "$numdebs" SIZEDEBS "Finding package sizes" + progress "$countdebs" "$numdebs" SIZEDEBS "Finding package sizes" + done done done @@ -855,15 +890,17 @@ download_release () { pkgs_to_get="$*" for s in $SUITE $EXTRA_SUITES; do - for c in $COMPONENTS; do - path="dists/$s/$c/binary-$ARCH/Packages" - for m in $MIRRORS; do - pkgdest="$TARGET/$($DLDEST pkg "$s" "$c" "$ARCH" "$m" "$path")" - if [ ! -e "$pkgdest" ]; then continue; fi - pkgs_to_get="$(download_debs "$m" "$pkgdest" $pkgs_to_get 5>&1 1>&6)" + for a in $archs; do + for c in $COMPONENTS; do + path="dists/$s/$c/binary-$a/Packages" + for m in $MIRRORS; do + pkgdest="$TARGET/$($DLDEST pkg "$s" "$c" "$a" "$m" "$path")" + if [ ! -e "$pkgdest" ]; then continue; fi + pkgs_to_get="$(download_debs "$m" "$pkgdest" $pkgs_to_get 5>&1 1>&6)" + if [ -z "$pkgs_to_get" ]; then break; fi + done 6>&1 if [ -z "$pkgs_to_get" ]; then break; fi - done 6>&1 - if [ -z "$pkgs_to_get" ]; then break; fi + done done if [ -z "$pkgs_to_get" ]; then break; fi done @@ -947,15 +984,23 @@ download_main () { ###################################################### deb choosing support get_debs () { - local field m1 s c path pkgdest + local field m1 s c a path pkgdest field="$1" shift + + archs="$ARCH" + if [ $ARCH_ALL_SUPPORTED -eq 1 ]; then + archs="all $ARCH" + fi + for m1 in $MIRRORS; do for s in $SUITE $EXTRA_SUITES; do for c in $COMPONENTS; do - path="dists/$s/$c/binary-$ARCH/Packages" - pkgdest="$TARGET/$($DLDEST pkg "$s" "$c" "$ARCH" "$m1" "$path")" - echo "$("$PKGDETAILS" FIELD "$field" "$m1" "$pkgdest" "$@" | sed 's/ .*//')" + for a in $archs; do + path="dists/$s/$c/binary-$a/Packages" + pkgdest="$TARGET/$($DLDEST pkg "$s" "$c" "$a" "$m1" "$path")" + echo "$("$PKGDETAILS" FIELD "$field" "$m1" "$pkgdest" "$@" | sed 's/ .*//')" + done done done done @@ -1587,14 +1632,22 @@ resolve_deps () { local PKGS="$*" local ALLPKGS="$PKGS"; local ALLPKGS2=""; - local s c + local s c a + + archs="$ARCH" + if [ $ARCH_ALL_SUPPORTED -eq 1 ]; then + archs="all $ARCH" + fi + while [ "$PKGS" != "" ]; do local NEWPKGS="" - for s in $SUITE $EXTRA_SUITES; do - for c in ${COMPONENTS:-$(echo ${USE_COMPONENTS} | tr '|' ' ')}; do - local path="dists/$s/$c/binary-$ARCH/Packages" - local pkgdest="$TARGET/$($DLDEST pkg "$s" "$c" "$ARCH" "$m1" "$path")" - NEWPKGS="$NEWPKGS $("$PKGDETAILS" GETDEPS "$pkgdest" $PKGS)" + for a in $archs; do + for s in $SUITE $EXTRA_SUITES; do + for c in ${COMPONENTS:-$(echo ${USE_COMPONENTS} | tr '|' ' ')}; do + local path="dists/$s/$c/binary-$a/Packages" + local pkgdest="$TARGET/$($DLDEST pkg "$s" "$c" "$a" "$m1" "$path")" + NEWPKGS="$NEWPKGS $("$PKGDETAILS" GETDEPS "$pkgdest" $PKGS)" + done done done if [ -n "${EXCLUDE_DEPENDENCY:-}" ]; then @@ -1602,11 +1655,13 @@ resolve_deps () { fi PKGS=$(echo "$PKGS $NEWPKGS" | tr ' ' '\n' | sort | uniq) local REALPKGS="" - for s in $SUITE $EXTRA_SUITES; do - for c in ${COMPONENTS:-$(echo ${USE_COMPONENTS} | tr '|' ' ')}; do - local path="dists/$s/$c/binary-$ARCH/Packages" - local pkgdest="$TARGET/$($DLDEST pkg "$s" "$c" "$ARCH" "$m1" "$path")" - REALPKGS="$REALPKGS $("$PKGDETAILS" PKGS REAL "$pkgdest" $PKGS | sed -n 's/ .*REAL.*$//p')" + for a in $archs; do + for s in $SUITE $EXTRA_SUITES; do + for c in ${COMPONENTS:-$(echo ${USE_COMPONENTS} | tr '|' ' ')}; do + local path="dists/$s/$c/binary-$a/Packages" + local pkgdest="$TARGET/$($DLDEST pkg "$s" "$c" "$a" "$m1" "$path")" + REALPKGS="$REALPKGS $("$PKGDETAILS" PKGS REAL "$pkgdest" $PKGS | sed -n 's/ .*REAL.*$//p')" + done done done PKGS="$REALPKGS" @@ -1621,15 +1676,22 @@ setup_available () { local m1 c path pkgdest pkg m1="${MIRRORS%% *}" + archs="$ARCH" + if [ $ARCH_ALL_SUPPORTED -eq 1 ]; then + archs="all $ARCH" + fi + for s in $SUITE $EXTRA_SUITES; do - for c in ${COMPONENTS:-$(echo ${USE_COMPONENTS} | tr '|' ' ')}; do - path="dists/$s/$c/binary-$ARCH/Packages" - pkgdest="$TARGET/$($DLDEST pkg "$s" "$c" "$ARCH" "$m1" "$path")" - # XXX: What if a package is in more than one component? - # -- cjwatson 2009-07-29 - # XXX: ...or suite? - # -- jrtc27 2019-06-11 - "$PKGDETAILS" STANZAS "$pkgdest" "$@" + for a in $archs; do + for c in ${COMPONENTS:-$(echo ${USE_COMPONENTS} | tr '|' ' ')}; do + path="dists/$s/$c/binary-$a/Packages" + pkgdest="$TARGET/$($DLDEST pkg "$s" "$c" "$a" "$m1" "$path")" + # XXX: What if a package is in more than one component? + # -- cjwatson 2009-07-29 + # XXX: ...or suite? + # -- jrtc27 2019-06-11 + "$PKGDETAILS" STANZAS "$pkgdest" "$@" + done done done >"$TARGET/var/lib/dpkg/available" -- GitLab
