Brought forward onto debmirror 2.4.5.
-Drake <[email protected]>
diff --git a/debmirror b/debmirror
index 089cdac..ecf7dbf 100755
--- a/debmirror
+++ b/debmirror
@@ -69,7 +69,7 @@ Usage: $0 [--progress] [--verbose] [--debug] [--dry-run] [--help]
[--postcleanup|--cleanup|--nocleanup] [--skippackages]
[--diff=use|mirror|none] [--gzip-options=options]
[--state-cache-days=number]
- [--ignore-small-errors] [--allow-dist-rename]
+ [--ignore-small-errors] [--allow-dist-rename] [--debmarshal]
<mirrordir>
For details, see man page.
@@ -394,6 +394,13 @@ An existing symlink S<codename -E<gt> suite> will be removed, but debmirror
will automatically create a new symlink S<suite -E<gt> codename> (immediately
after moving meta files in place). This conversion should only be needed once.
+=item --debmarshal
+
+On each pull, keep the repository meta data from dists/* in a numbered
+subdirectory, and maintain a symlink latest to the most recent pull.
+This is similar to Debmarshal in tracking mode, see
+debmarshal.debian.net for examples and use.
+
=back
=head1 USING DEBMIRROR
@@ -552,6 +559,7 @@ our $diff_mode="use";
our $gzip_options="-9 -n --rsyncable";
our $omit_suite_symlinks=0;
our $allow_dist_rename=0;
+our $debmarshal=0;
my @errlog;
my $HOME;
($HOME = $ENV{'HOME'}) or die "HOME not defined in environment!\n";
@@ -646,6 +654,7 @@ GetOptions('debug' => \$debug,
'diff=s' => \$diff_mode,
'omit-suite-symlinks' => \$omit_suite_symlinks,
'allow-dist-rename' => \$allow_dist_rename,
+ 'debmarshal' => \$debmarshal,
'help' => \$help,
) or usage;
usage if $help;
@@ -725,6 +734,7 @@ if ($post_cleanup) {
say("Will NOT clean up.") unless $cleanup;
}
say("Dry run.") if $dry_run;
+say("Debmarshal snapshots kept.") if $debmarshal;
my $md5;
$md5=Digest::MD5->new;
@@ -830,17 +840,32 @@ foreach my $dist (@dists) {
my ($codename, $suite, $dist_sdir) = name_release("mirror", $tdir, $dist);
if ($have_release) {
+ my $next;
make_dir ("dists/$codename$dist_sdir");
make_dir ("$tempdir/dists/$codename$dist_sdir");
rename("$tdir/Release", "$tempdir/dists/$codename$dist_sdir/Release")
or die "Error while moving $tdir/Release: $!\n";
$files{"dists/$codename$dist_sdir/Release"}=1;
$files{$tempdir."/"."dists/$codename$dist_sdir/Release"}=1;
+ if ($debmarshal) {
+ $next = get_nextlatest($dist);
+ make_dir("$mirrordir/dists/$dist/$next");
+ unlink("$mirrordir/dists/$dist/$next/Release");
+ link("$tempdir/dists/$codename$dist_sdir/Release",
+ "$mirrordir/dists/$dist/$next/Release")
+ or die "Error while linking $tempdir/dists/$codename$dist_sdir/Release: $!\n";
+ }
if (-f "$tdir/Release.gpg") {
rename("$tdir/Release.gpg", "$tempdir/dists/$codename$dist_sdir/Release.gpg")
or die "Error while moving $tdir/Release.gpg: $!\n";
$files{"dists/$codename$dist_sdir/Release.gpg"}=1;
$files{$tempdir."/"."dists/$codename$dist_sdir/Release.gpg"}=1;
+ if ($debmarshal) {
+ unlink("$mirrordir/dists/$dist/$next/Release.gpg");
+ link("$tempdir/dists/$codename$dist_sdir/Release.gpg",
+ "$mirrordir/dists/$dist/$next/Release.gpg")
+ or die "Error while linking $tempdir/dists/$codename$dist_sdir/Release.gpg: $!\n";
+ }
}
}
}
@@ -933,12 +958,14 @@ foreach my $dist (keys %distset) {
next if ($section =~ /debian-installer/ && $dist eq "breezy-security" );
foreach my $arch (@arches) {
get_index("dists/$dist/$section/binary-$arch", "Packages");
+ link_index($dist,$section,$arch) if $debmarshal;
}
# d-i does not have separate source sections
if ($do_source && $section !~ /debian-installer/) {
get_index("dists/$dist/$section/source", "Sources");
+ link_index($dist,$section,"source") if $debmarshal;
}
- get_i18n_index("dists/$dist/$section/i18n") if $i18n;
+ get_i18n_index($dist,"$section/i18n") if $i18n;
}
}
@@ -991,7 +1018,7 @@ if ($i18n) {
foreach my $dist (keys %distset) {
next unless exists $distset{$dist}{mirror};
foreach my $section (@sections) {
- parse_i18n_index("dists/$dist/$section/i18n");
+ parse_i18n_index($dist,"$section/i18n");
}
}
}
@@ -1134,7 +1161,7 @@ get_contents_files() if ($getcontents);
get_i18n_files() if ($i18n);
# Pre-mirror cleanup
-if ($cleanup && ! $post_cleanup) {
+if ($cleanup && ! $post_cleanup && ! $debmarshal) {
# close ftp connection during cleanup to avoid timeouts
if ($download_method eq 'ftp') { $ftp->quit; }
cleanup_unknown_files();
@@ -1297,6 +1324,21 @@ if (! @di_dists) {
}
say("Everything OK. Moving meta files.");
+if ($debmarshal) {
+ foreach my $dist (@dists) {
+ system("diff","-q","$mirrordir/dists/$dist/latest/Release",
+ "$tempdir/dists/$dist/Release");
+ if ($?) {
+ my $next = get_nextlatest($dist);
+ say("Updating $mirrordir/dists/$dist/latest to $next");
+ unlink("$mirrordir/dists/$dist/latest");
+ symlink($next,"$mirrordir/dists/$dist/latest")
+ or die "Error while symlinking $mirrordir/dists/$dist/latest to $next: $!\n";
+ } else {
+ say("Not updating $mirrordir/dists/$dist/latest");
+ }
+ }
+}
chdir($tempdir) or die "unable to chdir($tempdir): $!\n";
my $res=0;
foreach my $file (`find . -type f`) {
@@ -1316,6 +1358,7 @@ foreach my $file (`find . -type f`) {
}
chdir($mirrordir) or die "chdir $mirrordir: $!";
+
# Get optional directories using rsync.
rsync_extra(0, @rsync_extra);
@@ -1364,7 +1407,7 @@ if (! $do_dry_run) {
}
# Post mirror cleanup.
-cleanup_unknown_files() if ($post_cleanup);
+cleanup_unknown_files() if ($post_cleanup && ! $debmarshal);
# Mirror cleanup for directories.
if (! $use_cache && ($cleanup || $post_cleanup)) {
@@ -1768,6 +1811,17 @@ sub split_dist {
return ($dist_raw, $dist_sdir);
}
+sub get_nextlatest {
+ my ($dist) = @_;
+ my $latest = readlink("$mirrordir/dists/$dist/latest");
+ if (defined $latest) {
+ $latest++;
+ } else {
+ $latest = 0;
+ }
+ return $latest;
+}
+
sub get_release {
my ($tdir, $dist) = @_;
@@ -2068,12 +2122,47 @@ sub get_contents_files {
}
$files{"dists/$dist/Contents-$arch.gz"}=1;
$files{$tempdir."/"."dists/$dist/Contents-$arch.gz"}=1;
+ if ($debmarshal) {
+ my $next = get_nextlatest($dist);
+ unlink("$mirrordir/dists/$dist/$next/Contents-$arch.gz");
+ link("$tempdir/dists/$dist/Contents-$arch.gz",
+ "$mirrordir/dists/$dist/$next/Contents-$arch.gz")
+ or die "Error while linking $tempdir/dists/$dist/Contents-$arch.gz: $!\n";
+ }
}
}
}
+# hardlink index files from tempdir to next debmarshal snapshot location
+sub link_index {
+ my ($dist,$section,$arch) = @_;
+ my ($file,$archdir);
+ if ($arch eq "source") {
+ $file = "Sources";
+ $archdir = "source";
+ } else {
+ $file = "Packages";
+ $archdir = "binary-$arch";
+ }
+ my $next = get_nextlatest($dist);
+ make_dir("$mirrordir/dists/$dist/$next/$section/$archdir");
+ unlink("$mirrordir/dists/$dist/$next/$section/$archdir/$file");
+ link("$tempdir/dists/$dist/$section/$archdir/$file",
+ "$mirrordir/dists/$dist/$next/$section/$archdir/$file")
+ or warn "Error while linking $tempdir/dists/$dist/$section/$archdir/$file: $!\n";
+ unlink("$mirrordir/dists/$dist/$next/$section/$archdir/$file.gz");
+ link("$tempdir/dists/$dist/$section/$archdir/$file.gz",
+ "$mirrordir/dists/$dist/$next/$section/$archdir/$file.gz")
+ or die "Error while linking $tempdir/dists/$dist/$section/$archdir/$file.gz: $!\n";
+ unlink("$mirrordir/dists/$dist/$next/$section/$archdir/$file.bz2");
+ link("$tempdir/dists/$dist/$section/$archdir/$file.bz2",
+ "$mirrordir/dists/$dist/$next/$section/$archdir/$file.bz2")
+ or die "Error while linking $tempdir/dists/$dist/$section/$archdir/$file.bz2: $!\n";
+}
+
sub get_i18n_index {
- my $subdir=shift;
+ my ($dist,$distpath) = @_;
+ my $subdir = "dists/$dist/$distpath";
if (exists $file_lists{"$tempdir/$subdir/Index"}) {
make_dir($subdir);
make_dir("$tempdir/$subdir");
@@ -2082,6 +2171,13 @@ sub get_i18n_index {
say("$subdir/Release needs fetch");
if (!remote_get("$subdir/Index")) {
push (@errlog,"$subdir/Index failed md5sum check, removing\n");
+ } elsif ($debmarshal) {
+ my $next = get_nextlatest($dist);
+ say("linking $mirrordir/dists/$dist/$next/$distpath/Index");
+ unlink("$mirrordir/dists/$dist/$next/$distpath/Index");
+ link("$tempdir/$subdir/Index",
+ "$mirrordir/dists/$dist/$next/$distpath/Index")
+ or die "Error while linking $tempdir/$subdir/Index: $!\n";
}
} else {
$bytes_gotten += $file_lists{"$tempdir/$subdir/Index"}{size};
@@ -2092,7 +2188,8 @@ sub get_i18n_index {
}
sub parse_i18n_index {
- my $subdir = shift;
+ my ($dist,$distpath) = @_;
+ my $subdir = "dists/$dist/$distpath";
my ($sha1, $size, $filename);
my $exclude = "(".join("|", @excludes).")" if @excludes;
my $include = "(".join("|", @includes).")" if @includes;
@@ -2116,6 +2213,9 @@ sub parse_i18n_index {
$bytes_to_get += $size;
$i18n_get{"$subdir/$filename"}{sha1} = $sha1;
$i18n_get{"$subdir/$filename"}{size} = $size;
+ $i18n_get{"$subdir/$filename"}{dist} = $dist;
+ $i18n_get{"$subdir/$filename"}{distpath} = $distpath;
+ $i18n_get{"$subdir/$filename"}{filename} = $filename;
}
}
close INDEX;
@@ -2127,6 +2227,18 @@ sub get_i18n_files {
foreach my $file (sort keys %i18n_get) {
if (! check_i18n("$tempdir/$file", $i18n_get{$file}{size}, $i18n_get{$file}{sha1})) {
remote_get("$file");
+ if ($debmarshal) {
+ my $dist = $i18n_get{$file}{dist};
+ my $distpath = $i18n_get{$file}{distpath};
+ my $filename = $i18n_get{$file}{filename};
+ my $next = get_nextlatest($dist);
+ say("linking $file");
+ unlink("$mirrordir/dists/$dist/$next/$distpath/$filename");
+ link("$tempdir/$file",
+ "$mirrordir/dists/$dist/$next/$distpath/$filename")
+ or die "Erorr while linking $tempdir/$file: $!";
+
+ }
}
}
}
