bin/update/create_full_mar.py | 18 +++++++++++++
external/onlineupdate/lo.patch | 34 ++++++++++++++++++++++----
solenv/bin/modules/installer/globals.pm | 1
solenv/bin/modules/installer/simplepackage.pm | 31 ++++++++++++++++++++++-
4 files changed, 77 insertions(+), 7 deletions(-)
New commits:
commit 416b2ee59fe8209470d5cdcc2332682e22fbc719
Author: Stephan Bergmann <stephan.bergm...@allotropia.de>
AuthorDate: Thu Jan 11 08:52:22 2024 +0100
Commit: Stephan Bergmann <stephan.bergm...@allotropia.de>
CommitDate: Thu Jan 11 13:01:50 2024 +0100
Make create-update-info handle conditional content
That make target operates on an archive, but generates data to be applied
to an
msi installation, so suffers from any mismatch between archive and msi
install
sets. Two such mismatches, at least on Windows, are:
1 Files that msi will install outside the LibreOffice installation itself
(but
which for an archive install set are included under LibreOffice/). This
covers
(at least) .Net assemblies and associated files that are installed in the
GAC
(scp2 styles ASSEMBLY and ASSIGNCOMPONENT) and fonts (scp2 style FONT).
2 Files that msi will only install conditionally. This covers optional
components (many of which in scp2 are assigned to gid_Module_Optional_...
modules) and user interface languages (which in scp2 are assigned to
gid_Module_Langpack_... modules).
The approach taken here is to create a
workdir/installation/LibreOffice/archive/install/metadata file while
building an
archive install set, and to record any files matching 1 (as "skip" lines)
or 2
(as "cond" lines). Then, the create-update-info target uses that metadata
file
to act accordingly on those files:
1 Files from "skip" lines are simply removed for now from the extracted
archive
that is passed to Mozilla's make_full_update.sh script. (TODO: That means
that
changes to such files will not be updated with the MAR update mechanism.
This
would apparently need some extra processing during the MAR update.)
2 Files from "cond" lines shall be recorded as add-if in the mar file
manifest.
Mozilla's make_full_update.sh script already has support for emitting
add-if vs.
plain add, but only for files under distribution/extensions/, which doesn't
match our needs. So we generate from the metadata file an ifs file that we
pass
into the make_full_update.sh script, and patch that script to also take
that ifs
file into account. (Each line in the ifs file is of the form
"testfile" "file"
which works as long as none of the pathnames contains double quote
characters.
The Mozilla script code appears to be confused about the arguments to
make_add_instruction(), where this ifs file will be needed: There are
calls to
make_add_instruction() with two or three arguments across
make_full_update.sh
and make_incremental_update.sh, but make_add_instruction() checks $1, $2,
and
$4 (but not $3), so leave that mess alone and pass the ifs file as a global
IFSFILE variable instead.)
The mar file manifest `add-if "testfile" "file"` adds "file" only if
"testfile"
is already present, and those two can be different files. TODO: However,
for
simplicity, for now I always use "file" also as the "testfile" (so that an
add-if file only gets updated if it was already present). That avoids
having to
identify a specific "key file" for each optional component and for each user
interface language, where that key file would be used as the add-if
testfile.
But on the other hand, it means that if an optional component or a user
interface language will bring along a completely new file in the future, we
will
not install that file during a MAR update. What obviously remains to be
done is
to properly assign each add-if file to a specific key file. (And the
current
way of identifying add-if files by gid_Module_... names appears to be too
simplistic too. For example, there are some gid_Module_Optional_... that
are
installed unconditionally for msi; but it should be harmless that those
files
are recorded as add-if rather than as plain add.)
Change-Id: I2fdeed92604f3a2d8a0b500b9e3fa421cfb6a9cc
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/161917
Tested-by: Jenkins
Reviewed-by: Stephan Bergmann <stephan.bergm...@allotropia.de>
diff --git a/bin/update/create_full_mar.py b/bin/update/create_full_mar.py
index fb0f90491d92..1c2b249db3b5 100755
--- a/bin/update/create_full_mar.py
+++ b/bin/update/create_full_mar.py
@@ -2,6 +2,7 @@
import glob
import os
+import re
import subprocess
import json
import argparse
@@ -44,12 +45,27 @@ def main():
uncompress_dir = uncompress_file_to_dir(tar_file, temp_dir)
+ metadatafile = os.path.join(
+ update_path.get_workdir(), 'installation', product_name, 'archive',
'install', 'metadata')
+ ifsfile = os.path.join(update_path.get_mar_dir(), 'ifs')
+ with open(metadatafile) as meta, open(ifsfile, 'w') as ifs:
+ for l in meta:
+ m = re.fullmatch('(skip|cond) (.*)', l.rstrip())
+ if m and m.group(2).startswith(f'{product_name}/'):
+ path = m.group(2)[len(f'{product_name}/'):]
+ if m.group(1) == 'skip':
+ os.remove(os.path.join(uncompress_dir, path))
+ else:
+ ifs.write(f'"{path}" "{path}"
')
+
mar_file = make_complete_mar_name(target_dir, filename_prefix)
path = os.path.join(
workdir,
'UnpackedTarball/onlineupdate/tools/update-packaging/make_full_update.sh')
os.putenv('MOZ_PRODUCT_VERSION', version)
os.putenv('MAR_CHANNEL_ID', 'LOOnlineUpdater')
- subprocess.call([path, convert_to_native(mar_file),
convert_to_native(uncompress_dir)])
+ subprocess.call([
+ path, convert_to_native(mar_file), convert_to_native(uncompress_dir),
+ convert_to_native(ifsfile)])
sign_mar_file(target_dir, certificate_path, certificate_name, mar_file,
filename_prefix)
diff --git a/external/onlineupdate/lo.patch b/external/onlineupdate/lo.patch
index 0021c3a69fe7..870857b7ba46 100644
--- a/external/onlineupdate/lo.patch
+++ b/external/onlineupdate/lo.patch
@@ -223,9 +223,35 @@
LOG_WARN(("Install directory updater could not be determined."));
result = FALSE;
}
+--- tools/update-packaging/common.sh
++++ tools/update-packaging/common.sh
+@@ -76,6 +76,15 @@
+ forced=
+ fi
+
++ if [ -n "$IFSFILE" ]; then
++ ifsline=$(grep -F " \"$f\"" "$IFSFILE")
++ if [ -n "$ifsline" ]; then
++ testfile=$(printf '%s' "$ifsline" | cut -f 2 -d '"')
++ verbose_notice " add-if \"$testfile\" \"$f\""
++ echo "add-if \"$testfile\" \"$f\"" >> "$filev3"
++ return
++ fi
++ fi
+ is_extension=$(echo "$f" | grep -c 'distribution/extensions/.*/')
+ if [ $is_extension = "1" ]; then
+ # Use the subdirectory of the extensions folder as the file to test
--- tools/update-packaging/make_full_update.sh
+++ tools/update-packaging/make_full_update.sh
-@@ -53,9 +53,10 @@
+@@ -45,6 +45,7 @@
+
+ archive="$1"
+ targetdir="$2"
++IFSFILE=$3
+ # Prevent the workdir from being inside the targetdir so it isn't included in
+ # the update mar.
+ if [ $(echo "$targetdir" | grep -c '\/$') = 1 ]; then
+@@ -53,9 +54,10 @@
fi
workdir="$targetdir.work"
updatemanifestv3="$workdir/updatev3.manifest"
@@ -237,7 +263,7 @@
# Generate a list of all files in the target directory.
pushd "$targetdir"
-@@ -66,7 +67,6 @@
+@@ -66,7 +68,6 @@
if [ ! -f "precomplete" ]; then
if [ ! -f "Contents/Resources/precomplete" ]; then
notice "precomplete file is missing!"
@@ -245,7 +271,7 @@
fi
fi
-@@ -99,7 +99,7 @@
+@@ -99,7 +100,7 @@
$XZ $XZ_OPT --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64
--force --stdout "$targetdir/$f" > "$workdir/$f"
copy_perm "$targetdir/$f" "$workdir/$f"
@@ -254,7 +280,7 @@
done
# Append remove instructions for any dead files.
-@@ -110,7 +110,7 @@
+@@ -110,7 +111,7 @@
$XZ $XZ_OPT --compress $BCJ_OPTIONS --lzma2 --format=xz --check=crc64 --force
"$updatemanifestv3" && mv -f "$updatemanifestv3.xz" "$updatemanifestv3"
mar_command="$mar_command -C \"$workdir\" -c output.mar"
diff --git a/solenv/bin/modules/installer/globals.pm
b/solenv/bin/modules/installer/globals.pm
index 045d9d6afde9..d210cefa179a 100644
--- a/solenv/bin/modules/installer/globals.pm
+++ b/solenv/bin/modules/installer/globals.pm
@@ -185,6 +185,7 @@ BEGIN
$installer::globals::is_copy_only_project = 0;
$installer::globals::is_simple_packager_project = 0;
$installer::globals::patch_user_dir = 0;
+ $installer::globals::record_archive_metadata = 0;
$installer::globals::languagepack = 0;
$installer::globals::helppack = 0;
$installer::globals::refresh_includepaths = 0;
diff --git a/solenv/bin/modules/installer/simplepackage.pm
b/solenv/bin/modules/installer/simplepackage.pm
index a8b98a222e5f..3a876c4a36d7 100644
--- a/solenv/bin/modules/installer/simplepackage.pm
+++ b/solenv/bin/modules/installer/simplepackage.pm
@@ -47,8 +47,12 @@ sub check_simple_packager_project
$installer::globals::is_simple_packager_project = 1;
$installer::globals::patch_user_dir = 1;
}
- elsif(( $installer::globals::packageformat eq "archive" ) ||
- ( $installer::globals::packageformat eq "dmg" ) )
+ elsif( $installer::globals::packageformat eq "archive" )
+ {
+ $installer::globals::is_simple_packager_project = 1;
+ $installer::globals::record_archive_metadata = 1;
+ }
+ elsif( $installer::globals::packageformat eq "dmg" )
{
$installer::globals::is_simple_packager_project = 1;
}
@@ -608,6 +612,9 @@ sub create_simple_package
# stripping files ?!
if (( $installer::globals::strip ) && ( !
$installer::globals::iswindowsbuild )) { strip_libraries($filesref,
$languagestringref); }
+ my @archive_metadata_skip;
+ my @archive_metadata_cond;
+
# copy Files
installer::logger::print_message( "... copying files ...
" );
installer::logger::include_header_into_logfile("Copying files:");
@@ -658,6 +665,18 @@ sub create_simple_package
}
}
}
+
+ if ($installer::globals::record_archive_metadata)
+ {
+ if ($onefile->{'Styles'} =~ /(ASSEMBLY|ASSIGNCOMPONENT|FONT)/)
+ {
+ push(@archive_metadata_skip, $onefile->{'destination'});
+ }
+ elsif ($onefile->{'modules'} =~ /^gid_Module_(Langpack|Optional)_/)
+ {
+ push(@archive_metadata_cond, $onefile->{'destination'});
+ }
+ }
}
# creating Links
@@ -726,6 +745,14 @@ sub create_simple_package
create_package($installdir, $installdir, $packagename, $allvariables,
$includepatharrayref, $languagestringref, ".dmg");
}
+ if ($installer::globals::record_archive_metadata)
+ {
+ open(HANDLE, '>', "$installer::globals::csp_installdir/../metadata")
or die $!;
+ print HANDLE "skip $_
" foreach (sort(@archive_metadata_skip));
+ print HANDLE "cond $_
" foreach (sort(@archive_metadata_cond));
+ close HANDLE;
+ }
+
# Analyzing the log file
installer::worker::clean_output_tree(); # removing directories created in
the output tree
