Makefile.gbuild | 2
bin/update/create_partial_update.py | 185 ++++--------------
bin/update/path.py | 10
config_host.mk.in | 1
configure.ac | 16 -
distro-configs/Jenkins/LibreOfficeLinuxUpdater.conf | 1
external/onlineupdate/UnpackedTarball_onlineupdate.mk | 6
external/onlineupdate/cygpath.patch | 11 +
external/onlineupdate/lo.patch | 10
9 files changed, 74 insertions(+), 168 deletions(-)
New commits:
commit f7accbfdf3231bfa48f42533e2bcbb27b8a4af04
Author: Stephan Bergmann <stephan.bergm...@allotropia.de>
AuthorDate: Thu Jan 11 17:02:43 2024 +0100
Commit: Michael Stahl <michael.st...@allotropia.de>
CommitDate: Fri Jan 12 11:55:17 2024 +0100
Fix `make create-partial-info` (for Windows, at least)
I got lost trying to figure out how the original
bin/update/create_partial_update.py code was meant to obtain old and new
installation trees to diff, so I simplified that down to the
create-partial-info
make target now expecting an ONLINEUPDATE_MAR_OLDARCHIVE make variable that
points at the old archive install set. (And the
--with-online-update-mar-serverurl configure option is gone for good again.)
The remaining changes are similar to what was needed in
28bad382face10be75af3875e44dde89fbc78108 "Fix `make create-update-info` (for
Windows, at least)". (And the mbsdiff and mar tools expect Windows-style
pathnames, but mktemp returns a Unix-style pathname in cygwin shell
scripts, so
this needed an additional Windows-only external/onlineupdate/cygpath.patch.)
Change-Id: I40690210d62e3f26fb2d574914a0dd4323e6cd62
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/161924
Tested-by: Jenkins
Reviewed-by: Stephan Bergmann <stephan.bergm...@allotropia.de>
(cherry picked from commit 44ea2602e3ed8839012582a466775f10da86ee4b)
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/161937
Reviewed-by: Michael Stahl <michael.st...@allotropia.de>
diff --git a/Makefile.gbuild b/Makefile.gbuild
index 39eab31f95b2..211066becd37 100644
--- a/Makefile.gbuild
+++ b/Makefile.gbuild
@@ -60,7 +60,7 @@ create-partial-info:
$(eval VERSION :=
$(LIBO_VERSION_MAJOR).$(LIBO_VERSION_MINOR).$(LIBO_VERSION_MICRO).$(LIBO_VERSION_PATCH)$(LIBO_VERSION_SUFFIX)$(LIBO_VERSION_SUFFIX_SUFFIX))
$(eval PLATFORM := $(RTL_OS)_$(RTL_ARCH))
$(eval MAR_NAME_PREFIX :=
$(PRODUCTNAME)_$(VERSION)_$(PLATFORM)_$(BUILDID))
- MBSDIFF=$(WORKDIR)/LinkTarget/Executable/mbsdiff
MAR=$(INSTDIR)/program/mar $(SRCDIR)/bin/update/create_partial_update.py
"$(WORKDIR)" "$(MAR_NAME_PREFIX)" "$(ONLINEUPDATE_MAR_SERVERURL)"
LOOnlineUpdater "$(ONLINEUPDATE_MAR_CERTIFICATEPATH)"
"$(ONLINEUPDATE_MAR_CERTIFICATENAME)" "$(ONLINEUPDATE_MAR_BASEURL)"
"$(PLATFORM)" "$(BUILDID)"
+ MBSDIFF=$(WORKDIR)/LinkTarget/Executable/mbsdiff
MAR=$(INSTDIR)/program/mar $(if $(filter WNT,$(OS)),$(shell cygpath -u
$(SRCDIR)/bin/update/create_partial_update.py),$(SRCDIR)/bin/update/create_partial_update.py)
"$(WORKDIR)" "$(MAR_NAME_PREFIX)" LOOnlineUpdater
"$(ONLINEUPDATE_MAR_CERTIFICATEPATH)" "$(ONLINEUPDATE_MAR_CERTIFICATENAME)"
"$(ONLINEUPDATE_MAR_BASEURL)" "$(PRODUCTNAME)"
'$(LIBO_VERSION_MAJOR).$(LIBO_VERSION_MINOR).$(LIBO_VERSION_MICRO).$(LIBO_VERSION_PATCH)'
"$(ONLINEUPDATE_MAR_OLDARCHIVE)"
# also possible to bypass the dependencies/the gbuild processing by just
running
# LD_LIBRARY_PATH=instdir/program make cmd cmd='ALL_LANGS="$(ALL_LANGS)"
workdir/LinkTarget/Executable/pocheck'
diff --git a/bin/update/create_partial_update.py
b/bin/update/create_partial_update.py
index 2730c4765f14..cc9bb745852b 100755
--- a/bin/update/create_partial_update.py
+++ b/bin/update/create_partial_update.py
@@ -1,90 +1,14 @@
#!/usr/bin/env python3
+import glob
import json
import os
+import re
import subprocess
import sys
-import requests
-
-from path import UpdaterPath, mkdir_p, convert_to_unix, convert_to_native
+from path import UpdaterPath, convert_to_native
from signing import sign_mar_file
-from tools import get_file_info, get_hash
-from uncompress_mar import extract_mar
-
-BUF_SIZE = 1024
-current_dir_path = os.path.dirname(os.path.realpath(convert_to_unix(__file__)))
-
-
-class InvalidFileException(Exception):
-
- def __init__(self, *args, **kwargs):
- super().__init__(self, *args, **kwargs)
-
-
-def download_file(filepath, url, hash_string):
- with open(filepath, "wb") as f:
- response = requests.get(url, stream=True)
-
- if not response.ok:
- return
-
- for block in response.iter_content(1024):
- f.write(block)
-
- file_hash = get_hash(filepath)
-
- if file_hash != hash_string:
- raise InvalidFileException(
- "file hash does not match for file %s: Expected %s, Got: %s" %
(url, hash_string, file_hash))
-
-
-def handle_language(lang_entries, filedir):
- langs = {}
- for lang, data in lang_entries.items():
- lang_dir = os.path.join(filedir, lang)
- lang_file = os.path.join(lang_dir, "lang.mar")
- mkdir_p(lang_dir)
- download_file(lang_file, data["url"], data["hash"])
- dir_path = os.path.join(lang_dir, "lang")
- mkdir_p(dir_path)
- extract_mar(lang_file, dir_path)
- langs[lang] = dir_path
-
- return langs
-
-
-def download_mar_for_update_channel_and_platform(server_url, channel,
platform, temp_dir):
- base_url = server_url + "update/partial-targets/1/"
- url = base_url + platform + "/" + channel
- r = requests.get(url)
- if r.status_code != 200:
- print(r.content)
- raise Exception("download failed")
-
- update_info = json.loads(r.content.decode("utf-8"))
- update_files = update_info['updates']
- downloaded_updates = {}
- for update_file in update_files:
- build = update_file["build"]
- filedir = os.path.join(temp_dir, build)
-
- mkdir_p(filedir)
-
- filepath = filedir + "/complete.mar"
- url = update_file["update"]["url"]
- expected_hash = update_file["update"]["hash"]
- download_file(filepath, url, expected_hash)
-
- dir_path = os.path.join(filedir, "complete")
- mkdir_p(dir_path)
- extract_mar(filepath, dir_path)
-
- downloaded_updates[build] = {"complete": dir_path}
-
- langs = handle_language(update_file["languages"], filedir)
- downloaded_updates[build]["languages"] = langs
-
- return downloaded_updates
+from tools import get_file_info, uncompress_file_to_dir
def generate_file_name(old_build_id, mar_name_prefix):
@@ -92,16 +16,6 @@ def generate_file_name(old_build_id, mar_name_prefix):
return name
-def generate_lang_file_name(old_build_id, mar_name_prefix, lang):
- name = "%s_%s_from_%s_partial.mar" % (mar_name_prefix, lang, old_build_id)
- return name
-
-
-def add_single_dir(path):
- dir_name = [os.path.join(path, name) for name in os.listdir(path) if
os.path.isdir(os.path.join(path, name))]
- return dir_name[0]
-
-
def main():
workdir = sys.argv[1]
@@ -109,60 +23,53 @@ def main():
updater_path.ensure_dir_exist()
mar_name_prefix = sys.argv[2]
- server_url = sys.argv[3]
- channel = sys.argv[4]
- certificate_path = sys.argv[5]
- certificate_name = sys.argv[6]
- base_url = sys.argv[7]
- platform = sys.argv[8]
- build_id = sys.argv[9]
-
- current_build_path = updater_path.get_current_build_dir()
- mar_dir = updater_path.get_mar_dir()
- temp_dir = updater_path.get_previous_build_dir()
- update_dir = updater_path.get_update_dir()
-
- current_build_path = add_single_dir(current_build_path)
- if sys.platform == "cygwin":
- current_build_path = add_single_dir(current_build_path)
-
- updates = download_mar_for_update_channel_and_platform(server_url,
channel, platform, temp_dir)
-
- data = {"partials": []}
+ channel = sys.argv[3]
+ certificate_path = sys.argv[4]
+ certificate_name = sys.argv[5]
+ base_url = sys.argv[6]
+ product_name = sys.argv[7]
+ version = sys.argv[8]
+ old_archive = sys.argv[9]
+
+ old_uncompress_dir = uncompress_file_to_dir(old_archive,
updater_path.get_previous_build_dir())
+ versionini = os.path.join(old_uncompress_dir, 'program', 'version.ini')
#TODO: Linux, macOS
+ old_build_id = None
+ with open(versionini) as f:
+ for l in f:
+ m = re.fullmatch('buildid=(.*)', l.rstrip())
+ if m:
+ old_build_id = m.group(1)
+ break
+ if old_build_id is None:
+ raise Exception(f'Cannot find buildid in {versionini}')
+
+ new_tar_file_glob = os.path.join(updater_path.get_workdir(),
"installation", product_name, "archive", "install", "*",
f'{product_name}_*_archive*')
+ new_tar_files = glob.glob(new_tar_file_glob)
+ if len(new_tar_files) != 1:
+ raise Exception(f'`{new_tar_file_glob}` does not match exactly one
file')
+ new_tar_file = new_tar_files[0]
+ new_uncompress_dir = uncompress_file_to_dir(new_tar_file,
updater_path.get_current_build_dir())
- for build, update in updates.items():
- file_name = generate_file_name(build, mar_name_prefix)
- mar_file = os.path.join(update_dir, file_name)
- subprocess.call([os.path.join(current_dir_path,
'make_incremental_update.sh'), convert_to_native(mar_file),
- convert_to_native(update["complete"]),
convert_to_native(current_build_path)])
- sign_mar_file(update_dir, certificate_path, certificate_name,
mar_file, mar_name_prefix)
-
- partial_info = {"file": get_file_info(mar_file, base_url), "from":
build, "to": build_id,
- "languages": {}}
-
- # on Windows we don't use language packs
- if sys.platform != "cygwin":
- for lang, lang_info in update["languages"].items():
- lang_name = generate_lang_file_name(build, mar_name_prefix,
lang)
-
- # write the file into the final directory
- lang_mar_file = os.path.join(update_dir, lang_name)
+ update_dir = updater_path.get_update_dir()
- # the directory of the old language file is of the form
- #
workdir/mar/language/en-US/LibreOffice_<version>_<os>_archive_langpack_<lang>/
- language_dir = add_single_dir(os.path.join(mar_dir,
"language", lang))
- subprocess.call(
- [os.path.join(current_dir_path,
'make_incremental_update.sh'), convert_to_native(lang_mar_file),
- convert_to_native(lang_info),
convert_to_native(language_dir)])
- sign_mar_file(update_dir, certificate_path, certificate_name,
lang_mar_file, mar_name_prefix)
+ file_name = generate_file_name(old_build_id, mar_name_prefix)
+ mar_file = os.path.join(update_dir, file_name)
- # add the partial language info
- partial_info["languages"][lang] = get_file_info(lang_mar_file,
base_url)
+ os.putenv('MOZ_PRODUCT_VERSION', version)
+ os.putenv('MAR_CHANNEL_ID', 'LOOnlineUpdater')
+ subprocess.call([os.path.join(workdir,
'UnpackedTarball/onlineupdate/tools/update-packaging/make_incremental_update.sh'),
convert_to_native(mar_file),
+ convert_to_native(old_uncompress_dir),
convert_to_native(new_uncompress_dir)])
- data["partials"].append(partial_info)
+ sign_mar_file(update_dir, certificate_path, certificate_name, mar_file,
mar_name_prefix)
- with open(os.path.join(update_dir, "partial_update_info.json"), "w") as f:
- json.dump(data, f)
+ data = {
+ 'from': old_build_id,
+ 'see also': '',
+ 'update': get_file_info(mar_file, base_url),
+ 'languages': {}
+ };
+ with open(os.path.join(update_dir, channel), "w") as f:
+ json.dump(data, f, indent=4)
if __name__ == '__main__':
diff --git a/bin/update/path.py b/bin/update/path.py
index d91e9e7fba55..5acaafcace53 100644
--- a/bin/update/path.py
+++ b/bin/update/path.py
@@ -8,19 +8,9 @@
#
import os
-import errno
import subprocess
from sys import platform
-def mkdir_p(path):
- try:
- os.makedirs(path)
- except OSError as exc: # Python >2.5
- if exc.errno == errno.EEXIST and os.path.isdir(path):
- pass
- else:
- raise
-
def convert_to_unix(path):
if platform == "cygwin":
return subprocess.check_output(["cygpath", "-u",
path]).decode("utf-8", "strict").rstrip()
diff --git a/config_host.mk.in b/config_host.mk.in
index de08cbcac90e..2ac32c263dc2 100644
--- a/config_host.mk.in
+++ b/config_host.mk.in
@@ -514,7 +514,6 @@ export ONLINEUPDATE_MAR_BASEURL=@ONLINEUPDATE_MAR_BASEURL@
export ONLINEUPDATE_MAR_CERTIFICATEDER=@ONLINEUPDATE_MAR_CERTIFICATEDER@
export ONLINEUPDATE_MAR_CERTIFICATENAME=@ONLINEUPDATE_MAR_CERTIFICATENAME@
export ONLINEUPDATE_MAR_CERTIFICATEPATH=@ONLINEUPDATE_MAR_CERTIFICATEPATH@
-export ONLINEUPDATE_MAR_SERVERURL=@ONLINEUPDATE_MAR_SERVERURL@
export ONLINEUPDATE_MAR_UPLOADURL=@ONLINEUPDATE_MAR_UPLOADURL@
export OOO_JUNIT_JAR=@OOO_JUNIT_JAR@
export OOO_VENDOR=@OOO_VENDOR@
diff --git a/configure.ac b/configure.ac
index 665f954adfc0..18f2900fbd39 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1893,13 +1893,6 @@ libo_FUZZ_ARG_WITH(online-update-mar-certificatepath,
non-functional.)]),
,)
-libo_FUZZ_ARG_WITH(online-update-mar-serverurl,
- AS_HELP_STRING([--with-online-update-mar-serverurl=...],
- [Set the server URL value for --enable-online-update-mar.
- (Can be left off for debug purposes, even if that may render the
feature
- non-functional.)]),
-,)
-
libo_FUZZ_ARG_WITH(online-update-mar-uploadurl,
AS_HELP_STRING([--with-online-update-mar-uploadurl=...],
[Set the upload URL value for --enable-online-update-mar.
@@ -13690,15 +13683,6 @@ else
fi
AC_SUBST(ONLINEUPDATE_MAR_CERTIFICATEPATH)
-AC_MSG_CHECKING([for mar online update serverurl])
-ONLINEUPDATE_MAR_SERVERURL=$with_online_update_mar_serverurl
-if test -n "$ONLINEUPDATE_MAR_SERVERURL"; then
- AC_MSG_RESULT([yes])
-else
- AC_MSG_RESULT([no])
-fi
-AC_SUBST(ONLINEUPDATE_MAR_SERVERURL)
-
AC_MSG_CHECKING([for mar online update uploadurl])
ONLINEUPDATE_MAR_UPLOADURL=$with_online_update_mar_uploadurl
if test -n "$ONLINEUPDATE_MAR_UPLOADURL"; then
diff --git a/distro-configs/Jenkins/LibreOfficeLinuxUpdater.conf
b/distro-configs/Jenkins/LibreOfficeLinuxUpdater.conf
index e6b7448ccd43..9ff8d98add4e 100644
--- a/distro-configs/Jenkins/LibreOfficeLinuxUpdater.conf
+++ b/distro-configs/Jenkins/LibreOfficeLinuxUpdater.conf
@@ -36,7 +36,6 @@
--with-online-update-mar-certificateder=TODO
--with-online-update-mar-certificatename=TODO
--with-online-update-mar-certificatepath=TODO
---with-online-update-mar-serverurl=TODO
--with-online-update-mar-uploadurl=TODO
--with-jdk-home=/etc/alternatives/java_sdk_17
--enable-odk
diff --git a/external/onlineupdate/UnpackedTarball_onlineupdate.mk
b/external/onlineupdate/UnpackedTarball_onlineupdate.mk
index d27a191d50ee..166ede0a6b9b 100644
--- a/external/onlineupdate/UnpackedTarball_onlineupdate.mk
+++ b/external/onlineupdate/UnpackedTarball_onlineupdate.mk
@@ -18,6 +18,12 @@ $(eval $(call gb_UnpackedTarball_add_patches,onlineupdate, \
external/onlineupdate/lo.patch \
))
+ifeq ($(OS),WNT)
+$(eval $(call gb_UnpackedTarball_add_patches,onlineupdate, \
+ external/onlineupdate/cygpath.patch \
+))
+endif
+
# The update maintenance service that is used on Windows has a couple of
checks that files in the
# to-be-updated installation set are signed, which would fail for
--disable-windows-build-signing;
# so, as a HACK for debugging purposes, silence those problematic checks for
--enable-dbgutil:
diff --git a/external/onlineupdate/cygpath.patch
b/external/onlineupdate/cygpath.patch
new file mode 100644
index 000000000000..e5ca1d0bcf8c
--- /dev/null
+++ b/external/onlineupdate/cygpath.patch
@@ -0,0 +1,11 @@
+--- tools/update-packaging/make_incremental_update.sh
++++ tools/update-packaging/make_incremental_update.sh
+@@ -110,7 +110,7 @@
+ # Remove the /
+ newdir=$(echo "$newdir" | sed -e 's:\/$::')
+ fi
+-workdir="$(mktemp -d)"
++workdir=$(cygpath -m "$(mktemp -d)")
+ updatemanifestv3="$workdir/updatev3.manifest"
+ archivefiles="updatev3.manifest"
+
diff --git a/external/onlineupdate/lo.patch b/external/onlineupdate/lo.patch
index 870857b7ba46..6935afef1558 100644
--- a/external/onlineupdate/lo.patch
+++ b/external/onlineupdate/lo.patch
@@ -289,3 +289,13 @@
mv -f "$workdir/output.mar" "$archive"
# cleanup
+--- tools/update-packaging/make_incremental_update.sh
++++ tools/update-packaging/make_incremental_update.sh
+@@ -135,7 +135,6 @@
+ if [ ! -f "precomplete" ]; then
+ if [ ! -f "Contents/Resources/precomplete" ]; then
+ notice "precomplete file is missing!"
+- exit 1
+ fi
+ fi
+
