bin/pack-debug | 20 +++++++++++++-------
config_host.mk.in | 1 +
configure.ac | 10 ++++++++++
instsetoo_native/CustomTarget_install.mk | 1 +
solenv/bin/modules/installer/download.pm | 2 +-
solenv/bin/modules/installer/epmfile.pm | 6 +++---
solenv/bin/modules/installer/globals.pm | 2 ++
solenv/bin/modules/installer/helppack.pm | 2 +-
solenv/bin/modules/installer/languagepack.pm | 2 +-
solenv/bin/modules/installer/parameter.pm | 2 ++
solenv/bin/modules/installer/simplepackage.pm | 2 +-
solenv/bin/modules/installer/worker.pm | 2 +-
sysui/desktop/freedesktop/freedesktop-menus.spec | 2 +-
13 files changed, 38 insertions(+), 16 deletions(-)
New commits:
commit 07ef9cb7c2f80c975ebe6276259c5b2ff71b3488
Author: Andras Timar <andras.ti...@collabora.com>
AuthorDate: Sun Jan 23 12:17:50 2022 +0100
Commit: Andras Timar <andras.ti...@collabora.com>
CommitDate: Mon Jan 24 11:08:26 2022 +0100
[cp] set w1T.xzdio rpm compression in bin/pack-debug, too
Change-Id: I6d808c9ba9b5353dd74e41fced2f134f1ad70501
diff --git a/bin/pack-debug b/bin/pack-debug
index e7a6d31bb938..312f59e6f81f 100755
--- a/bin/pack-debug
+++ b/bin/pack-debug
@@ -106,7 +106,7 @@ cat $DEBUGSRC | awk '
# debug build source package
- rpmbuild -bb --define "_unpackaged_files_terminate_build 0"
${DEBUGSRC}-debugsource --target $build_cpu --buildroot=$buildroot
+ rpmbuild -bb --define "_unpackaged_files_terminate_build 0" --define
"_binary_payload w1T.xzdio" ${DEBUGSRC}-debugsource --target $build_cpu
--buildroot=$buildroot
###################################################################
echo create rpm debug info packages by processing logged spec files
@@ -156,7 +156,7 @@ do
# stripped package
- rpmbuild -bb --define "_unpackaged_files_terminate_build 0" $i
--target $build_cpu --buildroot=$buildroot
+ rpmbuild -bb --define "_unpackaged_files_terminate_build 0" --define
"_binary_payload w1T.xzdio" $i --target $build_cpu --buildroot=$buildroot
rm -rf $buildroot
mv $buildroot.copy $buildroot
mkdir -p $topdir/RPMS/BUILD $topdir/RPMS/RPMS
@@ -174,7 +174,7 @@ do
# create debug info package
- rpmbuild -bb --define "_unpackaged_files_terminate_build 0"
${i}-debuginfo --target $build_cpu --buildroot=$buildroot
+ rpmbuild -bb --define "_unpackaged_files_terminate_build 0" --define
"_binary_payload w1T.xzdio" ${i}-debuginfo --target $build_cpu
--buildroot=$buildroot
fi
done
commit 6e5f074366a3cd944cce2b20a8eb2d67f2b49e7d
Author: Andras Timar <andras.ti...@collabora.com>
AuthorDate: Sat Jan 22 20:35:00 2022 +0100
Commit: Andras Timar <andras.ti...@collabora.com>
CommitDate: Mon Jan 24 11:07:54 2022 +0100
[cp] use pigz (parallellized compression) instead of gzip, if we can
Change-Id: I3f97702a6febd3ac0cbbd16a0932d74ea6659bf3
diff --git a/bin/pack-debug b/bin/pack-debug
index 67c4cd50150d..e7a6d31bb938 100755
--- a/bin/pack-debug
+++ b/bin/pack-debug
@@ -13,6 +13,12 @@
export BUILD_PATH=$PWD
export BUILD_LOG=$BUILD_PATH/pack-debug.log
+if type -P pigz &>/dev/null; then
+ GZIP=pigz
+else
+ GZIP=gzip
+fi
+
if [ $# -gt 0 ]
then
for i in "$@"
@@ -180,8 +186,8 @@ TARGET_RPM=$(ls *_download/*.tar.gz)
TARGET_DEBUG=$(echo $TARGET_RPM | sed 's/.tar.gz$/-debug.tar.gz/')
SOURCE_RPM=$(find *_rpm -type f | grep -v debug)
SOURCE_DEBUG=$(find *_rpm -type f | grep -E '(debug|readme|README)')
-tar c $SOURCE_RPM | gzip >$TARGET_RPM
-tar c $SOURCE_DEBUG | gzip >$TARGET_DEBUG
+tar c $SOURCE_RPM | $GZIP >$TARGET_RPM
+tar c $SOURCE_DEBUG | $GZIP >$TARGET_DEBUG
cd $BUILD_PATH
rm -rf $topdir
}
@@ -323,8 +329,8 @@ TARGET_DEB=$(ls *_download/*.tar.gz)
TARGET_DEBUG=$(echo $TARGET_DEB | sed 's/.tar.gz$/-debug.tar.gz/')
SOURCE_DEB=$(find *_deb -type f | grep -v debug)
SOURCE_DEBUG=$(find *_deb -type f | grep -E '(debug|readme|README)')
-tar c $SOURCE_DEB | gzip >$TARGET_DEB
-tar c $SOURCE_DEBUG | gzip >$TARGET_DEBUG
+tar c $SOURCE_DEB | $GZIP >$TARGET_DEB
+tar c $SOURCE_DEBUG | $GZIP >$TARGET_DEBUG
cd $BUILD_PATH
rm -rf DEBS
commit d3ff5c4b9980a20a2da6cb93b986b69d0554f05a
Author: Andras Timar <andras.ti...@collabora.com>
AuthorDate: Sat Jan 22 22:00:56 2022 +0100
Commit: Andras Timar <andras.ti...@collabora.com>
CommitDate: Mon Jan 24 11:07:19 2022 +0100
change rpm compression algorithm to multi threaded xz
Our baseline is CentOS 7, it's safe to switch to xz that is supported there.
Here is my measurement on rpm package build time and package size on a
48 core machine, however 48 cores were never used (8-10 max). Compared to
reference w9.gzdio which was used before the patch.
Compression Build Package
method time % size %
w9.gzdio 100% 100%
w7T.xzdio 69% 66%
w6T.xzdio 62% 67%
w5T.xzdio 55% 67%
w4T.xzdio 41% 70%
w3T.xzdio 31% 73%
w2T.xzdio 26% 75%
w1T.xzdio 24% 77%
Conclusion: we'd like to optimize rather for the build time, therefore
w1T.xzdio was picked. The package size was still smaller than the reference
size, and the build time was 4x faster (in this measurement 18m16 -> 4m26).
That makes a difference.
Change-Id: Iabb532628d38a2d5091fa41fd08bfee69007d0ca
diff --git a/solenv/bin/modules/installer/epmfile.pm
b/solenv/bin/modules/installer/epmfile.pm
index d699e46e03ed..7b5a5da77198 100644
--- a/solenv/bin/modules/installer/epmfile.pm
+++ b/solenv/bin/modules/installer/epmfile.pm
@@ -1193,7 +1193,7 @@ sub set_autoprovreq_in_specfile
close (FINDREQUIRES);
}
- $autoreqprovline .= "%define _binary_filedigest_algorithm 1\n%define
_binary_payload w9.gzdio\n";
+ $autoreqprovline .= "%define _binary_filedigest_algorithm 1\n%define
_binary_payload w1T.xzdio\n";
for ( my $i = 0; $i <= $#{$changefile}; $i++ )
{
diff --git a/sysui/desktop/freedesktop/freedesktop-menus.spec
b/sysui/desktop/freedesktop/freedesktop-menus.spec
index ea618ca4ab77..e49833814b71 100644
--- a/sysui/desktop/freedesktop/freedesktop-menus.spec
+++ b/sysui/desktop/freedesktop/freedesktop-menus.spec
@@ -33,7 +33,7 @@ Conflicts: %pkgprefix-mandriva-menus
BuildArch: noarch
AutoReqProv: no
%define _binary_filedigest_algorithm 1
-%define _binary_payload w9.gzdio
+%define _binary_payload w1T.xzdio
%define gnome_dir /usr
%define gnome_mime_theme hicolor
commit 5fa243cd0104530843471053061879be3a343cd1
Author: Thorsten Behrens <thorsten.behr...@allotropia.de>
AuthorDate: Sat Jan 15 16:24:33 2022 +0100
Commit: Andras Timar <andras.ti...@collabora.com>
CommitDate: Mon Jan 24 11:06:55 2022 +0100
Make installer compression tool configurable
So we can use pigz or other parallelizable tools if available. Shaves
off noticeable build time when packaging install sets.
- figure out if pigz is available (fallback to gzip otherwise)
- pass compression tool down into make_installer
- and handle as one of many global options there
Change-Id: Ia9d1ea27a9f990874238b6f0be3e1fd30a662ec6
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/128469
Tested-by: Jenkins
Reviewed-by: Thorsten Behrens <thorsten.behr...@allotropia.de>
diff --git a/config_host.mk.in b/config_host.mk.in
index 88ee44b16548..c1e81a6ee105 100644
--- a/config_host.mk.in
+++ b/config_host.mk.in
@@ -81,6 +81,7 @@ export COMPILER_PLUGINS_CXXFLAGS=@COMPILER_PLUGINS_CXXFLAGS@
export COMPILER_PLUGINS_CXX_LINKFLAGS=@COMPILER_PLUGINS_CXX_LINKFLAGS@
export COMPILER_PLUGINS_DEBUG=@COMPILER_PLUGINS_DEBUG@
export COMPILER_PLUGINS_TOOLING_ARGS=@COMPILER_PLUGINS_TOOLING_ARGS@
+export COMPRESSIONTOOL=@COMPRESSIONTOOL@
export COM_IS_CLANG=@COM_IS_CLANG@
export CPPUNIT_CFLAGS=$(gb_SPACE)@CPPUNIT_CFLAGS@
export CPPUNIT_LIBS=$(gb_SPACE)@CPPUNIT_LIBS@
diff --git a/configure.ac b/configure.ac
index 7145e2453487..c9dfd480e767 100644
--- a/configure.ac
+++ b/configure.ac
@@ -2537,6 +2537,16 @@ if test -z "$BASH"; then
fi
AC_SUBST(BASH)
+# prefer parallel comression tools, if available
+AC_PATH_PROG(COMPRESSIONTOOL, pigz)
+if test -z "$COMPRESSIONTOOL"; then
+ AC_PATH_PROG(COMPRESSIONTOOL, gzip)
+ if test -z "$COMPRESSIONTOOL"; then
+ AC_MSG_ERROR([gzip not found in \$PATH])
+ fi
+fi
+AC_SUBST(COMPRESSIONTOOL)
+
AC_MSG_CHECKING([for GNU or BSD tar])
for a in $GNUTAR gtar gnutar bsdtar tar /usr/sfw/bin/gtar; do
$a --version 2> /dev/null | egrep "GNU|bsdtar" 2>&1 > /dev/null
diff --git a/instsetoo_native/CustomTarget_install.mk
b/instsetoo_native/CustomTarget_install.mk
index f014f192921c..f215fc3c974b 100644
--- a/instsetoo_native/CustomTarget_install.mk
+++ b/instsetoo_native/CustomTarget_install.mk
@@ -87,6 +87,7 @@ $(foreach pkgformat,$(5),\
-l $(subst $(WHITESPACE),$(COMMA),$(strip $(2))) \
-p $(PRODUCTNAME_no_spaces)$(3) \
-u $(instsetoo_OUT) \
+ -packer $(COMPRESSIONTOOL) \
-buildid $(if $(filter deb0
rpm0,$(pkgformat)$(LIBO_VERSION_PATCH)),1,$(LIBO_VERSION_PATCH)) \
$(if $(filter WNT,$(OS)), \
-msitemplate $(dir $@)msi_templates \
diff --git a/solenv/bin/modules/installer/download.pm
b/solenv/bin/modules/installer/download.pm
index c1776053ec1e..60cb9771436c 100644
--- a/solenv/bin/modules/installer/download.pm
+++ b/solenv/bin/modules/installer/download.pm
@@ -516,7 +516,7 @@ sub create_tar_gz_file_from_directory
unlink("$installdir/install");
}
- my $systemcall = "cd $changedir; $fakerootstring tar -cf - $packdir | gzip
> $targzname";
+ my $systemcall = "cd $changedir; $fakerootstring tar -cf - $packdir |
$installer::globals::packertool > $targzname";
my $returnvalue = system($systemcall);
diff --git a/solenv/bin/modules/installer/epmfile.pm
b/solenv/bin/modules/installer/epmfile.pm
index ed0066c65f2e..d699e46e03ed 100644
--- a/solenv/bin/modules/installer/epmfile.pm
+++ b/solenv/bin/modules/installer/epmfile.pm
@@ -1839,7 +1839,7 @@ sub create_packages_without_epm
# Solaris: pkgmk -o -f solaris-2.8-sparc/SUNWso8m34.prototype -d
solaris-2.8-sparc
# Solaris: pkgtrans solaris-2.8-sparc SUNWso8m34.pkg SUNWso8m34
- # Solaris: tar -cf - SUNWso8m34 | gzip > SUNWso8m34.tar.gz
+ # Solaris: tar -cf - SUNWso8m34 | $installer::globals::packertool >
SUNWso8m34.tar.gz
if ( $installer::globals::issolarispkgbuild )
{
@@ -1983,7 +1983,7 @@ sub create_packages_without_epm
#########################
# my $targzname = $packagename . ".tar.gz";
- # $systemcall = "cd $destinationdir; tar -cf - $packagename | gzip >
$targzname";
+ # $systemcall = "cd $destinationdir; tar -cf - $packagename |
$installer::globals::packertool > $targzname";
# print "... $systemcall ...\n";
# $returnvalue = system($systemcall);
diff --git a/solenv/bin/modules/installer/globals.pm
b/solenv/bin/modules/installer/globals.pm
index e57627bb441b..1938942b0a82 100644
--- a/solenv/bin/modules/installer/globals.pm
+++ b/solenv/bin/modules/installer/globals.pm
@@ -129,6 +129,8 @@ BEGIN
$strip = 0;
+ $packertool = "gzip"; # the default package compression tool for
*NIX
+
$logfilename = "logfile.log"; # the default logfile name for global
errors
@logfileinfo = ();
@errorlogfileinfo = ();
diff --git a/solenv/bin/modules/installer/helppack.pm
b/solenv/bin/modules/installer/helppack.pm
index 726157be2e27..456e91ecfdb0 100644
--- a/solenv/bin/modules/installer/helppack.pm
+++ b/solenv/bin/modules/installer/helppack.pm
@@ -130,7 +130,7 @@ sub create_tar_gz_file
$packagename =~ s/\.rpm\s*$//;
my $targzname = $packagename . ".tar.gz";
- my $systemcall = "cd $installdir; tar -cf - $packagestring | gzip >
$targzname";
+ my $systemcall = "cd $installdir; tar -cf - $packagestring |
$installer::globals::packertool > $targzname";
installer::logger::print_message( "... $systemcall ...\n" );
my $returnvalue = system($systemcall);
diff --git a/solenv/bin/modules/installer/languagepack.pm
b/solenv/bin/modules/installer/languagepack.pm
index a39769144de6..14a870866128 100644
--- a/solenv/bin/modules/installer/languagepack.pm
+++ b/solenv/bin/modules/installer/languagepack.pm
@@ -128,7 +128,7 @@ sub create_tar_gz_file
$packagename =~ s/\.rpm\s*$//;
my $targzname = $packagename . ".tar.gz";
- $systemcall = "cd $installdir; tar -cf - $packagestring | gzip >
$targzname";
+ $systemcall = "cd $installdir; tar -cf - $packagestring |
$installer::globals::packertool > $targzname";
installer::logger::print_message( "... $systemcall ...\n" );
my $returnvalue = system($systemcall);
diff --git a/solenv/bin/modules/installer/parameter.pm
b/solenv/bin/modules/installer/parameter.pm
index 9d8efbb558a6..657b543ee702 100644
--- a/solenv/bin/modules/installer/parameter.pm
+++ b/solenv/bin/modules/installer/parameter.pm
@@ -56,6 +56,7 @@ The following parameter are needed:
-languagepack : do create a languagepack, no product pack (optional)
-helppack : do create a helppack, no product pack (optional)
-strip: Stripping files (Unix only)
+-packer: Path and parameters for tarball packaging tool (default: gzip (Unix
only))
-log : Logging all available information (optional)
Examples for Windows:
@@ -127,6 +128,7 @@ sub getparameter
elsif ($param eq "-helppack") { $installer::globals::helppack = 1;}
elsif ($param eq "-debian") { $installer::globals::debian = 1; }
elsif ($param eq "-strip") { $installer::globals::strip = 1; }
+ elsif ($param eq "-packer") { $installer::globals::packertool =
shift(@ARGV); }
elsif ($param eq "-destdir") # new parameter for simple installer
{
$installer::globals::rootpath ne "" && die "must set destdir
before -i or -simple";
diff --git a/solenv/bin/modules/installer/simplepackage.pm
b/solenv/bin/modules/installer/simplepackage.pm
index 15cd195d8622..4c9be245b52b 100644
--- a/solenv/bin/modules/installer/simplepackage.pm
+++ b/solenv/bin/modules/installer/simplepackage.pm
@@ -474,7 +474,7 @@ sub create_package
$fakerootstring = "fakeroot";
}
- $systemcall = "cd $tempdir; $fakerootstring tar -cf - . | gzip >
$archive";
+ $systemcall = "cd $tempdir; $fakerootstring tar -cf - . |
$installer::globals::packertool > $archive";
}
if ( $makesystemcall )
diff --git a/solenv/bin/modules/installer/worker.pm
b/solenv/bin/modules/installer/worker.pm
index d002595c69fe..13c576cd54a9 100644
--- a/solenv/bin/modules/installer/worker.pm
+++ b/solenv/bin/modules/installer/worker.pm
@@ -913,7 +913,7 @@ sub collectpackagemaps
# Create a tar gz file with all package maps
my $tarfilename = $subdirname . ".tar";
my $targzname = $tarfilename . ".gz";
- $systemcall = "cd $pkgmapdir; tar -cf - $subdirname | gzip > $targzname";
+ $systemcall = "cd $pkgmapdir; tar -cf - $subdirname |
$installer::globals::packertool > $targzname";
installer::systemactions::make_systemcall($systemcall);
installer::systemactions::remove_complete_directory($pkgmapsubdir, 1);
}
