Hi,
On 2022-01-18 13:40:40 -0800, Andres Freund wrote:
> Maybe we really should do at least the most simplistic caching for initdbs, by
> doing one initdb as part of the creation of temp_install. Then Cluster::init
> would need logic to only use that if $params{extra} is empty.
I hacked this together. And the wins are bigger than I thought. On my
workstation, with plenty cpu and storage bandwidth, according to
/usr/bin/time check-world NO_TEMP_INSTALL=1
things go from
321.56user 74.00system 2:26.22elapsed 270%CPU (0avgtext+0avgdata
93768maxresident)k
24inputs+32781336outputs (2254major+8717121minor)pagefaults 0swaps
to
86.62user 57.10system 1:57.83elapsed 121%CPU (0avgtext+0avgdata
93752maxresident)k
8inputs+32683408outputs (1360major+6672618minor)pagefaults 0swaps
The difference in elapsed and system time is pretty good, but the user time
difference is quite staggering.
This doesn't yet actually address the case of the basebackup tests, because
that specifies a "non-default" option, preventing the use of the template
initdb. But the effects are already big enough that I thought it's worth
sharing.
On CI for windows this reduces the time for the subscription tests from 03:24,
to 2:39. There's some run-to-run variation, but it's a pretty clear signal...
Greetings,
Andres Freund
>From a0818b6c8d03f152baa5df231b27aa7b8a7fde45 Mon Sep 17 00:00:00 2001
From: Andres Freund <and...@anarazel.de>
Date: Tue, 18 Jan 2022 15:25:27 -0800
Subject: [PATCH v2] hack: use "template" initdb in tap tests.
---
src/test/perl/PostgreSQL/Test/Cluster.pm | 31 +++++++++++++-
src/test/regress/pg_regress.c | 45 ++++++++++++++------
src/Makefile.global.in | 52 +++++++++++++-----------
src/tools/msvc/Install.pm | 4 ++
src/tools/msvc/vcregress.pl | 1 +
5 files changed, 94 insertions(+), 39 deletions(-)
diff --git a/src/test/perl/PostgreSQL/Test/Cluster.pm b/src/test/perl/PostgreSQL/Test/Cluster.pm
index 7af0f8db139..7e235c90d8c 100644
--- a/src/test/perl/PostgreSQL/Test/Cluster.pm
+++ b/src/test/perl/PostgreSQL/Test/Cluster.pm
@@ -430,8 +430,35 @@ sub init
mkdir $self->backup_dir;
mkdir $self->archive_dir;
- PostgreSQL::Test::Utils::system_or_bail('initdb', '-D', $pgdata, '-A', 'trust', '-N',
- @{ $params{extra} });
+ if (defined $params{extra} or !defined $ENV{INITDB_TEMPLATE})
+ {
+ diag("*not* using initdb template");
+ PostgreSQL::Test::Utils::system_or_bail('initdb', '-D', $pgdata, '-A', 'trust', '-N',
+ @{ $params{extra} });
+ }
+ else
+ {
+ my $old_umask;
+
+ diag("using initdb template");
+
+ if (!$PostgreSQL::Test::Utils::windows_os)
+ {
+ $old_umask = umask;
+ umask 0077;
+ }
+
+ PostgreSQL::Test::RecursiveCopy::copypath(
+ $ENV{INITDB_TEMPLATE},
+ $pgdata,
+ );
+
+ if (!$PostgreSQL::Test::Utils::windows_os)
+ {
+ umask $old_umask;
+ }
+ }
+
PostgreSQL::Test::Utils::system_or_bail($ENV{PG_REGRESS}, '--config-auth', $pgdata,
@{ $params{auth_extra} });
diff --git a/src/test/regress/pg_regress.c b/src/test/regress/pg_regress.c
index e6f71c7582e..07f1beae6c6 100644
--- a/src/test/regress/pg_regress.c
+++ b/src/test/regress/pg_regress.c
@@ -2232,6 +2232,7 @@ regression_main(int argc, char *argv[],
FILE *pg_conf;
const char *env_wait;
int wait_seconds;
+ const char* initdb_template_dir;
/*
* Prepare the temp instance
@@ -2258,20 +2259,38 @@ regression_main(int argc, char *argv[],
if (!directory_exists(buf))
make_directory(buf);
- /* initdb */
- header(_("initializing database system"));
- snprintf(buf, sizeof(buf),
- "\"%s%sinitdb\" -D \"%s/data\" --no-clean --no-sync%s%s > \"%s/log/initdb.log\" 2>&1",
- bindir ? bindir : "",
- bindir ? "/" : "",
- temp_instance,
- debug ? " --debug" : "",
- nolocale ? " --no-locale" : "",
- outputdir);
- if (system(buf))
+ /* create data directory, either from a template, or by running initdb */
+ initdb_template_dir = getenv("INITDB_TEMPLATE");
+ if (initdb_template_dir == NULL)
{
- fprintf(stderr, _("\n%s: initdb failed\nExamine %s/log/initdb.log for the reason.\nCommand was: %s\n"), progname, outputdir, buf);
- exit(2);
+ header(_("initializing database system"));
+ snprintf(buf, sizeof(buf),
+ "\"%s%sinitdb\" -D \"%s/data\" --no-clean --no-sync%s%s > \"%s/log/initdb.log\" 2>&1",
+ bindir ? bindir : "",
+ bindir ? "/" : "",
+ temp_instance,
+ debug ? " --debug" : "",
+ nolocale ? " --no-locale" : "",
+ outputdir);
+ if (system(buf))
+ {
+ fprintf(stderr, _("\n%s: initdb failed\nExamine %s/log/initdb.log for the reason.\nCommand was: %s\n"), progname, outputdir, buf);
+ exit(2);
+ }
+ }
+ else
+ {
+ /* FIXME: this only works on windows when there's a GNU cp in PATH */
+ header(_("initializing database system from template"));
+ snprintf(buf, sizeof(buf),
+ "cp -a \"%s\" \"%s/data\"",
+ initdb_template_dir,
+ temp_instance);
+ if (system(buf))
+ {
+ fprintf(stderr, _("\n%s: copying of initdb template failed\n"), progname);
+ exit(2);
+ }
}
/*
diff --git a/src/Makefile.global.in b/src/Makefile.global.in
index 05c54b27def..6d07e76ad68 100644
--- a/src/Makefile.global.in
+++ b/src/Makefile.global.in
@@ -401,30 +401,6 @@ check: temp-install
.PHONY: temp-install
-temp-install: | submake-generated-headers
-ifndef NO_TEMP_INSTALL
-ifneq ($(abs_top_builddir),)
-ifeq ($(MAKELEVEL),0)
- rm -rf '$(abs_top_builddir)'/tmp_install
- $(MKDIR_P) '$(abs_top_builddir)'/tmp_install/log
- $(MAKE) -C '$(top_builddir)' DESTDIR='$(abs_top_builddir)'/tmp_install install >'$(abs_top_builddir)'/tmp_install/log/install.log 2>&1
- $(MAKE) -j1 $(if $(CHECKPREP_TOP),-C $(CHECKPREP_TOP),) checkprep >>'$(abs_top_builddir)'/tmp_install/log/install.log 2>&1
-endif
-endif
-endif
-
-# Tasks to run serially at the end of temp-install. Some EXTRA_INSTALL
-# entries appear more than once in the tree, and parallel installs of the same
-# file can fail with EEXIST.
-checkprep:
- $(if $(EXTRA_INSTALL),for extra in $(EXTRA_INSTALL); do $(MAKE) -C '$(top_builddir)'/$$extra DESTDIR='$(abs_top_builddir)'/tmp_install install || exit; done)
-
-PROVE = @PROVE@
-# There are common routines in src/test/perl, and some test suites have
-# extra perl modules in their own directory.
-PG_PROVE_FLAGS = -I $(top_srcdir)/src/test/perl/ -I $(srcdir)
-# User-supplied prove flags such as --verbose can be provided in PROVE_FLAGS.
-PROVE_FLAGS =
# prepend to path if already set, else just set it
define add_to_path
@@ -441,8 +417,36 @@ ld_library_path_var = LD_LIBRARY_PATH
with_temp_install = \
PATH="$(abs_top_builddir)/tmp_install$(bindir):$(CURDIR):$$PATH" \
$(call add_to_path,$(strip $(ld_library_path_var)),$(abs_top_builddir)/tmp_install$(libdir)) \
+ INITDB_TEMPLATE='$(abs_top_builddir)'/tmp_install/initdb-template \
$(with_temp_install_extra)
+temp-install: | submake-generated-headers
+ifndef NO_TEMP_INSTALL
+ifneq ($(abs_top_builddir),)
+ifeq ($(MAKELEVEL),0)
+ rm -rf '$(abs_top_builddir)'/tmp_install
+ $(MKDIR_P) '$(abs_top_builddir)'/tmp_install/log
+ $(MAKE) -C '$(top_builddir)' DESTDIR='$(abs_top_builddir)'/tmp_install install >'$(abs_top_builddir)'/tmp_install/log/install.log 2>&1
+ $(MAKE) -j1 $(if $(CHECKPREP_TOP),-C $(CHECKPREP_TOP),) checkprep >>'$(abs_top_builddir)'/tmp_install/log/install.log 2>&1
+
+ $(with_temp_install) initdb -A trust -N --no-instructions '$(abs_top_builddir)'/tmp_install/initdb-template >>'$(abs_top_builddir)'/tmp_install/log/initdb-template.log 2>&1
+endif
+endif
+endif
+
+# Tasks to run serially at the end of temp-install. Some EXTRA_INSTALL
+# entries appear more than once in the tree, and parallel installs of the same
+# file can fail with EEXIST.
+checkprep:
+ $(if $(EXTRA_INSTALL),for extra in $(EXTRA_INSTALL); do $(MAKE) -C '$(top_builddir)'/$$extra DESTDIR='$(abs_top_builddir)'/tmp_install install || exit; done)
+
+PROVE = @PROVE@
+# There are common routines in src/test/perl, and some test suites have
+# extra perl modules in their own directory.
+PG_PROVE_FLAGS = -I $(top_srcdir)/src/test/perl/ -I $(srcdir)
+# User-supplied prove flags such as --verbose can be provided in PROVE_FLAGS.
+PROVE_FLAGS =
+
ifeq ($(enable_tap_tests),yes)
ifndef PGXS
diff --git a/src/tools/msvc/Install.pm b/src/tools/msvc/Install.pm
index 8de79c618cb..cef069c6162 100644
--- a/src/tools/msvc/Install.pm
+++ b/src/tools/msvc/Install.pm
@@ -14,6 +14,7 @@ use Carp;
use File::Basename;
use File::Copy;
use File::Find ();
+use File::Path qw(rmtree);
use Exporter;
our (@ISA, @EXPORT_OK);
@@ -183,6 +184,9 @@ sub Install
GenerateNLSFiles($target, $config->{nls}, $majorver) if ($config->{nls});
+ rmtree("$target/initdb_template");
+ system("$target/bin/initdb -D $target/initdb_template -A trust -N") == 0 || die "template initdb failed";
+
print "Installation complete.\n";
return;
}
diff --git a/src/tools/msvc/vcregress.pl b/src/tools/msvc/vcregress.pl
index 6dcd742fa6c..d05fc364c55 100644
--- a/src/tools/msvc/vcregress.pl
+++ b/src/tools/msvc/vcregress.pl
@@ -73,6 +73,7 @@ $ENV{with_icu} = $config->{icu} ? 'yes' : 'no';
$ENV{with_gssapi} = $config->{gss} ? 'yes' : 'no';
$ENV{with_krb_srvnam} = $config->{krb_srvnam} || 'postgres';
$ENV{with_readline} = 'no';
+$ENV{INITDB_TEMPLATE} = "$tmp_installdir/initdb_template";
$ENV{PATH} = "$topdir/$Config/libpq;$ENV{PATH}";
--
2.34.0
