On Sat, Sep 10, 2022 at 03:05:42PM -0500, Justin Pryzby wrote:
> On Sun, Aug 28, 2022 at 02:28:02PM -0700, Andres Freund wrote:
> > On 2022-08-28 12:10:29 -0500, Justin Pryzby wrote:
> > > On Sun, Aug 28, 2022 at 09:07:52AM -0700, Andres Freund wrote:
> > > > > --- /dev/null
> > > > > +++ b/src/tools/ci/windows-compiler-warnings
> > > >
> > > > Wouldn't that be doable as something like
> > > > sh -c 'if test -s file; then cat file;exit 1; fi"
> > > > inside .cirrus.yml?
> > >
> > > I had written it inline in a couple ways, like
> > > - sh -exc 'f=msbuild.warn.log; if [ -s "$f" ]; then cat "$f"; exit 1;
> > > else exit 0; fi'
> > >
> > > but then separated it out as you suggested in
> > > 20220227010908.vz2a7dmfzgwg7...@alap3.anarazel.de
> > >
> > > after I complained about cmd.exe requiring escaping for && and ||
> > > That makes writing any shell script a bit perilous and a separate script
> > > seems better.
> >
> > I remember that I suggested it - but note that the way I wrote above doesn't
> > have anything needing escaping.
>
> It doesn't require it, but that still gives the impression that it's
> normally possible to write one-liner shell scripts there, which is
> misleading/wrong, and the reason why I took your suggestion to use a
> separate script file.
>
> > Anyway, what do you think of the multiline split I suggested?
>
> Done, and sorted.
Rewrote this and rebased some of the other stuff on top of the meson
commit, for which I also include some new patches.
>From 3825287f88ad03ed3335ab6c7ea1ca2b95d88356 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryz...@telsasoft.com>
Date: Fri, 30 Sep 2022 08:56:07 -0500
Subject: [PATCH 1/8] meson: PROVE is not required
It ought to be possible to build the application without running tests,
or without running TAP tests. And it's may be needed to support
cygwin/buildfarm, where TAP tests consistently fail..
See: 20221021034040.gt16...@telsasoft.com
---
meson.build | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/meson.build b/meson.build
index bfacbdc0af6..ce2f223a409 100644
--- a/meson.build
+++ b/meson.build
@@ -323,7 +323,7 @@ python = find_program(get_option('PYTHON'), required: true, native: true)
flex = find_program(get_option('FLEX'), native: true, version: '>= 2.5.35')
bison = find_program(get_option('BISON'), native: true, version: '>= 2.3')
sed = find_program(get_option('SED'), 'sed', native: true)
-prove = find_program(get_option('PROVE'), native: true)
+prove = find_program(get_option('PROVE'), native: true, required: false)
tar = find_program(get_option('TAR'), native: true)
gzip = find_program(get_option('GZIP'), native: true)
program_lz4 = find_program(get_option('LZ4'), native: true, required: false)
--
2.25.1
>From a994127b0361e2bc3ff59919176ec5f81faa426e Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryz...@telsasoft.com>
Date: Fri, 30 Sep 2022 13:39:43 -0500
Subject: [PATCH 2/8] meson: other fixes for cygwin
XXX: what about HAVE_BUGGY_STRTOF ?
See: 20221021034040.gt16...@telsasoft.com
---
meson.build | 8 ++++++--
src/port/meson.build | 4 ++++
src/test/regress/meson.build | 2 +-
3 files changed, 11 insertions(+), 3 deletions(-)
diff --git a/meson.build b/meson.build
index ce2f223a409..ed24370672a 100644
--- a/meson.build
+++ b/meson.build
@@ -211,6 +211,10 @@ if host_system == 'aix'
elif host_system == 'cygwin'
cppflags += '-D_GNU_SOURCE'
+ dlsuffix = '.dll'
+ mod_link_args_fmt = ['@0@']
+ mod_link_with_name = 'lib@0@.exe.a'
+ mod_link_with_dir = 'libdir'
elif host_system == 'darwin'
dlsuffix = '.dylib'
@@ -2301,8 +2305,8 @@ gnugetopt_dep = cc.find_library('gnugetopt', required: false)
# (i.e., allow '-' as a flag character), so use our version on those platforms
# - We want to use system's getopt_long() only if the system provides struct
# option
-always_replace_getopt = host_system in ['windows', 'openbsd', 'solaris']
-always_replace_getopt_long = host_system == 'windows' or not cdata.has('HAVE_STRUCT_OPTION')
+always_replace_getopt = host_system in ['windows', 'cygwin', 'openbsd', 'solaris']
+always_replace_getopt_long = host_system in ['windows', 'cygwin'] or not cdata.has('HAVE_STRUCT_OPTION')
# Required on BSDs
execinfo_dep = cc.find_library('execinfo', required: false)
diff --git a/src/port/meson.build b/src/port/meson.build
index c2222696f1b..0ba83cc7930 100644
--- a/src/port/meson.build
+++ b/src/port/meson.build
@@ -40,6 +40,10 @@ if host_system == 'windows'
'win32setlocale.c',
'win32stat.c',
)
+elif host_system == 'cygwin'
+ pgport_sources += files(
+ 'dirmod.c',
+ )
endif
if cc.get_id() == 'msvc'
diff --git a/src/test/regress/meson.build b/src/test/regress/meson.build
index 3dcfc11278f..6ec3c77af53 100644
--- a/src/test/regress/meson.build
+++ b/src/test/regress/meson.build
@@ -10,7 +10,7 @@ regress_sources = pg_regress_c + files(
# patterns like ".*-.*-mingw.*". We probably can do better, but for now just
# replace 'gcc' with 'mingw' on windows.
host_tuple_cc = cc.get_id()
-if host_system == 'windows' and host_tuple_cc == 'gcc'
+if host_system in ['windows', 'cygwin'] and host_tuple_cc == 'gcc'
host_tuple_cc = 'mingw'
endif
host_tuple = '@0@-@1@-@2@'.format(host_cpu, host_system, host_tuple_cc)
--
2.25.1
>From 8eddfa78e4ec82e14bff1c8af36efe0b622336f0 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryz...@telsasoft.com>
Date: Fri, 4 Nov 2022 11:53:20 -0500
Subject: [PATCH 3/8] meson: rename 'main' tasks to 'regress' and 'isolation'
See also:
20221001221514.2yy257v4zdfhw...@awork3.anarazel.de
20221021123435.gu16...@telsasoft.com
---
src/test/isolation/meson.build | 2 +-
src/test/regress/meson.build | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/src/test/isolation/meson.build b/src/test/isolation/meson.build
index ba27b8c1d44..e2ed400a80a 100644
--- a/src/test/isolation/meson.build
+++ b/src/test/isolation/meson.build
@@ -58,7 +58,7 @@ isolationtester = executable('isolationtester',
bin_targets += isolationtester
tests += {
- 'name': 'main',
+ 'name': 'isolation',
'sd': meson.current_source_dir(),
'bd': meson.current_build_dir(),
'isolation': {
diff --git a/src/test/regress/meson.build b/src/test/regress/meson.build
index 6ec3c77af53..4f7122f99df 100644
--- a/src/test/regress/meson.build
+++ b/src/test/regress/meson.build
@@ -64,7 +64,7 @@ testprep_targets += refint_regress
tests += {
- 'name': 'main',
+ 'name': 'regress',
'sd': meson.current_source_dir(),
'bd': meson.current_build_dir(),
'regress': {
--
2.25.1
>From 0acbbd2fdd97bbafc5c4552e26f92d52c597eea9 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryz...@telsasoft.com>
Date: Wed, 25 May 2022 21:53:22 -0500
Subject: [PATCH 4/8] cirrus/windows: add compiler_warnings_script
I'm not sure how to write this test in windows shell; it's also not easy to
write it in posix sh, since windows shell is somehow interpretting && and ||...
https://www.postgresql.org/message-id/20220212212310.f645c6vw3njkgxka%40alap3.anarazel.de
See also:
8a1ce5e54f6d144e4f8e19af7c767b026ee0c956
https://cirrus-ci.com/task/6241060062494720
https://cirrus-ci.com/task/6496366607204352
ci-os-only: windows
---
.cirrus.yml | 10 +++++++++-
src/tools/ci/windows-compiler-warnings | 24 ++++++++++++++++++++++++
2 files changed, 33 insertions(+), 1 deletion(-)
create mode 100755 src/tools/ci/windows-compiler-warnings
diff --git a/.cirrus.yml b/.cirrus.yml
index 9f2282471a9..99ac09dc679 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -451,12 +451,20 @@ task:
build_script: |
vcvarsall x64
- ninja -C build
+ ninja -C build |tee build/meson-logs/build.txt
+ REM Since pipes lose exit status of the preceding command, rerun compilation,
+ REM without the pipe exiting now if it fails, rather than trying to run checks
+ ninja -C build > nul
check_world_script: |
vcvarsall x64
meson test %MTEST_ARGS% --num-processes %TEST_JOBS%
+ # This should be last, so check_world is always run
+ always:
+ compiler_warnings_script:
+ - sh src\tools\ci\windows-compiler-warnings build/meson-logs/build.txt build/meson-logs/build-warnings.txt
+
on_failure:
<<: *on_failure_meson
crashlog_artifacts:
diff --git a/src/tools/ci/windows-compiler-warnings b/src/tools/ci/windows-compiler-warnings
new file mode 100755
index 00000000000..685aa995d5c
--- /dev/null
+++ b/src/tools/ci/windows-compiler-warnings
@@ -0,0 +1,24 @@
+#! /bin/sh
+# Success if the given exists and is empty, else fail
+# This is a separate file to avoid dealing with windows shell quoting and escaping,
+# or giving the impression that one-liner scripts will work trivially.
+set -e
+
+infile=$1
+outfile=$2
+
+# Looks like:
+# [19:39:27.130] c:\cirrus\src\backend\backup\basebackup_zstd.c(80) : warning C4715: 'bbsink_zstd_new': not all control paths return a value
+# should include linker warnings?
+grep ": warning " "$infile" > $outfile ||
+ [ $? -eq 1 ]
+
+if [ -s "$outfile" ]
+then
+ # Display the file's content, then exit indicating failure
+ cat "$outfile"
+ exit 1
+else
+ # Success
+ exit 0
+fi
--
2.25.1
>From e85fe83fd8a4b4c79a96d2bf66cd6a5e1bdfcd1e Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryz...@telsasoft.com>
Date: Sat, 26 Feb 2022 19:34:35 -0600
Subject: [PATCH 5/8] cirrus: build docs as a separate task..
This will run the doc build if any docs have changed, even if Linux
fails, to allow catch doc build failures.
This'll automatically show up as a separate "column" on cfbot.
Also, in the future, this will hopefully upload each patch's changed HTML docs
as an artifact, for easy review.
Note that this is currently building docs with both autoconf and meson.
ci-os-only: html
---
.cirrus.yml | 62 +++++++++++++++++++++++++++++++++++++----------------
1 file changed, 44 insertions(+), 18 deletions(-)
diff --git a/.cirrus.yml b/.cirrus.yml
index 99ac09dc679..37fd79e5b77 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -472,6 +472,9 @@ task:
type: text/plain
+###
+# Test that code can be built with gcc/clang without warnings
+###
task:
name: CompilerWarnings
@@ -515,10 +518,6 @@ task:
#apt-get update
#DEBIAN_FRONTEND=noninteractive apt-get -y install ...
- ###
- # Test that code can be built with gcc/clang without warnings
- ###
-
setup_script: echo "COPT=-Werror" > src/Makefile.custom
# Trace probes have a history of getting accidentally broken. Use the
@@ -580,20 +579,6 @@ task:
make -s -j${BUILD_JOBS} clean
time make -s -j${BUILD_JOBS} world-bin
- ###
- # Verify docs can be built
- ###
- # XXX: Only do this if there have been changes in doc/ since last build
- always:
- docs_build_script: |
- time ./configure \
- --cache gcc.cache \
- CC="ccache gcc" \
- CXX="ccache g++" \
- CLANG="ccache clang"
- make -s -j${BUILD_JOBS} clean
- time make -s -j${BUILD_JOBS} -C doc
-
###
# Verify headerscheck / cpluspluscheck succeed
#
@@ -617,3 +602,44 @@ task:
always:
upload_caches: ccache
+
+
+###
+# Verify docs can be built
+# changesInclude() will skip this task if none of the commits since
+# CIRRUS_LAST_GREEN_CHANGE touched any relevant files. The comparison appears
+# to be like "git log a..b -- ./file", not "git diff a..b -- ./file"
+###
+
+task:
+ name: Documentation
+
+ env:
+ CPUS: 1
+ BUILD_JOBS: 1
+
+ only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*(docs|html).*'
+ skip: "!changesInclude('.cirrus.yml', 'doc/**')"
+
+ container:
+ image: $CONTAINER_REPO/linux_debian_bullseye_ci:latest
+ cpu: $CPUS
+ memory: 2G
+
+ sysinfo_script: |
+ id
+ uname -a
+ cat /proc/cmdline
+ ulimit -a -H && ulimit -a -S
+ export
+
+ # Exercise HTML and other docs:
+ ninja_docs_build_script: |
+ mkdir build.ninja
+ cd build.ninja
+ time meson setup
+ time ninja docs
+
+ docs_build_script: |
+ time ./configure
+ make -s -j${BUILD_JOBS} -C doc
--
2.25.1
>From adebe93a4409990e929f2775d45c6613134a4243 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryz...@telsasoft.com>
Date: Tue, 26 Jul 2022 20:30:02 -0500
Subject: [PATCH 6/8] cirrus/ccache: add explicit cache keys..
Since otherwise, building with ci-os-only will probably fail to use the
normal cache, since the cache key is computed using both the task name
and its *index* in the list of caches (internal/executor/cache.go:184).
---
.cirrus.yml | 10 ++++++++++
1 file changed, 10 insertions(+)
diff --git a/.cirrus.yml b/.cirrus.yml
index 37fd79e5b77..221431e70c4 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -87,6 +87,9 @@ task:
ccache_cache:
folder: $CCACHE_DIR
+ fingerprint_key: ccache/freebsd
+ reupload_on_changes: true
+
# Workaround around performance issues due to 32KB block size
repartition_script: src/tools/ci/gcp_freebsd_repartition.sh
create_user_script: |
@@ -181,6 +184,8 @@ task:
ccache_cache:
folder: ${CCACHE_DIR}
+ fingerprint_key: ccache/linux
+ reupload_on_changes: true
sysinfo_script: |
id
@@ -359,6 +364,9 @@ task:
ccache_cache:
folder: $CCACHE_DIR
+ fingerprint_key: ccache/macos
+ reupload_on_changes: true
+
configure_script: |
brewpath="/usr/local"
PKG_CONFIG_PATH="${brewpath}/lib/pkgconfig:${PKG_CONFIG_PATH}"
@@ -513,6 +521,8 @@ task:
ccache_cache:
folder: $CCACHE_DIR
+ fingerprint_key: ccache/warnings
+ reupload_on_changes: true
setup_additional_packages_script: |
#apt-get update
--
2.25.1
>From f16739bc5d2087847129baf663aa25fa9edb8449 Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryz...@telsasoft.com>
Date: Sun, 3 Apr 2022 00:10:20 -0500
Subject: [PATCH 7/8] cirrus/ccache: disable compression and show stats
Since v4.0, ccache enables zstd compression by default, saving roughly
2x-3x. But, cirrus caches are compressed as tar.gz, so we could disable
ccache compression, allowing cirrus to gzip the uncompressed data
(better than ccache's default of zstd-1).
With default compression enabled (https://cirrus-ci.com/task/6692342840164352):
linux/debian/bullseye has 4.2; 99MB cirrus cache; cache_size_kibibyte 109616
macos has 4.5.1: 47MB cirrus cache; cache_size_kibibyte 52500
freebsd has 3.7.12: 42MB cirrus cache; cache_size_kibibyte 134064
XXX windows has 4.7.2; 180MB cirrus cache; cache_size_kibibyte 51179
todo: compiler warnings
With compression disabled (https://cirrus-ci.com/task/4614182514458624):
linux: 91MB cirrus cache; cache_size_kibibyte 316136
macos: 41MB cirrus cache; cache_size_kibibyte 118068
windows: 42MB cirrus cache; cache_size_kibibyte 134064
freebsd is the same
The stats should be shown and/or logged.
ccache --show-stats shows the *cumulative* stats (including prior
compilations)
ccache --zero-stats clears out not only the global stats, but the
per-file cache stats (from which the global stats are derived) - which
obviously makes the cache work poorly.
Note that ccache 4.4 supports CCACHE_STATSLOG, which seems ideal.
The log should be written *outside* the ccache folder - it shouldn't be
preserved across cirrusci task invocations.
freebsd, linux, macos
ci-os-only: linux
---
.cirrus.yml | 13 ++++++++-----
1 file changed, 8 insertions(+), 5 deletions(-)
diff --git a/.cirrus.yml b/.cirrus.yml
index 221431e70c4..d1c3119e4ff 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -16,7 +16,9 @@ env:
# Useful to be able to analyse what in a script takes long
CIRRUS_LOG_TIMESTAMP: true
- CCACHE_MAXSIZE: "250M"
+ CCACHE_MAXSIZE: "750M"
+ CCACHE_NOCOMPRESS: 1
+ #CCACHE_STATSLOG: ccache-stats.log
# target to test, for all but windows
CHECK: check-world PROVE_FLAGS=$PROVE_FLAGS
@@ -236,7 +238,7 @@ task:
- name: Linux - Debian Bullseye - Meson
env:
- CCACHE_MAXSIZE: "400M" # tests two different builds
+ CCACHE_MAXSIZE: "1200M" # tests two different builds
configure_script: |
su postgres <<-EOF
@@ -307,6 +309,7 @@ task:
CIRRUS_WORKING_DIR: ${HOME}/pgsql/
CCACHE_DIR: ${HOME}/ccache
+ CCACHE_STATSLOG: ${CCACHE_DIR}.stats.log
HOMEBREW_CACHE: ${HOME}/homebrew-cache
PERL5LIB: ${HOME}/perl5/lib/perl5
@@ -388,7 +391,7 @@ task:
-DPG_TEST_EXTRA="$PG_TEST_EXTRA" \
build
- build_script: ninja -C build -j${BUILD_JOBS}
+ build_script: ninja -C build -j${BUILD_JOBS} && ccache --show-log-stats
upload_caches: ccache
test_world_script: |
@@ -496,14 +499,14 @@ task:
# Use larger ccache cache, as this task compiles with multiple compilers /
# flag combinations
- CCACHE_MAXSIZE: "1GB"
+ CCACHE_MAXSIZE: "3G"
CCACHE_DIR: "/tmp/ccache_dir"
LINUX_CONFIGURE_FEATURES: *LINUX_CONFIGURE_FEATURES
LINUX_MESON_FEATURES: *LINUX_MESON_FEATURES
# task that did not run, count as a success, so we need to recheck Linux'
- # condition here ...
+ # condition here; cirus warns if the "only_if" condition doesn't match the task being depended on
only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\nci-os-only:.*' || $CIRRUS_CHANGE_MESSAGE =~ '.*\nci-os-only:[^\n]*linux.*'
container:
--
2.25.1
>From de86a044d663ed0a6d5c292e44a4d56bd9b0cd4b Mon Sep 17 00:00:00 2001
From: Justin Pryzby <pryz...@telsasoft.com>
Date: Tue, 19 Jul 2022 12:38:45 -0500
Subject: [PATCH 8/8] cirrus/warnings: use a single/common 'always' block
ci-os-only: warnings
---
.cirrus.yml | 44 +++++++++++++++++++-------------------------
1 file changed, 19 insertions(+), 25 deletions(-)
diff --git a/.cirrus.yml b/.cirrus.yml
index d1c3119e4ff..9ab8ce30e09 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -522,11 +522,6 @@ task:
clang -v
export
- ccache_cache:
- folder: $CCACHE_DIR
- fingerprint_key: ccache/warnings
- reupload_on_changes: true
-
setup_additional_packages_script: |
#apt-get update
#DEBIAN_FRONTEND=noninteractive apt-get -y install ...
@@ -537,8 +532,13 @@ task:
# different compilers to build with different combinations of dtrace on/off
# and cassert on/off.
- # gcc, cassert off, dtrace on
always:
+ ccache_cache:
+ folder: $CCACHE_DIR
+ fingerprint_key: ccache/warnings
+ reupload_on_changes: true
+
+ # gcc, cassert off, dtrace on
gcc_warning_script: |
time ./configure \
--cache gcc.cache \
@@ -548,8 +548,7 @@ task:
make -s -j${BUILD_JOBS} clean
time make -s -j${BUILD_JOBS} world-bin
- # gcc, cassert on, dtrace off
- always:
+ # gcc, cassert on, dtrace off
gcc_a_warning_script: |
time ./configure \
--cache gcc.cache \
@@ -559,8 +558,7 @@ task:
make -s -j${BUILD_JOBS} clean
time make -s -j${BUILD_JOBS} world-bin
- # clang, cassert off, dtrace off
- always:
+ # clang, cassert off, dtrace off
clang_warning_script: |
time ./configure \
--cache clang.cache \
@@ -569,8 +567,7 @@ task:
make -s -j${BUILD_JOBS} clean
time make -s -j${BUILD_JOBS} world-bin
- # clang, cassert on, dtrace on
- always:
+ # clang, cassert on, dtrace on
clang_a_warning_script: |
time ./configure \
--cache clang.cache \
@@ -581,8 +578,7 @@ task:
make -s -j${BUILD_JOBS} clean
time make -s -j${BUILD_JOBS} world-bin
- # cross-compile to windows
- always:
+ # cross-compile to windows
mingw_cross_warning_script: |
time ./configure \
--host=x86_64-w64-mingw32 \
@@ -592,16 +588,15 @@ task:
make -s -j${BUILD_JOBS} clean
time make -s -j${BUILD_JOBS} world-bin
- ###
- # Verify headerscheck / cpluspluscheck succeed
- #
- # - Don't use ccache, the files are uncacheable, polluting ccache's
- # cache
- # - Use -fmax-errors, as particularly cpluspluscheck can be very verbose
- # - XXX have to disable ICU to avoid errors:
- # https://postgr.es/m/20220323002024.f2g6tivduzrktgfa%40alap3.anarazel.de
- ###
- always:
+ ###
+ # Verify headerscheck / cpluspluscheck succeed
+ #
+ # - Don't use ccache, the files are uncacheable, polluting ccache's
+ # cache
+ # - Use -fmax-errors, as particularly cpluspluscheck can be very verbose
+ # - XXX have to disable ICU to avoid errors:
+ # https://postgr.es/m/20220323002024.f2g6tivduzrktgfa%40alap3.anarazel.de
+ ###
headers_headerscheck_script: |
time ./configure \
${LINUX_CONFIGURE_FEATURES} \
@@ -613,7 +608,6 @@ task:
headers_cpluspluscheck_script: |
time make -s cpluspluscheck EXTRAFLAGS='-fmax-errors=10'
- always:
upload_caches: ccache
--
2.25.1
