On Fri, Jan 22, 2021 at 11:07:22AM +0100, Thomas Huth wrote: > Currently, our check-system-* jobs are recompiling the whole sources > again. This happens due to the fact that the jobs are checking out > the whole source tree and required submodules again, and only try > to use the "build" directory with the binaries and object files > as an artifact from the previous stage - which simply does not work > anymore (with the current version of meson). Due to some changed > time stamps, meson is always trying to rebuild the whole tree.
This used to work in the past didn't it ? Did something change in meson to break this, or have we just not noticed before. > So instead of trying to marry a freshly checked out source tree > with the pre-built binaries in these jobs, let's simply pass the > whole source including the submodules and the build tree as artifact > to the test jobs. That way timestamps get preserved and there is > no rebuild of the sources anymore. This saves ca. 15 - 20 minutes > of precious CI cycles in each run. I'm a little worried we might end up hitting the artifact size limit which is supposedly 1GB on gitlab.com. Im guessing this must be measuring the compressed size though, as a src checkout with build dir and .git dir is already way over 1GB. > > Signed-off-by: Thomas Huth <th...@redhat.com> > --- > This is how a job looked like before my patch, running for 42 minutes: > https://gitlab.com/huth/qemu/-/jobs/978432757 > > And this is how it looks like afterwards - it just took 18 minutes: > https://gitlab.com/huth/qemu/-/jobs/979500316 > > .gitlab-ci.d/containers.yml | 1 + > .gitlab-ci.yml | 40 +++++++++++++++++++++++++------------ > 2 files changed, 28 insertions(+), 13 deletions(-) > > diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml > index e2f9c99e27..d55280661f 100644 > --- a/.gitlab-ci.yml > +++ b/.gitlab-ci.yml > @@ -39,7 +39,6 @@ include: > image: $CI_REGISTRY_IMAGE/qemu/$IMAGE:latest > script: > - cd build > - - find . -type f -exec touch {} + > - make $MAKE_CHECK_ARGS > > .acceptance_template: &acceptance_definition > @@ -83,8 +82,7 @@ build-system-alpine: > artifacts: > expire_in: 2 days > paths: > - - .git-submodule-status > - - build > + - "*" > > check-system-alpine: > <<: *native_test_job_definition > @@ -92,6 +90,7 @@ check-system-alpine: > - job: build-system-alpine > artifacts: true > variables: > + GIT_CHECKOUT: "false" > IMAGE: alpine > MAKE_CHECK_ARGS: check > > @@ -101,6 +100,7 @@ acceptance-system-alpine: > - job: build-system-alpine > artifacts: true > variables: > + GIT_CHECKOUT: "false" > IMAGE: alpine > MAKE_CHECK_ARGS: check-acceptance > <<: *acceptance_definition > @@ -116,7 +116,7 @@ build-system-ubuntu: > artifacts: > expire_in: 2 days > paths: > - - build > + - "*" > > check-system-ubuntu: > <<: *native_test_job_definition > @@ -124,6 +124,7 @@ check-system-ubuntu: > - job: build-system-ubuntu > artifacts: true > variables: > + GIT_CHECKOUT: "false" > IMAGE: ubuntu2004 > MAKE_CHECK_ARGS: check > > @@ -133,6 +134,7 @@ acceptance-system-ubuntu: > - job: build-system-ubuntu > artifacts: true > variables: > + GIT_CHECKOUT: "false" > IMAGE: ubuntu2004 > MAKE_CHECK_ARGS: check-acceptance > <<: *acceptance_definition > @@ -148,7 +150,7 @@ build-system-debian: > artifacts: > expire_in: 2 days > paths: > - - build > + - "*" > > check-system-debian: > <<: *native_test_job_definition > @@ -156,6 +158,7 @@ check-system-debian: > - job: build-system-debian > artifacts: true > variables: > + GIT_CHECKOUT: "false" > IMAGE: debian-amd64 > MAKE_CHECK_ARGS: check > > @@ -170,7 +173,7 @@ build-tools-and-docs-debian: > artifacts: > expire_in: 2 days > paths: > - - build > + - "*" > > acceptance-system-debian: > <<: *native_test_job_definition > @@ -178,6 +181,7 @@ acceptance-system-debian: > - job: build-system-debian > artifacts: true > variables: > + GIT_CHECKOUT: "false" > IMAGE: debian-amd64 > MAKE_CHECK_ARGS: check-acceptance > <<: *acceptance_definition > @@ -194,7 +198,7 @@ build-system-fedora: > artifacts: > expire_in: 2 days > paths: > - - build > + - "*" > > check-system-fedora: > <<: *native_test_job_definition > @@ -202,6 +206,7 @@ check-system-fedora: > - job: build-system-fedora > artifacts: true > variables: > + GIT_CHECKOUT: "false" > IMAGE: fedora > MAKE_CHECK_ARGS: check > > @@ -211,6 +216,7 @@ acceptance-system-fedora: > - job: build-system-fedora > artifacts: true > variables: > + GIT_CHECKOUT: "false" > IMAGE: fedora > MAKE_CHECK_ARGS: check-acceptance > <<: *acceptance_definition > @@ -226,7 +232,7 @@ build-system-centos: > artifacts: > expire_in: 2 days > paths: > - - build > + - "*" > > check-system-centos: > <<: *native_test_job_definition > @@ -234,6 +240,7 @@ check-system-centos: > - job: build-system-centos > artifacts: true > variables: > + GIT_CHECKOUT: "false" > IMAGE: centos8 > MAKE_CHECK_ARGS: check > > @@ -243,6 +250,7 @@ acceptance-system-centos: > - job: build-system-centos > artifacts: true > variables: > + GIT_CHECKOUT: "false" > IMAGE: centos8 > MAKE_CHECK_ARGS: check-acceptance > <<: *acceptance_definition > @@ -257,7 +265,7 @@ build-system-opensuse: > artifacts: > expire_in: 2 days > paths: > - - build > + - "*" > > check-system-opensuse: > <<: *native_test_job_definition > @@ -265,6 +273,7 @@ check-system-opensuse: > - job: build-system-opensuse > artifacts: true > variables: > + GIT_CHECKOUT: "false" > IMAGE: opensuse-leap > MAKE_CHECK_ARGS: check > > @@ -274,6 +283,7 @@ acceptance-system-opensuse: > - job: build-system-opensuse > artifacts: true > variables: > + GIT_CHECKOUT: "false" > IMAGE: opensuse-leap > MAKE_CHECK_ARGS: check-acceptance > <<: *acceptance_definition > @@ -444,7 +454,7 @@ build-deprecated: > artifacts: > expire_in: 2 days > paths: > - - build > + - "*" > > # We split the check-tcg step as test failures are expected but we still > # want to catch the build breaking. > @@ -454,6 +464,7 @@ check-deprecated: > - job: build-deprecated > artifacts: true > variables: > + GIT_CHECKOUT: "false" > IMAGE: debian-all-test-cross > MAKE_CHECK_ARGS: check-tcg > allow_failure: true > @@ -526,7 +537,7 @@ build-crypto-old-nettle: > MAKE_CHECK_ARGS: check-build > artifacts: > paths: > - - build > + - "*" > > check-crypto-old-nettle: > <<: *native_test_job_definition > @@ -534,6 +545,7 @@ check-crypto-old-nettle: > - job: build-crypto-old-nettle > artifacts: true > variables: > + GIT_CHECKOUT: "false" > IMAGE: centos7 > MAKE_CHECK_ARGS: check > > @@ -547,7 +559,7 @@ build-crypto-old-gcrypt: > MAKE_CHECK_ARGS: check-build > artifacts: > paths: > - - build > + - "*" > > check-crypto-old-gcrypt: > <<: *native_test_job_definition > @@ -555,6 +567,7 @@ check-crypto-old-gcrypt: > - job: build-crypto-old-gcrypt > artifacts: true > variables: > + GIT_CHECKOUT: "false" > IMAGE: centos7 > MAKE_CHECK_ARGS: check > > @@ -568,7 +581,7 @@ build-crypto-only-gnutls: > MAKE_CHECK_ARGS: check-build > artifacts: > paths: > - - build > + - "*" > > check-crypto-only-gnutls: > <<: *native_test_job_definition > @@ -576,6 +589,7 @@ check-crypto-only-gnutls: > - job: build-crypto-only-gnutls > artifacts: true > variables: > + GIT_CHECKOUT: "false" > IMAGE: centos7 > MAKE_CHECK_ARGS: check > > -- > 2.27.0 > > Regards, Daniel -- |: https://berrange.com -o- https://www.flickr.com/photos/dberrange :| |: https://libvirt.org -o- https://fstop138.berrange.com :| |: https://entangle-photo.org -o- https://www.instagram.com/dberrange :|
