Signed-off-by: Glenn Washburn <developm...@efficientek.com>
---
scripts/ci/functions.sh | 923 +++++++++++++++++++++++++++++++++++++++-
1 file changed, 922 insertions(+), 1 deletion(-)
diff --git a/scripts/ci/functions.sh b/scripts/ci/functions.sh
index 2f4cecaa1..734b36d3f 100644
--- a/scripts/ci/functions.sh
+++ b/scripts/ci/functions.sh
@@ -1,5 +1,412 @@
#!/bin/bash
+### Documentation on configuration environment variables
+
+# SDIR [dirname of $0]:
+# Path to directory the caller of this script resides in.
+# ROOT [./grub-tests]:
+# Path to directory where all file operations take place, unless
overridden
+# by another environment variable.
+# CI_PROJECT_DIR [$ROOT]:
+# Base directory to use in commands.
+# CI_TYPE [local]:
+# Value used to control sourcing of ${CI_SCRIPT_DIR}/functions.$CI_TYPE.sh
+# which overrides the variables/functions in this file.
+# SCRIPTS_DIR [${SDIR} if ${SDIR}/ci else ${SRCDIR}/scripts]:
+# Path to scripts directory from source.
+# CI_SCRIPTS_DIR [${SCRIPTS_DIR}/ci]:
+# Path to CI scripts directory, normally the ci subdirectory of SCRITS_DIR
+# and where this file resides.
+# SHELL_TRACE [n]:
+# If set, turn on shell tracing of everything. NOTE: TEST_VERBOSITY=3
turns
+# on more targeted shell tracing
+# TMPDIR [$ROOT/testtmp]:
+# Path to directory to use for temporary files.
+# MATRIX [see definition below]:
+# A list of $ARCH:$TARGET[,...]:$CROSS_ARCH, one per line, with lines
+# starting with # being ignored.
+# JOBS:
+# Number of make jobs to run in parallel. Defaulted in build.sh and
test.sh
+# to the number of available processors.
+# NUM_FAILED_LOG_LINES [100]:
+# Set to integer number of log lines to display from the end of a failed
+# log file.
+
+### Source checkout variables
+# GIT_REPO:
+# URL to git repo
+# GIT_BRANCH:
+# Branch name in git repo
+# GIT_CLONE_ARGS [--depth=1 --no-single-branch]:
+# Extra args to use when cloning repository
+# SRCDIR [${CI_PROJECT_DIR}/grub]:
+# Path to source checkout
+
+### Distro package variables
+# APT_OPTS [--no-upgrade]:
+# Extra options when running apt.
+# PACKAGES [see definition below]:
+# A list of packages to add to a default list of packages needing to be
+# installed. The default list can only be appended to.
+# PACKAGE_CACHE [${CI_PROJECT_DIR}/packages]:
+# Path to directory where downloaded packages will be cached.
+
+### Ccache variables
+# CCACHE_DIR [${CI_PROJECT_DIR}/ccache]:
+# Directory to use for ccache
+
+### Cross compiler tool chain variables
+# CROSS_DIR [$CI_PROJECT_DIR/cross]:
+# Path to directory containing cross compilers tool chains for each
+# architecture. If it doesn't exist, it will be created and required tool
+# chains will be downloaded/installed there.
+# CROSS_VERSION [10.1.0]:
+# Version of cross-compiler to use (suggested: 10.1.0, 9.3.0, 8.1.0,
+# 7.5.0 [riscv32/64 builds fail before 7.3.0]).
+
+### Configure/Build variables
+# CONFIGURE_OPTS [--enable-boot-time]:
+# Extra options to pass to configure.
+# FONT_SOURCE:
+# Path to unicode font named "unifont.<ext>" where ext is one of:
+# pcf pcf.gz bdf bdf.gz ttf ttf.gz
+# DJVU_FONT_SOURCE:
+# Path to DejaVu font named "DejaVuSans.<ext>" where ext is one of:
+# pcf pcf.gz bdf bdf.gz ttf ttf.gz
+# BUILD_ALL_TARGETS:
+# If set 'y', all targets defined in the build matrix will be
+# built ignoring DISABLED_BUILDS, but can still be disabled by
+# DISABLE_ALL_BUILDS.
+# DISABLED_BUILDS [see definition below]:
+# A list of targets for which builds are disabled, one per line. These
+# default targets are disabled generally because they do not work yet.
+# Lines beginning with '#' may be used as comments and are ignored. Some
+# hints as to why the tests fail are included in comment lines. Patches
+# which get these tests working are very welcome.
+# DISABLE_ALL_BUILDS [n]:
+# If set to 'y', disable completely building (eg. the run_build command).
+
+### Tests variables
+# TESTS_TIMEOUT:
+# Set timeout for completion of all make check tests (see "man timeout"
+# duration argument). This is most useful when the make check job is
+# taking longer than the job timeout configured by the runner. In this
+# case, set this to 5-10 minutes less than runner timeout so that there
+# is time to package up the logs for debugging.
+# IGNORE_TIMEDOUT_TEST_FAILURE [y]:
+# If set to 'y', tests which fail due to a timeout in grub-shell of qemu
+# will not be counted as a failure. These failures are almost always the
+# result of insufficient runner resources to complete the execution of
qemu
+# within the timeout period.
+# FAIL_ON_HARD_ERRORS [n]:
+# If set to 'y', hard errors will cause a failure. A hard error indicates
+# that the test was not able to be run (eg. dependencies not found), and
+# as such a hard error does not constitute a true failure of the test.
+# DISABLE_ALL_TESTS [n]:
+# If set to 'y', make check tests will be disabled. Effectively only test
+# whether the building is successful for configured platforms.
+# TEST_ALL_TARGETS:
+# If set 'y', failing targets which have been marked disabled by
+# DISABLED_TESTS will be enabled.
+# TEST_VERBOSITY [3]:
+# Verbosity level when running tests [supported values: '' and 0-9]. Level
+# 9 turns on all debug messages in grub, so tests looking for exact output
+# will likely fail.
+# STRACE_TESTS:
+# If set, strace individual tests. WARNING: This will cause testing to
take
+# much longer which can cause some test to fail by timing out or having
+# incorrect timing
+# TEST_DATA_PREFIX:
+# Filename prefix to use when saving test data. If not set, test data
+# will not be saved.
+# GRUB_QEMU_OPTS:
+# Add options to qemu when used in tests (ex: -m size=64M)
+# GRUB_SHELL_DEFAULT_TIMEOUT [600s]:
+# Set grub-shell timeout for qemu instance (see man timeout duration
+# argument). The timeout has been raised from the default of 60 seconds
+# in the grub-shell script to 600 seconds because CI runners may run in
+# a slow virtual machine. This may need to be raised further if grub-shell
+# is failing with a timeout (exit code 124 or 137).
+# EXPECTED_FAILURES [see definition below]:
+# A list of make check tests allowed to fail, one per line. Test may be
+# prefixed with "$ARCHITECTURE-$PLATFORM:" to only apply to certain
+# targets. Lines beginning with '#' may be used as comments and are
+# ignored.
+# EXPECTED_FUNCTIONAL_FAILURES [see definition below]:
+# A list of functional tests from the grub_func_test make check that are
+# allowed to fail, one per line. Lines beginning with '#' may be used as
+# comments and are ignored.
+# DISABLED_TESTS [see definition below]:
+# A list of targets for which make check is not run, one per line. These
+# targets are disabled because tests mostly do not work. Lines beginning
+# with '#' may be used as comments and are ignored. Some hints as to why
+# the tests fail are included in comment lines. Patches which get these
+# tests working are very welcome.
+
+### Make image variables
+# MAKE_ALL_IMAGE_TARGETS:
+# If set 'y', all target image formats will be built, ignoring
+# DISABLED_IMAGES.
+# DISABLED_IMAGES [see definition below]:
+# A list of target formats as accepted by grub-mkimage for which images
+# are not built. These target formats are disabled generally because they
+# do not work. Lines beginning with '#' may be used as comments and are
+# ignored. Some hints as to why the tests fail are included in comment
+# lines. Patches which get these tests working are very welcome.
+
+
+##### CI configuration variables #####
+SDIR=$(realpath -e "${SDIR:-"${0%/*}"}")
+ROOT=$(realpath -m "${ROOT:-"./grub-tests"}")
+TMPDIR=${TMPDIR:-"$ROOT/testtmp"}
+CI_PROJECT_DIR=${CI_PROJECT_DIR:-"$ROOT"}
+CI_TYPE=${CI_TYPE:-"local"}
+SHELL_TRACE=${SHELL_TRACE:-"n"}
+
+GIT_REPO=${GIT_REPO}
+GIT_BRANCH=${GIT_BRANCH}
+GIT_REPO_DEFAULT="https://git.savannah.gnu.org/git/grub.git";
+GIT_BRANCH_DEFAULT="master"
+
+GIT_CLONE_ARGS=${GIT_CLONE_ARGS:-"--depth=1 --no-single-branch"}
+CI_COMMIT_SHORT_SHA=${CI_COMMIT_SHORT_SHA}
+SRCDIR=${SRCDIR:-"${CI_PROJECT_DIR}/grub"}
+
+if [ -d "${SDIR}/ci" ]; then
+ SCRIPTS_DIR=${SCRIPTS_DIR:-"${SDIR}"}
+else
+ SCRIPTS_DIR=${SCRIPTS_DIR:-"${SRCDIR}/scripts"}
+fi
+CI_SCRIPTS_DIR=${CI_SCRIPTS_DIR:-"${SCRIPTS_DIR}/ci"}
+
+CROSS_DIR=${CROSS_DIR:-$CI_PROJECT_DIR/cross}
+CROSS_VERSION=${CROSS_VERSION:-10.1.0}
+
+CONFIGURE_OPTS=${CONFIGURE_OPTS:-"--enable-boot-time"}
+DISABLE_ALL_BUILDS=${DISABLE_ALL_BUILDS:-"n"}
+
+IGNORE_TIMEDOUT_TEST_FAILURE=${IGNORE_TIMEDOUT_TEST_FAILURE:-"y"}
+GRUB_SHELL_DEFAULT_TIMEOUT=${GRUB_SHELL_DEFAULT_TIMEOUT:-"600s"}
+TEST_VERBOSITY=${TEST_VERBOSITY:-3}
+DISABLE_ALL_TESTS=${DISABLE_ALL_TESTS:-"n"}
+
+MATRIX=${MATRIX:-"
+ x86_64:x86_64-efi,x86_64-xen:x86_64-linux
+
i386:i386-coreboot,i386-efi,i386-ieee1275,i386-multiboot,i386-pc,i386-qemu,i386-xen,i386-xen_pvh:i386-linux
+ powerpc:powerpc-ieee1275:powerpc-linux
+ powerpc64:powerpc64-ieee1275:powerpc64-linux
+ sparc:sparc-ieee1275:sparc-linux
+ sparc64:sparc64-ieee1275:sparc64-linux
+ ia64:ia64-efi:ia64-linux
+
mips:mips-arc,mips-qemu_mips,mipsel-arc,mipsel-qemu_mips,mipsel-loongson:mips64-linux
+ arm:arm-coreboot,arm-efi,arm-uboot:arm-linux-gnueabi
+ arm64:arm64-efi:aarch64-linux
+ riscv32:riscv32-efi:riscv32-linux
+ riscv64:riscv64-efi:riscv64-linux
+"}
+
+APT_OPTS=${APT_OPTS:-"--no-upgrade"}
+PACKAGE_CACHE="${CI_PROJECT_DIR}/packages"
+PACKAGES="
+ multitee
+ git
+ ccache
+ ### Install cross compilers
+ tar
+ wget
+ xz-utils
+ ### Bootstrap Setup
+ autoconf
+ automake
+ autopoint
+ gettext
+ git
+ locales
+ patch
+ pkg-config
+ python
+ ### Grub build requirements
+ automake
+ bison
+ flex
+ gettext
+ libdevmapper-dev
+ libfreetype6-dev
+ libfuse-dev
+ liblzma-dev
+ libzfslinux-dev
+ make
+ pkg-config
+ python
+ unifont
+ ### Grub testing requirements
+ g++
+ gawk
+ kmod
+ strace
+ ### Virtualization tools
+ qemu-system
+ # qemu-efi-aarch64
+ # qemu-efi-arm
+ # qemu-efi
+ ovmf
+ # ovmf-ia32
+ openbios-ppc
+ openbios-sparc
+ ### Used by grub-fs-tester
+ wamerican
+ ### needed by grub-mkrescue
+ xorriso
+ ### needed by grub-mkrescue for efi platforms
+ mtools
+ ### Requirements for specific tests
+ parted
+ util-linux
+ ### Archivers
+ cpio
+ squashfs-tools
+ tar
+ ### Compression tools
+ gzip
+ lzop
+ xz-utils
+ ### File system tools
+ btrfs-progs
+ dosfstools
+ e2fsprogs
+ exfat-utils
+ f2fs-tools
+ genromfs
+ hfsprogs
+ jfsutils
+ nilfs-tools
+ ntfs-3g
+ reiserfsprogs
+ udftools
+ xfsprogs
+ zfs-fuse
+ ### Needed by hfs test
+ recode
+ ### Needed for ntfs test
+ attr
+ ### Extra packages specified by user
+ $PACKAGES
+"
+
+DEB_URLS="
+
http://mirrors.edge.kernel.org/ubuntu/pool/universe/e/edk2/ovmf-ia32_2020.11-4_all.deb
+
http://mirrors.edge.kernel.org/ubuntu/pool/main/e/edk2/qemu-efi-arm_2020.11-4_all.deb
+
http://mirrors.edge.kernel.org/ubuntu/pool/main/e/edk2/qemu-efi-aarch64_2020.11-4_all.deb
+"
+
+DISABLED_BUILDS=${DISABLED_BUILDS:-"
+ ### qemu-system-mips64: Could not load MIPS bios 'mips_bios.bin',
+ ### and no -kernel argument was specified
+ mipsel-arc
+ mipsel-qemu_mips
+ mipsel-loongson
+"}
+
+DISABLED_TESTS=${DISABLED_TESTS:-"
+ ### Failing on unreachable target error when invoking grub-mkimage
+ riscv32-efi
+ ### Need to get coreboot ROM and cbfstool
+ i386-coreboot
+ ### qemu-system-mips64: Could not load MIPS bios 'mips_bios.bin',
+ ### and no -kernel argument was specified
+ mips-arc
+ mips-qemu_mips
+ mipsel-arc
+ mipsel-qemu_mips
+ ### Need an i386 built open firmware image, otherwise uses regular bios
+ ### and stalls with 'Booting from Hard Disk...'
+ i386-ieee1275
+ ### Qemu logs 'Boot failed: Could not read from CDROM (code 0004)', which
+ ### according to
https://lists.gnu.org/archive/html/qemu-devel/2008-06/msg00476.html
+ ### means 'cd is not eltorito (BRVD)'
+ i386-multiboot
+ ### TODO: These have no support in grub-shell and use the default
+ ### qemu-system-i386, which is clearly not correct
+ arm-coreboot
+ arm-uboot
+ ia64-efi
+ riscv64-efi
+ ### Also not supported in grub-shell
+ i386-xen
+ i386-xen_pvh
+ x86_64-xen
+"}
+
+DISABLED_IMAGES=${DISABLED_IMAGES:-"
+ ### Failing on unreachable target error when invoking grub-mkimage
+ riscv32-efi
+ ### FIXME: There is a bug in grub-mkimage which causes the modules
+ ### directory for these target formats to be not found
+ arm-coreboot-vexpress
+ arm-coreboot-veyron
+"}
+
+EXPECTED_FAILURES=${EXPECTED_FAILURES:-"
+ ### Usually this test passes, but occasionally the virtual machine is
+ ### too slow and the time delta threshold is exceeded. So allow failures.
+ #grub_cmd_sleep
+ ### This can fail on the volume label check because newer Debian versions
+ ### hfsprogs has a bug where mkfs.hfs does not write the volume label.
+ hfs_test
+ ### This fails because grub-fstest ls -la is not outputting file time
+ ### on files, but it is on directories. In linux there are file times.
+ ### Is this a bug in the zfs module? Also the SYMLINK test fails.
+ ### And the LIST test fails, where the error message 'error: unsupported
+ ### embedded BP (type=255)' is outputted.
+ zfs_test
+ ### sparc64-ieee1275 fails these tests, but the rest should succeed
+ ###
https://marc.info/?i=20201219002902.4490b844%20()%20crass-HP-ZBook-15-G2
+ sparc64-ieee1275:grub_script_expansion
+ sparc64-ieee1275:gzcompress_test
+ sparc64-ieee1275:grub_func_test
+ sparc64-ieee1275:file_filter_test
+ ### This test is skipped by sparc64-ieee1275 due to OpenBIOS not
+ ### implementing RTC, could this be the same for powerpc-ieee1275?
+ ### This test fails because the returned date is +12h, why??
+ powerpc-ieee1275:grub_cmd_date
+ powerpc64-ieee1275:grub_cmd_date
+ ### This is supposed to sleep for 10 seconds, but the returned date
+ ### before and after is a difference of 2 seconds. RTC issue??
+ powerpc-ieee1275:grub_cmd_sleep
+ powerpc64-ieee1275:grub_cmd_sleep
+"}
+
+EXPECTED_FUNCTIONAL_FAILURES=${EXPECTED_FUNCTIONAL_FAILURES:-"
+ ### These have not worked for a very long time
+ gfxterm_menu
+ cmdline_cat_test
+ ### Sometimes the machine the test are running on are slow causing
+ ### timing tests to fail.
+ sleep_test
+"}
+
+# Exports for build.sh
+export SRCDIR BUILDDIR INSTALLDIR ARCH PLATFORM CONFIGURE_OPTS \
+ FONT_SOURCE DJVU_FONT_SOURCE SHELL_TRACE CI_TYPE
+# Exports for test.sh
+export SRCDIR BUILDDIR TARGET TEST_ALL_TARGETS DISABLED_TESTS TESTS_TIMEOUT \
+ TEST_VERBOSITY TEST_DATA_PREFIX STRACE_TESTS SHELL_TRACE \
+ GRUB_SHELL_DEFAULT_TIMEOUT GRUB_TEST_DEFAULT_DEBUG \
+ IGNORE_TIMEDOUT_TEST_FAILURE CI_TYPE
+# Exports for process-tests.sh
+export SRCDIR BUILDDIR TARGET TEST_ALL_TARGETS DISABLED_TESTS \
+ EXPECTED_FAILURES EXPECTED_FUNCTIONAL_FAILURES FAIL_ON_HARD_ERRORS \
+ IGNORE_TIMEDOUT_TEST_FAILURE NUM_FAILED_LOG_LINES SHELL_TRACE CI_TYPE
+# Exports for make-images.sh
+export MAKE_ALL_IMAGE_TARGETS DISABLED_IMAGES CI_TYPE
+
+
+if [ -z "$EUID" ]; then
+ export EUID=$(id -u)
+fi
+
+
+##### Logging functions #####
TXT_RED="\e[31m"
TXT_YELLOW="\e[33m"
TXT_CLEAR="\e[0m"
@@ -30,4 +437,518 @@ function end_log() {
echo -e "End:${LOG_NAME}"
}
-:;
+
+##### Miscelaneous functions #####
+wait_anypid() {
+ while kill -SIGWINCH "$@" 2>/dev/null; do
+ sleep .25
+ done
+}
+
+helper_process_redirects() {
+ # Process redirects by starting from the last argument and continuing
+ # until the first, stopping when the argument being looked at does not
+ # fit the format of a redirection. Each redirect argument is removed
+ # from the exec arguments because the command is not expecting them.
+ # Each redirect found is saved in REDIRS until right before the exec
+ # so that they can be done all at once and not be potentially cluttered
+ # with output from this function.
+ local REDIRS= i="$#"
+ while [ "$i" -gt 0 ]; do
+ local LASTARG=$(eval echo "\${$i}")
+ if echo "$LASTARG" | grep -qE '^[0-9]*[<>].*'; then
+ REDIRS="$LASTARG $REDIRS"
+ i=$((i-1))
+ continue
+ fi
+ break
+ done
+
+ eval $(echo -n "set -- \$"; seq -s ' ' 1 $i | sed 's/ / $/g')
+ (
+ set +x
+ eval exec "$REDIRS"
+ exec "$@"
+ )
+}
+
+# multipipe is a less featureful version of the shell's native pipe, except
+# that stderr is piped into the next command in the pipe as fd 3.
+multipipe() {
+ local ADELIM=$'\x01'
+ local ERROR_ANY RET_PID RET_PID_NUM IFS_OLD CMD
+ local PIPEFILEOUT PIPEFILEERR PIPEFILEOUTPREV PIPEFILEERRPREV
+
+ while [ "$#" -gt 0 ]; do
+ case "$1" in
+ -e) ERROR_ANY=1; shift;;
+ -r) RET_PID_NUM="$2"; shift 2;;
+ *) break;;
+ esac
+ done
+
+ # For each command from last to first, start command with input and output
+ # as file descriptors to named pipes.
+ local i=0 max=$(echo "$*" | sed 's/|/\n/g' | wc -l) PID=
+ while read CMD; do
+ IFSOLD=$IFS
+ IFS=$ADELIM
+ set -- $(echo -n "$CMD")
+ if [ "$i" -eq 0 ]; then
+ PIPEFILEOUT=$(mktemp -u "${TMPDIR:-/tmp}"/multipipe.XXXXXXXXXX.fifo)
+ mkfifo $PIPEFILEOUT
+ PIPEFILEERR=$(mktemp -u "${TMPDIR:-/tmp}"/multipipe.XXXXXXXXXX.fifo)
+ mkfifo $PIPEFILEERR
+ helper_process_redirects "$@" <$PIPEFILEOUT 3<$PIPEFILEERR &
+ PID=$!
+ elif [ "$i" -eq "$((max-1))" ]; then
+ helper_process_redirects "$@" >$PIPEFILEOUT 2>$PIPEFILEERR &
+ PID=$!
+ rm -f $PIPEFILEOUT $PIPEFILEERR
+ else
+ PIPEFILEOUTPREV=$PIPEFILEOUT
+ PIPEFILEERRPREV=$PIPEFILEERR
+ PIPEFILEOUT=$(mktemp -u "${TMPDIR:-/tmp}"/multipipe.XXXXXXXXXX.fifo)
+ mkfifo $PIPEFILEOUT
+ PIPEFILEERR=$(mktemp -u "${TMPDIR:-/tmp}"/multipipe.XXXXXXXXXX.fifo)
+ mkfifo $PIPEFILEERR
+ (
+ # Need to run in subshell so we aren't overwriting file descriptors in
+ # the parent. Also we need to split up the redirects. First set stdout
+ # and stderr to the write ends of the pipes for the next command. We
+ # can not do this when calling helper_process_redirects as with the
+ # other redirects because helper_process_redirects is backgrounded
+ # causing a race with the following rm of those pipes, and the rm tends
+ # to win. We also can not open the read ends of the pipes for the
+ # previous command because the write end is not opened. So the exec
+ # which is not backgrounded would block, not allowing us to get to the
+ # point where the write ends of those pipes would be opened by the
+ # previous command in the pipeline.
+ exec >$PIPEFILEOUTPREV 2>$PIPEFILEERRPREV
+ helper_process_redirects "$@" <$PIPEFILEOUT 3<$PIPEFILEERR &
+ exit $!
+ )
+ PID=$?
+ rm -f $PIPEFILEOUTPREV $PIPEFILEERRPREV
+ fi
+
+ if [ "$ERROR_ANY" = "1" ]; then
+ RET_PID="$RET_PID $PID"
+ elif [ -z "$RET_PID_NUM" ] || [ "$RET_PID_NUM" -eq "$((max-$i))" ]; then
+ RET_PID=$PID
+ elif [ "$RET_PID_NUM" -eq "$((max-$i))" ]; then
+ RET_PID=$PID
+ fi
+
+ i=$((i+1))
+ IFS=$IFSOLD
+ done < <(IFS=$ADELIM; { echo -n "$*" | sed "s/${ADELIM}|${ADELIM}/\n/g";
echo; } | tac)
+
+ for PID in $RET_PID; do
+ wait -f $PID || return $?
+ done
+}
+
+multipipe_test() {
+ exec 2>/tmp/multipipe.trace.log
+ multipipe bash -x -c 'echo XXX $1' cmd arg1 '|' multitee 0:1,5 3:2,5
"5>/tmp/multipipe.file.log" ">/tmp/multipipe.stdout.log"
"2>/tmp/multipipe.stderr.log"
+ multipipe bash -x -c 'echo XXX $1; echo FD3 >&4' cmd arg1
"4>/tmp/multipipe.bash.stderr.log" \
+ '|' multitee 0:1 3:2 5:4 "4>/tmp/multipipe.1.multitee.file.log"
"5</etc/hosts" \
+ '|' multitee 0:1,5 3:2,5 "5>/tmp/multipipe.file.log"
">/tmp/multipipe.stdout.log" "2>/tmp/multipipe.stderr.log"
+}
+
+
+##### Functions for CI stages #####
+setup_distro() {
+ if [ ! -f /etc/os-release ]; then
+ echo "Could not determine OS version"
+ return
+ fi
+ . /etc/os-release
+ if [ "$ID" == "debian" ]; then
+ setup_debian
+ elif [ "$ID" == "ubuntu" ]; then
+ :
+ fi
+}
+
+setup_debian() {
+ if [ "$EUID" = 0 ]; then
+ sed -i 's/ main$/ main contrib non-free/' /etc/apt/sources.list
+ fi
+}
+
+setup_packages() {
+ start_log -c -n "setup_packages" "Installing required packages"
+ if ! which apt >/dev/null 2>&1; then
+ echo "No apt binary found, skipping package installation..."
+ return
+ fi
+ if [ "$EUID" != 0 ]; then
+ echo "Not root, continuing assuming that all needed packages are
installed..."
+ return
+ fi
+ apt $APT_OPTS update -yqq &&
+ apt $APT_OPTS install -y $(grep -vE "^\s*#" <<<"$PACKAGES" | sort -u)
+ end_log -n "setup_packages"
+}
+
+setup_repo() {
+ start_log -c -n "setup_repo" "Setting up source repository"
+ # First find out if we are being run from a checked out branch
+ if [ -n "$(cd ${SDIR%/*}; git log -n1 "${SDIR##*/}" 2>/dev/null)" ]; then
+ # The script dir is in a git repo, assume we want to use that repo's
+ # working dir, unless a specific branch and repo url were given
+ GIT_REPO_DEFAULT="${SDIR%/*}"
+ while [ ! -d "${GIT_REPO_DEFAULT}/.git" ]; do
+ GIT_REPO_DEFAULT="${GIT_REPO_DEFAULT%/*}"
+ done
+ GIT_REPO_DEFAULT="file://${GIT_REPO_DEFAULT}"
+ fi
+
+ if [ -d "$SRCDIR" ]; then
+ # If the source directory already exists, assume that we should be using
it.
+ # Make sure its up to date. Note, this can cause problems if the checked
out
+ # repo is not in a clean state.
+ git -C "$SRCDIR" pull
+ if [ -n "${GIT_BRANCH}" ]; then
+ git -C "$SRCDIR" checkout "${GIT_BRANCH}"
+ fi
+ elif [ -n "$GIT_REPO" ]; then
+ # If the source dir does not exist, and a remote repo was specified, then
+ # clone that repo
+ git clone $GIT_CLONE_ARGS "$GIT_REPO" -b
"${GIT_BRANCH:-$GIT_BRANCH_DEFAULT}" "$SRCDIR"
+ elif [ -d "$SRCDIR".external ]; then
+ # If a directory that is the source dir postfixed with .external exists
+ # then use that as the repo to clone from and use its gnulib dir.
+ git clone $GIT_CLONE_ARGS file://"${SRCDIR}.external" -b
"${GIT_BRANCH:-$GIT_BRANCH_DEFAULT}" "$SRCDIR"
+ [ -d "${SRCDIR}.external/gnulib" ] &&
+ git clone --depth=1 file://"${SRCDIR}.external/gnulib" "$SRCDIR"/gnulib
+ else
+ # If all else fails use the default, which is official master
+ git clone $GIT_CLONE_ARGS "$GIT_REPO_DEFAULT" -b
"${GIT_BRANCH:-$GIT_BRANCH_DEFAULT}" "$SRCDIR"
+ fi
+
+ export CI_COMMIT_SHORT_SHA=$(git -C "$SRCDIR" log -n1 --oneline | cut -d' '
-f1)
+
+ [ -f "${CI_SCRIPTS_DIR}/functions.${CI_TYPE}.sh" ] &&
+ . "${CI_SCRIPTS_DIR}/functions.${CI_TYPE}.sh"
+
+ end_log -n "setup_repo"
+}
+
+setup_build_scripts() {
+ if [ ! -d "${CI_SCRIPTS_DIR}" ]; then (
+ echo "No CI helper scripts found, retrieving from alternate repository"
+ cd "$SRCDIR"
+ git remote add ci "$GIT_CI_REPO"
+ # git ls-remote "$GIT_CI_REPO" "$GIT_CI_BRANCH"
+ git fetch ci "$GIT_CI_BRANCH"
+ git checkout "ci/$GIT_CI_BRANCH" -- scripts/ci
+ ) fi
+}
+
+setup_fonts() {
+ FONT_SOURCE=${FONT_SOURCE:-"/usr/share/fonts/X11/misc/unifont.pcf.gz"}
+
DJVU_FONT_SOURCE=${DJVU_FONT_SOURCE:-"/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf"}
+ export FONT_SOURCE DJVU_FONT_SOURCE
+}
+
+setup_loopback() {
+ if [ "$EUID" != 0 ]; then
+ echo "Insufficient privileges to create device files, skipping loopback
setup..."
+ return
+ fi
+ if [ ! -e /dev/loop-control ]; then
+ mknod -m 0660 /dev/loop-control c 10 237 || :
+ fi
+ for i in $(seq 0 64); do
+ [ -e "/dev/loop$i" ] ||
+ mknod -m 0660 "/dev/loop$i" b 7 "$i" || :
+ done
+}
+
+setup_cross_compilers() {
+ local RET= CROSS_TARGET="$1"
+ start_log -c -n "setup_cross_compilers" "Downloading/Installing Cross
Compilers for ${CROSS_TARGET}"
+ export HOST_ARCH=$(uname -m)
+ # These give us binaries like
$CROSS_DIR/gcc-8.1.0-nolibc/ia64-linux/bin/ia64-linux-gcc
+ export
CROSS_BIN_DIR="${CROSS_DIR}/gcc-${CROSS_VERSION}-nolibc/${CROSS_TARGET}/bin"
+ [ -d "${CROSS_BIN_DIR}" ] || {
+ # If not writable, fail immediately. This is stricter than we need to be.
+ test -w "$CROSS_DIR" &&
+ mkdir -pv "$CROSS_DIR" &&
+ wget -nv -t 3 -O -
"https://mirrors.edge.kernel.org/pub/tools/crosstool/files/bin/$HOST_ARCH/$CROSS_VERSION/$HOST_ARCH-gcc-$CROSS_VERSION-nolibc-$CROSS_TARGET.tar.xz";
| \
+ tar xJ -C "$CROSS_DIR" &&
+ test -d "$CROSS_BIN_DIR" || RET=$?
+ }
+ export PATH="${CROSS_BIN_DIR}:${PATH}"
+ end_log -n "setup_cross_compilers"
+ return ${RET:-0}
+}
+
+setup_ccache() {
+ if ! which ccache >/dev/null 2>&1; then
+ echo "No ccache binary found, skipping ccache setup..."
+ return
+ fi
+ # CCache Config
+ export CCACHE_DIR="${1:-${CCACHE_DIR:-${CI_PROJECT_DIR}/ccache}}"
+ export CCACHE_CONFIGPATH=${CCACHE_DIR}/ccache.conf
+ export CCACHE_HARDLINK=true
+ export CCACHE_TEMPDIR="${XDG_RUNTIME_DIR:-/tmp}"/.ccache.$UID
+ export CCACHE_LOGFILE="${CCACHE_DIR}/ccache.log"
+ export CC="ccache gcc"
+ mkdir -p "${CCACHE_DIR}/bin" &&
+ ccache --show-stats
+}
+
+setup_qemu() {
+ for DEB_URL in $DEB_URLS; do
+ local DEB_FILENAME="${DEB_URL##*/}"
+ local DEB_PATH="$PACKAGE_CACHE/archives/${DEB_FILENAME}"
+ if [ ! -f "$DEB_PATH" ]; then
+ wget -nv -P "$PACKAGE_CACHE/archives" "$DEB_URL"
+ fi
+ [ "$EUID" = 0 ] && ! dpkg -l "${DEB_FILENAME%%_*}" >/dev/null 2>&1 &&
+ dpkg -i "$DEB_PATH"
+ done
+ # TODO: i386-efi: Unfortunately the non-Ubuntu debian package
+ # ovmf-ia32_2020.11-2_all.deb installs OVMF images that are not usable by
+ # QEMU's -bios option. This is the most official looking build found. Its
+ # probably possible to modify grub-shell to use successfully the files
+ # installed by ovmf-ia32 by using the QEMU option -drive if=pflash,...[1].
+ # I've submitted a bug report to debian to proposing that the file acceptable
+ # via -bios is added to the package.
+ # [1] https://wiki.debian.org/SecureBoot/VirtualMachine
+ [ -f "${SRCDIR}/OVMF-ia32.fd" ] ||
+ wget -nv -t 3 -O "${SRCDIR}/OVMF-ia32.fd" \
+ https://github.com/retrage/edk2-nightly/raw/master/bin/RELEASEIa32_OVMF.fd
||:
+ cp -vau /usr/share/qemu-efi-aarch64/QEMU_EFI.fd "${SRCDIR}/OVMF-aarch64.fd"
||:
+ cp -vau /usr/share/AAVMF/AAVMF32_CODE.fd "${SRCDIR}/OVMF-arm.fd" ||:
+}
+
+setup_bootstrap() {
+ local RET
+ start_log -c -n "bootstrap" "Bootstrap log"
+ # We cache the bootstrap, so if the bootstrap canary exists, don't run the
bootstrap
+ if [ ! -f "$SRCDIR"/.bootstrap.finished ]; then
+ ( cd "$SRCDIR" && ./bootstrap ) &&
+ touch "$SRCDIR"/.bootstrap.finished || RET=1
+ fi
+ end_log -n "bootstrap"
+ return ${RET:-0}
+}
+
+run_matrix() {
+ local RET=0 MATRIX_ROW
+ for MATRIX_ROW in $(echo "${MATRIX}" | grep -vE '^\s*#'); do
+ run_matrix_row "$MATRIX_ROW" || RET=$?
+ done
+ return ${RET:-0}
+}
+
+run_matrix_row() {
+ local RET=0 MATRIX_ROW="$1"
+ if [ -n "$MATRIX_ROW" ]; then
+ export ARCH="${MATRIX_ROW%%:*}"
+ export GRUB_TARGETS=$(IFS=:; echo "${MATRIX_ROW}" | (read A G C; echo
"$G"))
+ export CROSS_TARGET="${MATRIX_ROW##*:}"
+ fi
+
+ setup_cross_compilers "${CROSS_TARGET}" || return $?
+
+ for TARGET in $(echo "$GRUB_TARGETS" | tr , ' '); do
+ run_target "$TARGET" || RET=$?
+ if [ ${RET:-0} -ne 0 ]; then
+ echo -e "${TXT_RED}${TARGET} failed, continuing to next
target.${TXT_CLEAR}"
+ fi
+ done
+ return ${RET:-0}
+}
+
+run_target() {
+ local RET=0
+ setup_env &&
+ (
+ run_build &&
+ run_test
+ ) || RET=$?
+
+ # Do processing of testing output
+ "${CI_SCRIPTS_DIR}/process-tests.sh" || RET=$?
+ # Make images for all formats
+ "${CI_SCRIPTS_DIR}/make-images.sh" || RET=$?
+
+ return ${RET:-0}
+}
+
+setup_env() {
+ local TARGET="${1:-${TARGET}}"
+ export BUILDDIR=$CI_PROJECT_DIR/build/obj-$TARGET
+ export INSTALLDIR=$CI_PROJECT_DIR/install/grub-$TARGET
+ export TEST_DATA_PREFIX=$CI_PROJECT_DIR/build/obj-${TARGET}/test-data
+
+ mkdir -pv "${BUILDDIR}" "${INSTALLDIR}"
+
+ export ARCH=${TARGET%-*}
+ export PLATFORM=${TARGET#*-}
+ if [ "x${BUILD_ALL_TARGETS}" != "xy" ] && ( echo "${DISABLED_BUILDS}" | grep
-qE "^\s*${TARGET}$" ); then
+ echo -e "${TXT_CLEAR}${TXT_YELLOW}Building $TARGET has been disabled,
skipping.${TXT_CLEAR}"
+ return
+ fi
+}
+
+run_build() {
+ local RET=0
+ if [ "x${DISABLE_ALL_BUILDS}" = "xy" ]; then
+ echo "All builds are disabled"
+ return
+ fi
+
+ case "$ARCH" in
+ arm64) ARCH=aarch64-linux;;
+ arm) ARCH=arm-linux-gnueabi;;
+ mips) ARCH=mips64-linux;;
+ ia64|powerpc*|riscv32|riscv64|sparc*) ARCH=$ARCH-linux;;
+ i386|x86_64) ;;
+ *) echo "Unknown architecture":" $ARCH"; return 1;;
+ esac
+ case "$PLATFORM" in
+ coreboot)
+ export GRUB_CBFSTOOL="/where/is/cbfstool"
+ export GRUB_COREBOOT_ROM="/where/is/coreboot/rom";;
+ esac
+
+ cat >${BUILDDIR}/shell-wrapper.sh <<EOF
+#!$SHELL
+if [ "x\${SHELL_TRACE}" = "xy" ]; then
+ set -x
+ if [ -z "\$SHELL_OPTS" ] || [ -n "\${SHELL_OPTS##* -x*}" ]; then
+ SHELL_OPTS="\$SHELL_OPTS -x"
+ fi
+fi
+REAL_SHELL=\${REAL_SHELL:-\$(readlink /proc/\$\$/exe)}
+exec \$REAL_SHELL \$SHELL_OPTS "\$@"
+EOF
+ chmod +x ${BUILDDIR}/shell-wrapper.sh
+ export CONFIG_SHELL=${BUILDDIR}/shell-wrapper.sh
+ export CONFIGURE_OPTS="${CONFIGURE_OPTS}
CONFIG_SHELL=${BUILDDIR}/shell-wrapper.sh"
+
+ if type multipipe >/dev/null; then
+ multipipe -e "${CI_SCRIPTS_DIR}/build.sh" \
+ '|' multitee 0:1,5 3:2,5 "5>${BUILDDIR}/build.log" || RET=$?
+ else
+ { { "${CI_SCRIPTS_DIR}/build.sh" || RET=$?; } |
+ tee >(exec cat >&3) >"${BUILDDIR}/build.log" 2>&1; } 3>&1 || :
+ fi
+ return $RET
+}
+
+run_test() {
+ local RET=0
+ if [ "x${DISABLE_ALL_TESTS}" = "xy" ]; then
+ echo "All tests are disabled"
+ return
+ fi
+
+ export TESTTMPDIR="${TMPDIR:-/tmp}/grub-test-$TARGET-${CI_COMMIT_SHORT_SHA}"
+ if type multipipe >/dev/null; then
+ multipipe -e "${CI_SCRIPTS_DIR}/test.sh" \
+ '|' multitee 0:1,5 3:2,5 "5>${BUILDDIR}/test.log" || RET=$?
+ else
+ { { "${CI_SCRIPTS_DIR}/test.sh" || RET=$?; } |
+ tee >(exec cat >&3) >"${BUILDDIR}/test.log" 2>&1; } 3>&1 || :
+ fi
+ return $RET
+}
+
+post_processing() {
+ local RET=0 SUMLOG="${CI_PROJECT_DIR}/build/test-summary.log"
+ : >"${SUMLOG}"
+ for F in "${CI_PROJECT_DIR}"/build/obj-*/test-results.log; do
+ [ ! -e "$F" ] && continue
+ local TARGETDIR="${F%/*}"
+ local TARGET="${TARGETDIR##*/}"
+ TARGET="${TARGET#*-}"
+ if ! echo "${MATRIX}" | grep -vE "^\s*#" | grep -qE "${TARGET}"; then
+ # Skip processing if the TARGET is not one of the ones in the MATRIX
+ continue
+ fi
+ echo "Ran test suite for target: ${TARGET}"
+ start_log -n "test-$TARGET" "Ran test suite for target: ${TARGET}"
>>${SUMLOG}
+ grep "^${TARGET}:" "${TARGETDIR}/test-results.log" | sort |
+ ( IFS=:;
+ while read TARGET FTYPE TESTNAME; do
+ case "$FTYPE" in
+ skip)
+ echo -e "${TXT_CLEAR}${TXT_YELLOW} Skipped test:
${TESTNAME}${TXT_CLEAR}" >>${SUMLOG};;
+ error)
+ if [ "x${FAIL_ON_HARD_ERRORS}" = "xy" ]; then
+ RET=1
+ fi
+ echo -e "${TXT_CLEAR}${TXT_YELLOW} Hard error in test:
${TESTNAME}${TXT_CLEAR}" >>${SUMLOG};;
+ expected)
+ echo -e "${TXT_CLEAR}${TXT_YELLOW} Expected failed test:
${TESTNAME}${TXT_CLEAR}" >>${SUMLOG};;
+ timeout)
+ if [ "x${IGNORE_TIMEDOUT_TEST_FAILURE}" != "xy" ]; then
+ RET=1
+ TEST_IGNORED=" ignored"
+ fi
+ echo -e "${TXT_CLEAR}${TXT_YELLOW} Timed-out test${TEST_IGNORED}:
${TESTNAME}${TXT_CLEAR}" >>${SUMLOG};;
+ unexpected)
+ RET=1
+ echo -e "${TXT_CLEAR}${TXT_RED} Failed test:
${TESTNAME}${TXT_CLEAR}" >>${SUMLOG};;
+ *)
+ RET=1
+ echo -e "${TXT_CLEAR}${TXT_RED} Unhandled test type for
${TESTNAME} (type=$FTYPE)${TXT_CLEAR}" >>${SUMLOG};;
+ esac
+ done
+ exit ${RET:-0}
+ ) || RET=$?
+ echo -e -n "${TXT_CLEAR}" >>${SUMLOG}
+ end_log -n "test-$TARGET" >>${SUMLOG}
+ done
+ if [ -f "${SUMLOG}" ]; then
+ cat "${SUMLOG}"
+ fi
+ return ${RET:-0}
+}
+
+setup_all() {
+ setup_distro &&
+ setup_packages &&
+ setup_repo &&
+ setup_build_scripts &&
+ setup_fonts &&
+ setup_loopback &&
+ setup_ccache &&
+ setup_qemu &&
+ setup_bootstrap
+}
+
+all() {
+ setup_all && {
+ run_matrix ||:
+ post_processing
+ }
+}
+
+main() {
+ local CMD
+ mkdir -p "$ROOT" "$TMPDIR"
+ cd "$ROOT"
+
+ if [ -z "$*" ]; then
+ all
+ fi
+
+ for CMD in "$@"; do
+ $CMD
+ done
+}
+
+# If found source helper functions based on CI type. Do this here allows the
+# CI type functions to override these general default ones.
+[ -f "${CI_SCRIPTS_DIR}/functions.${CI_TYPE}.sh" ] &&
+. "${CI_SCRIPTS_DIR}/functions.${CI_TYPE}.sh"
--
2.27.0
_______________________________________________
Grub-devel mailing list
Grub-devel@gnu.org
https://lists.gnu.org/mailman/listinfo/grub-devel
