Creates different Docker images containing Mesa built with different tools (autotools, meson, scons, etc).
The build is done in 3 levels: the first level creates a base image with all the requirements to build Mesa. The second level (based of the first one), builds different images with different versions of LLVM. As Gallium drivers heavily relies on LLVM, this will help to test the build with different LLVM versions. Finally, the latest level creates different images versions of Mesa. The main differences is the tool to build them: autotools, meson, scons, building Gallium drivers with different LLVM versions, and so on. As the purpose is just to test that everything can be built correctly, all the images are discarded, except one (the autotools), which is stored in the registry. Thus, anyone can just pull it locally and test against their local system. In order to build the images, Rocker is used. This is a tool that extends the Dockerfiles with new features that are quite interested here. The main features we use is the support for templating, and the support for mounting external directories during the image building. This help to use tools like ccache to improve the build speed. Signed-off-by: Juan A. Suarez Romero <jasua...@igalia.com> --- .gitlab-ci.yml | 177 +++++++++++++++++++++++++++++++++ gitlab-ci/Rockerfile.base | 199 ++++++++++++++++++++++++++++++++++++++ gitlab-ci/Rockerfile.llvm | 57 +++++++++++ gitlab-ci/Rockerfile.mesa | 145 +++++++++++++++++++++++++++ 4 files changed, 578 insertions(+) create mode 100644 .gitlab-ci.yml create mode 100644 gitlab-ci/Rockerfile.base create mode 100644 gitlab-ci/Rockerfile.llvm create mode 100644 gitlab-ci/Rockerfile.mesa diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 00000000000..5cee333dd45 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,177 @@ +image: docker:latest + +services: + - docker:dind + +stages: + - base + - llvm + - mesa + +variables: + DOCKER_IMAGE: $CI_REGISTRY_IMAGE + CCACHE_DIR: $CI_PROJECT_DIR/../ccache + LLVM: "6.0" + +cache: + paths: + - ccache/ + key: "$CI_JOB_STAGE" + +before_script: + - mkdir -p ccache + - rm -fr ../ccache + - mv ccache ../ + - export MAKEFLAGS=-j$(nproc) + - apk --no-cache add libc6-compat + - wget https://github.com/grammarly/rocker/releases/download/1.3.1/rocker-1.3.1-linux_amd64.tar.gz + - tar xvf rocker-1.3.1-linux_amd64.tar.gz + - rm rocker-1.3.1-linux_amd64.tar.gz + - mv rocker .. + - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY + +after_script: + - mv ../ccache ./ + +.build_llvm: &build_llvm + stage: llvm + cache: {} + script: + - ../rocker build -f gitlab-ci/Rockerfile.llvm --var LLVM=$LLVM + - docker push $CI_REGISTRY_IMAGE:llvm-$LLVM + +.build_mesa: &build_mesa + stage: mesa + script: + - ../rocker build -f gitlab-ci/Rockerfile.mesa --var BUILD=$BUILD --var LLVM=$LLVM --var TAG=$CI_COMMIT_REF_SLUG . + +base: + stage: base + script: + - DOCKERFILE_SHA256=$(cat gitlab-ci/Rockerfile.base | sha256sum | cut -c-64) + - IMAGE_DOCKERFILE_SHA256=$(./gitlab-ci/inspect-remote-image.sh gitlab-ci-token $CI_BUILD_TOKEN $CI_PROJECT_PATH "base" ".config.Labels[\"dockerfile.sha256\"]" || echo -n "<notfound>") + - if [ "$DOCKERFILE_SHA256" != "$IMAGE_DOCKERFILE_SHA256" ] ; then FORCE_BUILD=true ; fi + - if [ "$FORCE_BUILD" ] ; then ../rocker build -f gitlab-ci/Rockerfile.base --var DOCKERFILE_SHA256=$DOCKERFILE_SHA256 ; fi + - if [ "$FORCE_BUILD" ] ; then docker push $CI_REGISTRY_IMAGE:base ; fi + +llvm:3.3: + variables: + LLVM: "3.3" + <<: *build_llvm + +llvm:3.6: + variables: + LLVM: "3.6" + <<: *build_llvm + +llvm:3.8: + variables: + LLVM: "3.8" + <<: *build_llvm + +llvm:3.9: + variables: + LLVM: "3.9" + <<: *build_llvm + +llvm:4.0: + variables: + LLVM: "4.0" + <<: *build_llvm + +llvm:5.0: + variables: + LLVM: "5.0" + <<: *build_llvm + +llvm:6.0: + variables: + LLVM: "6.0" + <<: *build_llvm + +meson: + variables: + BUILD: "meson" + <<: *build_mesa + +autotools: + variables: + BUILD: "autotools" + after_script: + - mv ../ccache ./ + - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG + <<: *build_mesa + +distcheck: + variables: + BUILD: "distcheck" + artifacts: + name: "release" + paths: + - release-output/ + expire_in: 1 week + <<: *build_mesa + +gallium:llvm:3.3: + variables: + BUILD: "gallium" + LLVM: "3.3" + <<: *build_mesa + +gallium:llvm:3.6: + variables: + BUILD: "gallium" + LLVM: "3.6" + <<: *build_mesa + +gallium:llvm:3.8: + variables: + BUILD: "gallium" + LLVM: "3.8" + <<: *build_mesa + +gallium:llvm:3.9: + variables: + BUILD: "gallium" + LLVM: "3.9" + <<: *build_mesa + +gallium:llvm:4.0: + variables: + BUILD: "gallium" + LLVM: "4.0" + <<: *build_mesa + +gallium:llvm:5.0: + variables: + BUILD: "gallium" + LLVM: "5.0" + <<: *build_mesa + +gallium:llvm:6.0: + variables: + BUILD: "gallium" + LLVM: "6.0" + <<: *build_mesa + +gallium:nollvm: + variables: + BUILD: "gallium" + LLVM: "0.0" + <<: *build_mesa + +scons: + variables: + BUILD: "scons" + <<: *build_mesa + +scons:nollvm: + variables: + BUILD: "scons" + LLVM: "0.0" + <<: *build_mesa + +windows: + variables: + BUILD: "windows" + <<: *build_mesa diff --git a/gitlab-ci/Rockerfile.base b/gitlab-ci/Rockerfile.base new file mode 100644 index 00000000000..a0cb5e5290d --- /dev/null +++ b/gitlab-ci/Rockerfile.base @@ -0,0 +1,199 @@ +# +# Base image for building Mesa. +# +# ~~~ +# rocker build -f Rockerfile.base [--attach] [--pull] +# ~~~ +# +# Environment variables that are used in the build: +# - DOCKER_IMAGE: name of the final image to be tagged (default: mesa:base) +# - MAKEFLAGS: flags to pass to make (e.g., "-j8") +# - CCACHE_DIR: ccache directory (e.g, ~/.ccache) +# + +{{ $image := (or .Env.DOCKER_IMAGE "mesa") }} + +FROM ubuntu:xenial + +LABEL maintainer "Juan A. Suarez Romero <jasua...@igalia.com>" + +ENV LC_ALL=C.UTF-8 + +RUN apt-get update \ + && apt-get --no-install-recommends -y install autoconf gcc g++ sudo cmake patch git \ + automake pkg-config libtool-bin bison flex python-pip libpthread-stubs0-dev wget \ + libxau-dev libx11-dev libxext-dev libxdamage-dev libx11-xcb-dev gettext xutils-dev \ + zlib1g-dev scons libelf-dev libxvmc-dev libvdpau-dev libva-dev libclc-dev ccache \ + libpciaccess-dev libxxf86vm-dev python-setuptools python-wheel bzip2 make pciutils \ + mesa-common-dev libexpat1-dev xz-utils libedit-dev libffi-dev libunwind-dev \ + libxml2-dev libomxil-bellagio-dev mesa-utils waffle-utils vulkan-utils python3-pip \ + python3-setuptools libgtest-dev \ + && rm -fr /var/lib/apt/lists/* + +RUN pip install mako + +RUN pip3 install meson ninja + +RUN adduser --gecos "" local && passwd -d local && adduser local sudo + +USER local + +WORKDIR /home/local + +ENV LDFLAGS="-L/usr/local/lib $LDFLAGS" + +{{ if .Env.MAKEFLAGS }} +ENV MAKEFLAGS={{ .Env.MAKEFLAGS }} +{{ end }} + +{{ if .Env.CCACHE_DIR }} +MOUNT {{ .Env.CCACHE_DIR }}:/home/local/.ccache:Z +RUN sudo chown -R local:local /home/local/.ccache +ENV PATH=/usr/lib/ccache:$PATH +{{ end }} + +ATTACH [ "/bin/bash" ] + +RUN wget https://xorg.freedesktop.org/releases/individual/proto/glproto-1.4.14.tar.bz2 \ + && tar -jxvf glproto-1.4.14.tar.bz2 \ + && rm glproto-1.4.14.tar.bz2 \ + && cd glproto-1.4.14 \ + && ./configure \ + && make \ + && sudo make install \ + && sudo ldconfig \ + && sudo rm -fr ../glproto-1.4.14 + +RUN wget https://xorg.freedesktop.org/releases/individual/proto/dri2proto-2.8.tar.bz2 \ + && tar -jxvf dri2proto-2.8.tar.bz2 \ + && rm dri2proto-2.8.tar.bz2 \ + && cd dri2proto-2.8 \ + && ./configure \ + && make \ + && sudo make install \ + && sudo ldconfig \ + && sudo rm -fr ../dri2proto-2.8 + +RUN wget https://xorg.freedesktop.org/releases/individual/proto/dri3proto-1.0.tar.bz2 \ + && tar -jxvf dri3proto-1.0.tar.bz2 \ + && rm dri3proto-1.0.tar.bz2 \ + && cd dri3proto-1.0 \ + && ./configure \ + && make \ + && sudo make install \ + && sudo ldconfig \ + && sudo rm -fr ../dri3proto-1.0 + +RUN wget https://xorg.freedesktop.org/releases/individual/proto/presentproto-1.0.tar.bz2 \ + && tar -jxvf presentproto-1.0.tar.bz2 \ + && rm presentproto-1.0.tar.bz2 \ + && cd presentproto-1.0 \ + && ./configure \ + && make \ + && sudo make install \ + && sudo ldconfig \ + && sudo rm -fr ../presentproto-1.0 + +RUN wget https://xcb.freedesktop.org/dist/xcb-proto-1.13.tar.bz2 \ + && tar -jxvf xcb-proto-1.13.tar.bz2 \ + && rm xcb-proto-1.13.tar.bz2 \ + && cd xcb-proto-1.13 \ + && ./configure \ + && make \ + && sudo make install \ + && sudo ldconfig \ + && sudo rm -fr ../xcb-proto-1.13 + +RUN wget https://xcb.freedesktop.org/dist/libxcb-1.13.tar.bz2 \ + && tar -jxvf libxcb-1.13.tar.bz2 \ + && rm libxcb-1.13.tar.bz2 \ + && cd libxcb-1.13 \ + && ./configure \ + && make \ + && sudo make install \ + && sudo ldconfig \ + && sudo rm -fr ../libxcb-1.13 + +RUN wget https://xorg.freedesktop.org//releases/individual/proto/renderproto-0.11.1.tar.bz2 \ + && tar -jxvf renderproto-0.11.1.tar.bz2 \ + && rm renderproto-0.11.1.tar.bz2 \ + && cd renderproto-0.11.1 \ + && ./configure \ + && make \ + && sudo make install \ + && sudo ldconfig \ + && sudo rm -fr ../renderproto-0.11.1 + +RUN wget https://xorg.freedesktop.org/releases/individual/lib/libXrender-0.9.10.tar.bz2 \ + && tar -jxvf libXrender-0.9.10.tar.bz2 \ + && rm libXrender-0.9.10.tar.bz2 \ + && cd libXrender-0.9.10 \ + && ./configure \ + && make \ + && sudo make install \ + && sudo ldconfig \ + && sudo rm -fr ../libXrender-0.9.10 + +RUN wget https://xorg.freedesktop.org/releases/individual/proto/randrproto-1.3.0.tar.bz2 \ + && tar -jxvf randrproto-1.3.0.tar.bz2 \ + && rm randrproto-1.3.0.tar.bz2 \ + && cd randrproto-1.3.0 \ + && ./configure \ + && make \ + && sudo make install \ + && sudo ldconfig \ + && sudo rm -fr ../randrproto-1.3.0 + +RUN wget https://xorg.freedesktop.org/releases/individual/lib/libXrandr-1.3.0.tar.bz2 \ + && tar -jxvf libXrandr-1.3.0.tar.bz2 \ + && rm libXrandr-1.3.0.tar.bz2 \ + && cd libXrandr-1.3.0 \ + && ./configure \ + && make \ + && sudo make install \ + && sudo ldconfig \ + && sudo rm -fr ../libXrandr-1.3.0 + +RUN wget https://xorg.freedesktop.org/releases/individual/lib/libxshmfence-1.1.tar.bz2 \ + && tar -jxvf libxshmfence-1.1.tar.bz2 \ + && rm libxshmfence-1.1.tar.bz2 \ + && cd libxshmfence-1.1 \ + && ./configure \ + && make \ + && sudo make install \ + && sudo ldconfig \ + && sudo rm -fr ../libxshmfence-1.1 + +RUN wget https://people.freedesktop.org/~cbrill/libtxc_dxtn/libtxc_dxtn-1.0.1.tar.bz2 \ + && tar -jxvf libtxc_dxtn-1.0.1.tar.bz2 \ + && rm libtxc_dxtn-1.0.1.tar.bz2 \ + && cd libtxc_dxtn-1.0.1 \ + && ./configure \ + && make \ + && sudo make install \ + && sudo ldconfig \ + && sudo rm -fr ../libtxc_dxtn-1.0.1 + +RUN wget https://wayland.freedesktop.org/releases/wayland-1.15.0.tar.xz \ + && tar -Jxvf wayland-1.15.0.tar.xz \ + && rm wayland-1.15.0.tar.xz \ + && cd wayland-1.15.0 \ + && ./configure --disable-documentation \ + && make \ + && sudo make install \ + && sudo ldconfig \ + && sudo rm -fr ../wayland-1.15.0 + +RUN wget https://wayland.freedesktop.org/releases/wayland-protocols-1.8.tar.xz \ + && tar -Jxvf wayland-protocols-1.8.tar.xz \ + && rm wayland-protocols-1.8.tar.xz \ + && cd wayland-protocols-1.8 \ + && ./configure \ + && make \ + && sudo make install \ + && sudo ldconfig \ + && sudo rm -fr ../wayland-protocols-1.8 + +USER root + +TAG {{ $image }}:base diff --git a/gitlab-ci/Rockerfile.llvm b/gitlab-ci/Rockerfile.llvm new file mode 100644 index 00000000000..ed75f4f0f02 --- /dev/null +++ b/gitlab-ci/Rockerfile.llvm @@ -0,0 +1,57 @@ +# +# This builds the base image for building Mesa. +# +# ~~~ +# rocker build -f Rockerfile.llvm [--pull] \ +# --var LLVM=3.3 # 3.3, 3.6, 3.8, 3.9, 4.0, 5.0, 6.0 +# ~~~ +# +# Environment variables that are used in the build: +# - DOCKER_IMAGE: name of the final image to be tagged (default: mesa:$LLVM) +# +# NOTE: LLVM 3.3 package has been created with checkinstall using Base image +# +# ~~~ +# wget https://releases.llvm.org/3.3/llvm-3.3.src.tar.gz \ +# && tar xzpf llvm-3.3.src.tar.gz \ +# && cd llvm-3.3.src \ +# && mkdir build \ +# && cd build \ +# && ../configure --enable-optimized --enable-shared --prefix=/usr/lib/llvm-3.3 \ +# && make \ +# && echo "LLVM Toolchain 3.3" > description-pak \ +# && checkinstall -D -y --pkgname=llvm-3.3 --pkgversion=3.3 --pkgrelease=+checkinstall1 --maintainer=jasua...@igalia.com --provides=llvm +# ~~~ +# + +{{ $image := (or .Env.DOCKER_IMAGE "mesa") }} +{{ $distro := "xenial" }} + +FROM {{ $image }}:base + +LABEL maintainer "Juan A. Suarez Romero <jasua...@igalia.com>" + +{{ if ge .LLVM "3.9" }} +RUN apt-get update \ + && apt-get --no-install-recommends -y install gnupg \ + && wget -qO - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - \ + && echo "deb http://apt.llvm.org/{{ $distro }}/ llvm-toolchain-{{ $distro }}-{{ .LLVM }} main" >> /etc/apt/sources.list \ + && rm -fr /var/lib/apt/lists/* +{{ end }} + + +{{ if eq .LLVM "3.3" }} +RUN wget https://people.igalia.com/jasuarez/packages/llvm-3.3_3.3-+checkinstall1_amd64.deb \ + && dpkg -i llvm-3.3_3.3-+checkinstall1_amd64.deb \ + && rm llvm-3.3_3.3-+checkinstall1_amd64.deb +ENV LD_LIBRARY_PATH=/usr/lib/llvm-{{ .LLVM }}/lib:$LD_LIBRARY_PATH + +{{ else }} + +RUN apt-get update \ + && apt-get --no-install-recommends -y install libclang-{{ .LLVM }}-dev \ + llvm-{{ .LLVM }}-dev \ + && rm -fr /var/lib/apt/lists/* +{{ end }} + +TAG {{ $image }}:llvm-{{ .LLVM }} diff --git a/gitlab-ci/Rockerfile.mesa b/gitlab-ci/Rockerfile.mesa new file mode 100644 index 00000000000..7b52e3e3a74 --- /dev/null +++ b/gitlab-ci/Rockerfile.mesa @@ -0,0 +1,145 @@ +# +# This builds and installs Mesa. +# +# ~~~ +# rocker build -f Rockerfile.mesa [--attach] [--pull] \ +# --var BUILD=autotools # meson, scons, autotools, windows, distcheck, gallium \ +# [--var DEBUG=true] # build with debug options (true, false) \ +# [--var LLVM=3.3] # 3.3, 3.6, 3.8, 3.9, 4.0, ... \ +# [--var TAG=master] # master, staging-18.0, staging-18.1, ... \ +# ~~~ +# +# Environment variables that are used in the build: +# - DOCKER_IMAGE: name of the final image to be tagged (default: mesa[:$TAG]) +# - MAKEFLAGS: flags to pass to make (e.g., "-j8") +# - CCACHE_DIR: ccache directory (e.g., ~/.ccache) +# + +{{ $image := (or .Env.DOCKER_IMAGE "mesa") }} +{{ $llvm_version := (or .LLVM "0.0") }} +{{ $debug_build := (or .DEBUG "true") }} + +{{ if eq .BUILD "windows"}} +FROM {{ $image }}:base +{{ else if eq $llvm_version "0.0" }} +FROM {{ $image }}:base +{{ else }} +FROM {{ $image }}:llvm-{{ .LLVM }} +{{ end }} + +LABEL maintainer "Juan A. Suarez Romero <jasua...@igalia.com>" + +{{ if eq .BUILD "windows" }} +RUN apt-get update \ + && apt-get -y --no-install-recommends install mingw-w64 \ + && rm -fr /var/lib/apt/lists/* +{{ end }} + +USER local + +ENV PATH=/home/local/.bin/:/usr/lib/llvm-{{ $llvm_version }}/bin:$PATH + +ADD . /home/local/mesa +RUN sudo chown -R local:local /home/local/mesa + +{{ if .Env.MAKEFLAGS }} +ENV MAKEFLAGS={{ .Env.MAKEFLAGS }} +ENV SCONSFLAGS={{ .Env.MAKEFLAGS }} +{{ end }} + +{{ if .Env.CCACHE_DIR }} +MOUNT {{ .Env.CCACHE_DIR }}:/home/local/.ccache:Z +RUN sudo chown -R local:local /home/local/.ccache +ENV PATH=/usr/lib/ccache:$PATH +ENV USE_CCACHE=1 +{{ end }} + +WORKDIR /home/local/mesa + +RUN if [ -d .git ] ; then git show --stat > /home/local/mesa-head.txt ; else echo "FROM TARBALL" > /home/local/mesa-head.txt ; fi + +ATTACH [ "/bin/bash" ] + +RUN export DRM_VERSION=`cat /home/local/mesa/configure.ac | egrep ^LIBDRM.*REQUIRED| cut -f2 -d= | sort -nr | head -n 1` \ + && wget https://dri.freedesktop.org/libdrm/libdrm-$DRM_VERSION.tar.bz2 \ + && tar -jxvf libdrm-$DRM_VERSION.tar.bz2 \ + && rm libdrm-$DRM_VERSION.tar.bz2 \ + && cd libdrm-$DRM_VERSION \ + && ./configure --enable-freedreno --enable-vc4 --enable-etnaviv-experimental-api --enable-tegra-experimental-api \ + && make \ + && sudo make install \ + && sudo ldconfig \ + && sudo rm -fr ../libdrm-$DRM_VERSION \ + && unset DRM_VERSION + + +{{ if eq .BUILD "scons" }} +{{ if eq $llvm_version "0.0" }} +RUN scons llvm=0 \ + && scons llvm=0 check \ + && sudo rm -fr /home/local/mesa +{{ else }} +RUN scons llvm=1 \ + && scons llvm=1 check \ + && sudo rm -fr /home/local/mesa +{{ end }} + +{{ else if eq .BUILD "windows" }} + +RUN scons platform=windows toolchain=crossmingw \ + && sudo rm -fr /home/local/mesa + +{{ else if eq .BUILD "distcheck" }} + +MOUNT .:/context +RUN ./autogen.sh \ + && make distcheck \ + && __version=`cat VERSION` \ + && mkdir -p /context/release-output \ + && mv /home/local/mesa-head.txt /context/release-output \ + && mv mesa-$__version.tar.xz /context/release-output \ + && sudo rm -fr /home/local/mesa + +{{ else if eq .BUILD "autotools" "gallium" }} + +RUN export LLVM={{ $llvm_version }}.0\ + && eval `cat configure.ac | egrep ^LLVM_REQUIRED` \ + && if dpkg --compare-versions $LLVM ge $LLVM_REQUIRED_GALLIUM ; then GALLIUM_DRIVERS=i915,etnaviv,freedreno,imx,nouveau,pl111,r300,svga,swrast,tegra,v3d,vc4,virgl ; fi \ + && if dpkg --compare-versions $LLVM ge $LLVM_REQUIRED_R600 ; then GALLIUM_DRIVERS=$GALLIUM_DRIVERS,r600 ; fi \ + && if dpkg --compare-versions $LLVM ge $LLVM_REQUIRED_RADEONSI ; then GALLIUM_DRIVERS=$GALLIUM_DRIVERS,radeonsi ; fi \ + && if dpkg --compare-versions $LLVM ge $LLVM_REQUIRED_SWR ; then GALLIUM_DRIVERS=$GALLIUM_DRIVERS,swr ; fi \ + && DRI_DRIVERS=i915,i965,nouveau,r200,radeon,swrast \ + && VULKAN_DRIVERS=intel \ + && if dpkg --compare-versions $LLVM ge $LLVM_REQUIRED_RADV ; then VULKAN_DRIVERS=$VULKAN_DRIVERS,radeon ; fi \ + && ./autogen.sh \ + --with-gallium-drivers=$GALLIUM_DRIVERS \ + {{ if eq .BUILD "gallium" }} --with-dri-drivers="" \ + {{ else }} --with-dri-drivers=$DRI_DRIVERS \ + --with-vulkan-drivers=$VULKAN_DRIVERS \ + --with-egl-platforms=x11,drm,wayland {{ end }} \ + {{ if ne $llvm_version "0.0" }} --enable-llvm --enable-llvm-shared-libs {{ end }} \ + {{ if ne $debug_build "false" }} --enable-debug {{ end }} \ + --enable-glx-tls --enable-gbm --enable-egl \ + && make \ + && make check \ + && sudo make install \ + && sudo ldconfig \ + && sudo rm -fr /home/local/mesa + +{{ else if eq .BUILD "meson" }} + +RUN meson _build \ + && ninja -C _build \ + && sudo ninja -C _build install \ + && sudo ldconfig \ + && sudo rm -fr /home/local/mesa + +{{ end }} + +WORKDIR /home/local + +USER root + +{{ if .TAG }} +TAG {{ $image }}:{{ .TAG }} +{{ end }} -- 2.17.1 _______________________________________________ mesa-dev mailing list mesa-dev@lists.freedesktop.org https://lists.freedesktop.org/mailman/listinfo/mesa-dev
