On Fri, May 29, 2020 at 03:00:42PM +0200, Andrea Bolognani wrote: > Instead of using pre-built containers hosted on Quay, build > containers as part of the GitLab CI pipeline and upload them to the > GitLab container registry for later use. > > This will not significantly slow down builds, because containers are > only rebuilt when the corresponding Dockerfile has been modified. > > Signed-off-by: Andrea Bolognani <abologna@xxxxxxxxxx> > --- > .gitlab-ci.yml | 234 +++++++++++++++++- > ci/containers/README.rst | 14 ++ > ci/containers/ci-centos-7.Dockerfile | 137 ++++++++++ > ci/containers/ci-centos-8.Dockerfile | 108 ++++++++ > .../ci-debian-10-cross-aarch64.Dockerfile | 122 +++++++++ > .../ci-debian-10-cross-armv6l.Dockerfile | 120 +++++++++ > .../ci-debian-10-cross-armv7l.Dockerfile | 121 +++++++++ > .../ci-debian-10-cross-i686.Dockerfile | 121 +++++++++ > .../ci-debian-10-cross-mips.Dockerfile | 121 +++++++++ > .../ci-debian-10-cross-mips64el.Dockerfile | 121 +++++++++ > .../ci-debian-10-cross-mipsel.Dockerfile | 121 +++++++++ > .../ci-debian-10-cross-ppc64le.Dockerfile | 121 +++++++++ > .../ci-debian-10-cross-s390x.Dockerfile | 121 +++++++++ > ci/containers/ci-debian-10.Dockerfile | 112 +++++++++ > .../ci-debian-9-cross-aarch64.Dockerfile | 126 ++++++++++ > .../ci-debian-9-cross-armv6l.Dockerfile | 124 ++++++++++ > .../ci-debian-9-cross-armv7l.Dockerfile | 125 ++++++++++ > .../ci-debian-9-cross-mips.Dockerfile | 125 ++++++++++ > .../ci-debian-9-cross-mips64el.Dockerfile | 125 ++++++++++ > .../ci-debian-9-cross-mipsel.Dockerfile | 125 ++++++++++ > .../ci-debian-9-cross-ppc64le.Dockerfile | 125 ++++++++++ > .../ci-debian-9-cross-s390x.Dockerfile | 125 ++++++++++ > ci/containers/ci-debian-9.Dockerfile | 116 +++++++++ > .../ci-debian-sid-cross-aarch64.Dockerfile | 122 +++++++++ > .../ci-debian-sid-cross-armv6l.Dockerfile | 120 +++++++++ > .../ci-debian-sid-cross-armv7l.Dockerfile | 121 +++++++++ > .../ci-debian-sid-cross-i686.Dockerfile | 121 +++++++++ > .../ci-debian-sid-cross-mips.Dockerfile | 121 +++++++++ > .../ci-debian-sid-cross-mips64el.Dockerfile | 121 +++++++++ > .../ci-debian-sid-cross-mipsel.Dockerfile | 120 +++++++++ > .../ci-debian-sid-cross-ppc64le.Dockerfile | 121 +++++++++ > .../ci-debian-sid-cross-s390x.Dockerfile | 121 +++++++++ > ci/containers/ci-debian-sid.Dockerfile | 112 +++++++++ > ci/containers/ci-fedora-31.Dockerfile | 109 ++++++++ > ci/containers/ci-fedora-32.Dockerfile | 109 ++++++++ > ...ci-fedora-rawhide-cross-mingw32.Dockerfile | 129 ++++++++++ > ...ci-fedora-rawhide-cross-mingw64.Dockerfile | 129 ++++++++++ > ci/containers/ci-fedora-rawhide.Dockerfile | 110 ++++++++ > ci/containers/ci-opensuse-151.Dockerfile | 109 ++++++++ > ci/containers/ci-ubuntu-1804.Dockerfile | 117 +++++++++ > ci/containers/ci-ubuntu-2004.Dockerfile | 113 +++++++++ > ci/containers/refresh | 43 ++++ > 42 files changed, 4973 insertions(+), 5 deletions(-) > create mode 100644 ci/containers/README.rst > create mode 100644 ci/containers/ci-centos-7.Dockerfile > create mode 100644 ci/containers/ci-centos-8.Dockerfile > create mode 100644 ci/containers/ci-debian-10-cross-aarch64.Dockerfile > create mode 100644 ci/containers/ci-debian-10-cross-armv6l.Dockerfile > create mode 100644 ci/containers/ci-debian-10-cross-armv7l.Dockerfile > create mode 100644 ci/containers/ci-debian-10-cross-i686.Dockerfile > create mode 100644 ci/containers/ci-debian-10-cross-mips.Dockerfile > create mode 100644 ci/containers/ci-debian-10-cross-mips64el.Dockerfile > create mode 100644 ci/containers/ci-debian-10-cross-mipsel.Dockerfile > create mode 100644 ci/containers/ci-debian-10-cross-ppc64le.Dockerfile > create mode 100644 ci/containers/ci-debian-10-cross-s390x.Dockerfile > create mode 100644 ci/containers/ci-debian-10.Dockerfile > create mode 100644 ci/containers/ci-debian-9-cross-aarch64.Dockerfile > create mode 100644 ci/containers/ci-debian-9-cross-armv6l.Dockerfile > create mode 100644 ci/containers/ci-debian-9-cross-armv7l.Dockerfile > create mode 100644 ci/containers/ci-debian-9-cross-mips.Dockerfile > create mode 100644 ci/containers/ci-debian-9-cross-mips64el.Dockerfile > create mode 100644 ci/containers/ci-debian-9-cross-mipsel.Dockerfile > create mode 100644 ci/containers/ci-debian-9-cross-ppc64le.Dockerfile > create mode 100644 ci/containers/ci-debian-9-cross-s390x.Dockerfile > create mode 100644 ci/containers/ci-debian-9.Dockerfile > create mode 100644 ci/containers/ci-debian-sid-cross-aarch64.Dockerfile > create mode 100644 ci/containers/ci-debian-sid-cross-armv6l.Dockerfile > create mode 100644 ci/containers/ci-debian-sid-cross-armv7l.Dockerfile > create mode 100644 ci/containers/ci-debian-sid-cross-i686.Dockerfile > create mode 100644 ci/containers/ci-debian-sid-cross-mips.Dockerfile > create mode 100644 ci/containers/ci-debian-sid-cross-mips64el.Dockerfile > create mode 100644 ci/containers/ci-debian-sid-cross-mipsel.Dockerfile > create mode 100644 ci/containers/ci-debian-sid-cross-ppc64le.Dockerfile > create mode 100644 ci/containers/ci-debian-sid-cross-s390x.Dockerfile > create mode 100644 ci/containers/ci-debian-sid.Dockerfile > create mode 100644 ci/containers/ci-fedora-31.Dockerfile > create mode 100644 ci/containers/ci-fedora-32.Dockerfile > create mode 100644 ci/containers/ci-fedora-rawhide-cross-mingw32.Dockerfile > create mode 100644 ci/containers/ci-fedora-rawhide-cross-mingw64.Dockerfile > create mode 100644 ci/containers/ci-fedora-rawhide.Dockerfile > create mode 100644 ci/containers/ci-opensuse-151.Dockerfile > create mode 100644 ci/containers/ci-ubuntu-1804.Dockerfile > create mode 100644 ci/containers/ci-ubuntu-2004.Dockerfile > create mode 100755 ci/containers/refresh > > diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml > index 8a5b3372de..0e7917d6cd 100644 > --- a/.gitlab-ci.yml > +++ b/.gitlab-ci.yml > @@ -3,6 +3,7 @@ variables: > > stages: > - prebuild > + - containers > - native_build > - cross_build > - other > @@ -16,10 +17,35 @@ stages: > > # Common templates > > +.container_job_template: &container_job_definition > + image: docker:stable > + stage: containers > + services: > + - docker:dind > + before_script: > + - export TAG="$CI_REGISTRY_IMAGE/ci-$NAME:latest" > + - export COMMON_TAG="$CI_REGISTRY/libvirt/libvirt/ci-$NAME:latest" > + - docker info > + - docker login registry.gitlab.com -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" > + script: > + - docker pull "$TAG" || docker pull "$COMMON_TAG" || true > + - docker build --cache-from "$TAG" --cache-from "$COMMON_TAG" --tag "$TAG" -f "ci/containers/ci-$NAME.Dockerfile" ci/containers > + - docker push "$TAG" > + after_script: > + - docker logout > + > +# We build many containers which can be useful to debug problems but are not > +# needed for the pipeline itself to complete: those sometimes fail, and when > +# that happens it's mostly because of temporary issues with Debian sid. We > +# don't want those failures to affect the overall pipeline status > +.container_optional_job_template: &container_optional_job_definition > + <<: *container_job_definition > + allow_failure: true I don't think we should be building container images that we're not going to be using in any of the jobs, as it can only ever slow down the build overall. > + > # Default native build jobs that are always run > .native_build_default_job_template: &native_build_default_job_definition > stage: native_build > - image: quay.io/libvirt/buildenv-libvirt-$NAME:latest > + image: $CI_REGISTRY_IMAGE/ci-$NAME:latest > cache: > paths: > - ccache/ > @@ -44,7 +70,7 @@ stages: > # Default cross build jobs that are always run > .cross_build_default_job_template: &cross_build_default_job_definition > stage: cross_build > - image: quay.io/libvirt/buildenv-libvirt-$NAME-cross-$CROSS:latest > + image: $CI_REGISTRY_IMAGE/ci-$NAME-cross-$CROSS:latest > cache: > paths: > - ccache/ > @@ -66,6 +92,204 @@ stages: > - /^ci-full-.*$/ > > > +# Container build jobs > + > +centos-7-container: IMHO we should name these to match the build job. eg arch, then distro x64-centos-7-container > + <<: *container_job_definition > + variables: > + NAME: centos-7 > + > +centos-8-container: > + <<: *container_job_definition > + variables: > + NAME: centos-8 > + > +debian-9-container: > + <<: *container_job_definition > + variables: > + NAME: debian-9 > + > +debian-9-cross-aarch64-container: > + <<: *container_optional_job_definition > + variables: > + NAME: debian-9-cross-aarch64 > + > +debian-9-cross-armv6l-container: > + <<: *container_job_definition > + variables: > + NAME: debian-9-cross-armv6l This container, and many others are only used by the "extra" build jobs, so should be subject to the same filtering. > + > +debian-9-cross-armv7l-container: > + <<: *container_optional_job_definition > + variables: > + NAME: debian-9-cross-armv7l > + > +debian-9-cross-mips-container: > + <<: *container_job_definition > + variables: > + NAME: debian-9-cross-mips > + > +debian-9-cross-mips64el-container: > + <<: *container_job_definition > + variables: > + NAME: debian-9-cross-mips64el > + > +debian-9-cross-mipsel-container: > + <<: *container_optional_job_definition > + variables: > + NAME: debian-9-cross-mipsel > + > +debian-9-cross-ppc64le-container: > + <<: *container_optional_job_definition > + variables: > + NAME: debian-9-cross-ppc64le > + > +debian-9-cross-s390x-container: > + <<: *container_optional_job_definition > + variables: > + NAME: debian-9-cross-s390x > + > +debian-10-container: > + <<: *container_job_definition > + variables: > + NAME: debian-10 > + > +debian-10-cross-aarch64-container: > + <<: *container_job_definition > + variables: > + NAME: debian-10-cross-aarch64 > + > +debian-10-cross-armv6l-container: > + <<: *container_optional_job_definition > + variables: > + NAME: debian-10-cross-armv6l > + > +debian-10-cross-armv7l-container: > + <<: *container_optional_job_definition > + variables: > + NAME: debian-10-cross-armv7l > + > +debian-10-cross-i686-container: > + <<: *container_optional_job_definition > + variables: > + NAME: debian-10-cross-i686 > + > +debian-10-cross-mips-container: > + <<: *container_optional_job_definition > + variables: > + NAME: debian-10-cross-mips > + > +debian-10-cross-mips64el-container: > + <<: *container_optional_job_definition > + variables: > + NAME: debian-10-cross-mips64el > + > +debian-10-cross-mipsel-container: > + <<: *container_optional_job_definition > + variables: > + NAME: debian-10-cross-mipsel > + > +debian-10-cross-ppc64le-container: > + <<: *container_job_definition > + variables: > + NAME: debian-10-cross-ppc64le > + > +debian-10-cross-s390x-container: > + <<: *container_job_definition > + variables: > + NAME: debian-10-cross-s390x > + > +debian-sid-container: > + <<: *container_job_definition > + variables: > + NAME: debian-sid > + > +debian-sid-cross-aarch64-container: > + <<: *container_optional_job_definition > + variables: > + NAME: debian-sid-cross-aarch64 > + > +debian-sid-cross-armv6l-container: > + <<: *container_optional_job_definition > + variables: > + NAME: debian-sid-cross-armv6l > + > +debian-sid-cross-armv7l-container: > + <<: *container_job_definition > + variables: > + NAME: debian-sid-cross-armv7l > + > +debian-sid-cross-i686-container: > + <<: *container_job_definition > + variables: > + NAME: debian-sid-cross-i686 > + > +debian-sid-cross-mips-container: > + <<: *container_optional_job_definition > + variables: > + NAME: debian-sid-cross-mips > + > +debian-sid-cross-mips64el-container: > + <<: *container_optional_job_definition > + variables: > + NAME: debian-sid-cross-mips64el > + > +debian-sid-cross-mipsel-container: > + <<: *container_job_definition > + variables: > + NAME: debian-sid-cross-mipsel > + > +debian-sid-cross-ppc64le-container: > + <<: *container_optional_job_definition > + variables: > + NAME: debian-sid-cross-ppc64le > + > +debian-sid-cross-s390x-container: > + <<: *container_optional_job_definition > + variables: > + NAME: debian-sid-cross-s390x > + > +fedora-31-container: > + <<: *container_job_definition > + variables: > + NAME: fedora-31 > + > +fedora-32-container: > + <<: *container_job_definition > + variables: > + NAME: fedora-32 > + > +fedora-rawhide-container: > + <<: *container_job_definition > + variables: > + NAME: fedora-rawhide > + > +fedora-rawhide-cross-mingw32-container: > + <<: *container_job_definition > + variables: > + NAME: fedora-rawhide-cross-mingw32 > + > +fedora-rawhide-cross-mingw64-container: > + <<: *container_job_definition > + variables: > + NAME: fedora-rawhide-cross-mingw64 > + > +opensuse-151-container: > + <<: *container_job_definition > + variables: > + NAME: opensuse-151 > + RPM: skip > + > +ubuntu-1804-container: > + <<: *container_job_definition > + variables: > + NAME: ubuntu-1804 > + > +ubuntu-2004-container: > + <<: *container_job_definition > + variables: > + NAME: ubuntu-2004 > + > # Native architecture build + test jobs > > x64-debian-9: > @@ -198,6 +422,7 @@ mingw64-fedora-rawhide: > # https://gitlab.com/libvirt/libvirt/-/jobs/artifacts/master/download?job=website > website: > stage: other > + image: $CI_REGISTRY_IMAGE/ci-centos-8:latest > before_script: > - *script_variables > script: > @@ -208,7 +433,6 @@ website: > - $MAKE -C docs install > - cd .. > - mv vroot/share/doc/libvirt/html/ website > - image: quay.io/libvirt/buildenv-libvirt-centos-8:latest > artifacts: > expose_as: 'Website' > name: 'website' > @@ -220,6 +444,7 @@ website: > > codestyle: > stage: other > + image: $CI_REGISTRY_IMAGE/ci-centos-8:latest > before_script: > - *script_variables > script: > @@ -227,7 +452,6 @@ codestyle: > - cd build > - ../autogen.sh || (cat config.log && exit 1) > - $MAKE syntax-check > - image: quay.io/libvirt/buildenv-libvirt-centos-8:latest > > > # This artifact published by this job is downloaded to push to Weblate > @@ -235,6 +459,7 @@ codestyle: > # https://gitlab.com/libvirt/libvirt/-/jobs/artifacts/master/download?job=potfile > potfile: > stage: prebuild > + image: $CI_REGISTRY_IMAGE/ci-centos-8:latest > only: > - master > before_script: > @@ -247,7 +472,6 @@ potfile: > - $MAKE -C po libvirt.pot > - cd .. > - mv build/po/libvirt.pot libvirt.pot > - image: quay.io/libvirt/buildenv-libvirt-centos-8:latest > artifacts: > expose_as: 'Potfile' > name: 'potfile' Regards, Daniel -- |: https://berrange.com -o- https://www.flickr.com/photos/dberrange :| |: https://libvirt.org -o- https://fstop138.berrange.com :| |: https://entangle-photo.org -o- https://www.instagram.com/dberrange :|
