[gentoo-commits] proj/docker-images:master commit in: /

[Alexys Jacob]

commit:     2a77ebf7e7c02d43eb9dd385dc808efd87a54af0
Author:     Konstantinos Smanis <konstantinos.smanis <AT> gmail <DOT> com>
AuthorDate: Sun Aug 23 18:54:34 2020 +0000
Commit:     Alexys Jacob <ultrabug <AT> gentoo <DOT> org>
CommitDate: Thu Sep  3 20:34:00 2020 +0000
URL:        https://gitweb.gentoo.org/proj/docker-images.git/commit/?id=2a77ebf7

Add CI builds for all supported stage3 architectures

Build multiarch images using buildx [1] instead of modifying the image
architecture post-creation with docker-copyedit. Although still
experimental, buildx is the recommended way of building multi-platform
images.

All stage3 architectures that are supported by Docker [2] were added.

Closes: #61

[1] https://docs.docker.com/buildx/working-with-buildx/#build-multi-platform-images
[2] https://github.com/docker-library/official-images#architectures-other-than-amd64

Signed-off-by: Konstantinos Smanis <konstantinos.smanis <AT> gmail.com>
Closes: https://github.com/gentoo/gentoo-docker-images/pull/92
Signed-off-by: Alexys Jacob <ultrabug <AT> gentoo.org>

 .gitmodules        |  3 ---
 .travis.yml        | 67 ++++++++++++++++++++++++++++++++-------------------
 README.md          | 59 +++++++++++++++++++++++++++++++++++++--------
 build-multiarch.sh |  9 -------
 build.sh           | 71 ++++++++++++++++++++++++++++++++++--------------------
 docker-copyedit    |  1 -
 portage.Dockerfile |  2 +-
 stage3.Dockerfile  |  2 +-
 8 files changed, 138 insertions(+), 76 deletions(-)

diff --git a/.gitmodules b/.gitmodules
deleted file mode 100644
index 52c678d..0000000
--- a/.gitmodules
+++ /dev/null
@@ -1,3 +0,0 @@
-[submodule "docker-copyedit"]
-	path = docker-copyedit
-	url = https://github.com/gdraheim/docker-copyedit.git

diff --git a/.travis.yml b/.travis.yml
index 874e1ce..50ba3f4 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,46 +1,63 @@
-services: docker
-language: bash
+language: shell
 env:
   global:
+    - DOCKER_CLI_EXPERIMENTAL=enabled  # required by buildx
     - ORG=gentoo
-  matrix:
+  jobs:
     - TARGET=portage
     - TARGET=stage3-amd64
     - TARGET=stage3-amd64-hardened
     - TARGET=stage3-amd64-hardened-nomultilib
+    - TARGET=stage3-amd64-musl-hardened
+    - TARGET=stage3-amd64-musl-vanilla
     - TARGET=stage3-amd64-nomultilib
     - TARGET=stage3-amd64-systemd
-    - TARGET=stage3-amd64-musl-vanilla
-    - TARGET=stage3-amd64-musl-hardened
+    - TARGET=stage3-amd64-uclibc-hardened
+    - TARGET=stage3-amd64-uclibc-vanilla
+    - TARGET=stage3-arm64
+    - TARGET=stage3-arm64-systemd
+    - TARGET=stage3-armv5tel
+    - TARGET=stage3-armv6j_hardfp
+    - TARGET=stage3-armv7a_hardfp
+    - TARGET=stage3-ppc64le
+    - TARGET=stage3-s390x
     - TARGET=stage3-x86
     - TARGET=stage3-x86-hardened
-    - TARGET=stage3-armv7a
     - TARGET=stage3-x86-musl-vanilla
-    - TARGET=stage3-ppc
-    - TARGET=stage3-ppc64
-    - TARGET=stage3-ppc64le
+    - TARGET=stage3-x86-systemd
+    - TARGET=stage3-x86-uclibc-hardened
+    - TARGET=stage3-x86-uclibc-vanilla
 
 before_install:
-  # Install latest Docker
+  # Install latest Docker (>=19.03.0 required by buildx)
+  # https://docs.travis-ci.com/user/docker/#installing-a-newer-docker-version
   - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
   - sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
   - sudo apt-get update -qq
   - sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce
-  - echo '{"experimental":true}' | sudo tee /etc/docker/daemon.json
-  - sudo apt-get install qemu-user-static binfmt-support
-  - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+  - docker info
+before_script:
+  # Create multiarch buildx builder
+  - docker buildx create --driver docker-container --use
 script:
-  - VERSION="$(date -u +%Y%m%d)"
-  - sudo ./build.sh
-  - if [[ "${TARGET}" == stage* ]]; then 
-    sudo docker run -it --rm ${ORG}/${TARGET}:${VERSION} /bin/bash -c "emerge --info" ;
-    fi
-
-# Travis is set up to push daily to dockerhub
+  # Build image
+  - ./build.sh
 after_success:
-  - if [[ "${TRAVIS_PULL_REQUEST_BRANCH:-${TRAVIS_BRANCH}}" == "master" && "${TRAVIS_EVENT_TYPE}" == "cron" ]]; then
-    VERSION=$(date -u +%Y%m%d);
-    echo "${DOCKER_PASSWORD}" | docker login -u="${DOCKER_USERNAME}" --password-stdin;
-    docker push "${ORG}/${TARGET}:latest";
-    docker push "${ORG}/${TARGET}:${VERSION}";
+  # Inspect built image
+  - docker image inspect "${ORG}/${TARGET}:latest"
+  # Run `emerge --info` for stage builds
+  - |
+    if [[ "${TARGET}" == stage* ]]; then
+      # Check if QEMU emulation support is required
+      if [[ ! "${TARGET}" =~ -(amd64|x86)($|-) ]]; then
+        # Enable execution of foreign binary formats (i.e., non-amd64/x86)
+        docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+      fi
+      docker run --rm "${ORG}/${TARGET}:latest" emerge --info
+    fi
+  # Push all built images to Docker Hub (cron daily task)
+  - |
+    if [[ "${TRAVIS_PULL_REQUEST_BRANCH:-${TRAVIS_BRANCH}}" == "master" && "${TRAVIS_EVENT_TYPE}" == "cron" ]]; then
+      echo "${DOCKER_PASSWORD}" | docker login -u "${DOCKER_USERNAME}" --password-stdin
+      docker push "${ORG}/${TARGET}"
     fi

diff --git a/README.md b/README.md
index eb5eba5..10eba12 100644
--- a/README.md
+++ b/README.md
@@ -14,19 +14,58 @@ https://hub.docker.com/u/gentoo/
 
 ## Inventory
 
-* portage
-* stage3
-  * stage3-amd64
-    * stage3-amd64-hardened
-    * stage3-amd64-hardened-nomultilib
-    * stage3-amd64-nomultilib
-    * stage3-amd64-systemd
-  * stage3-x86
-    * stage3-x86-hardened
+The following targets are built by Travis (bold targets are also pushed to Docker Hub):
+ * **`portage`**
+ * `stage3`
+   * `amd64`
+     * **`stage3-amd64`**
+     * **`stage3-amd64-hardened`**
+     * **`stage3-amd64-hardened-nomultilib`**
+     * `stage3-amd64-musl-hardened`
+     * `stage3-amd64-musl-vanilla`
+     * **`stage3-amd64-nomultilib`**
+     * `stage3-amd64-systemd`
+     * `stage3-amd64-uclibc-hardened`
+     * `stage3-amd64-uclibc-vanilla`
+   * `arm64`
+     * `stage3-arm64`
+     * `stage3-arm64-systemd`
+   * `arm`
+     * `stage3-armv5tel`
+     * `stage3-armv6j_hardfp`
+     * `stage3-armv7a_hardfp`
+   * `ppc`
+     * `stage3-ppc64le`
+   * `s390`
+     * `stage3-s390x`
+   * `x86`
+     * **`stage3-x86`**
+     * **`stage3-x86-hardened`**
+     * `stage3-x86-musl-vanilla`
+     * `stage3-x86-systemd`
+     * `stage3-x86-uclibc-hardened`
+     * `stage3-x86-uclibc-vanilla`
+
+The following upstream stage3 targets are not built at all (see [rationale](https://github.com/gentoo/gentoo-docker-images/issues/75#issuecomment-680776939)):
+ * `amd64`
+   * `stage3-amd64-hardened-selinux`
+   * `stage3-amd64-hardened-selinux+nomultilib`
+   * `stage3-x32`
+ * `arm`
+   * `stage3-armv4tl`
+   * `stage3-armv6j`
+   * `stage3-armv7a`
+ * `ppc`
+   * `stage3-ppc`
+   * `stage3-ppc64`
+ * `s390`
+   * `stage3-s390`
+ * `x86`
+   * `stage3-i486`
 
 # Building the containers
 
-The containers are created using a multi-stage build, which requires docker-17.05.0 or later.
+The containers are created using a multi-stage build, which requires Docker >= 19.03.0.
 The container being built is defined by the TARGET environment variable:
 
 `` TARGET=stage3-amd64 ./build.sh ``

diff --git a/build-multiarch.sh b/build-multiarch.sh
deleted file mode 100755
index bc2fde9..0000000
--- a/build-multiarch.sh
+++ /dev/null
@@ -1,9 +0,0 @@
-#!/bin/bash
-docker manifest create gentoo/stage3 \
-	gentoo/stage3-amd64  \
-	gentoo/stage3-x86    \
-	gentoo/stage3-armv7a \
-	gentoo/stage3-amd64  \
-	gentoo/stage3-ppc    \
-	gentoo/stage3-ppc64  \
-	gentoo/stage3-ppc64le

diff --git a/build.sh b/build.sh
index 10651cf..c0100d7 100755
--- a/build.sh
+++ b/build.sh
@@ -11,32 +11,45 @@ fi
 
 # Split the TARGET variable into three elements separated by hyphens
 IFS=- read -r NAME ARCH SUFFIX <<< "${TARGET}"
-DOCKER_ARCH="${ARCH}"
-
-# Ensure upstream directories for stage3-amd64-hardened+nomultilib work
-# unless we're building for musl targets (vanilla/hardened)
-if [[ "${SUFFIX}" != *musl* ]]; then
-	SUFFIX=${SUFFIX/-/+}
-fi
 
 VERSION=${VERSION:-$(date -u +%Y%m%d)}
 
 ORG=${ORG:-gentoo}
 
-# x86 requires the i686 subfolder
-if [[ "${ARCH}" == "x86" ]]; then
-	DOCKER_ARCH="386"
-	MICROARCH="i686"
-	BOOTSTRAP="multiarch/alpine:x86-v3.11"
-elif [[ "${ARCH}" = ppc* ]]; then
-	MICROARCH="${ARCH}"
-	ARCH=ppc
-elif [[ "${ARCH}" = arm* ]]; then
-	DOCKER_ARCH=$(echo $ARCH | sed -e 's-\(v.\).*-/\1-g')
-	MICROARCH="${ARCH}"
-	ARCH=arm
-else
-	MICROARCH="${ARCH}"
+case $ARCH in
+	"amd64" | "arm64")
+		DOCKER_ARCH="${ARCH}"
+		MICROARCH="${ARCH}"
+		;;
+	"armv"*)
+		# armv6j_hardfp -> arm/v6
+		# armv7a_hardfp -> arm/v7
+		DOCKER_ARCH=$(echo "$ARCH" | sed -e 's#arm\(v.\).*#arm/\1#g')
+		MICROARCH="${ARCH}"
+		ARCH="arm"
+		;;
+	"ppc64le")
+		DOCKER_ARCH="${ARCH}"
+		MICROARCH="${ARCH}"
+		ARCH="ppc"
+		;;
+	"s390x")
+		DOCKER_ARCH="${ARCH}"
+		MICROARCH="${ARCH}"
+		ARCH="s390"
+		;;
+	"x86")
+		DOCKER_ARCH="386"
+		MICROARCH="i686"
+		;;
+	*)  # portage
+		DOCKER_ARCH="amd64"
+		;;
+esac
+
+# Handle targets with special characters in the suffix
+if [[ "${TARGET}" == "stage3-amd64-hardened-nomultilib" ]]; then
+	SUFFIX="hardened+nomultilib"
 fi
 
 # Prefix the suffix with a hyphen to make sure the URL works
@@ -44,8 +57,14 @@ if [[ -n "${SUFFIX}" ]]; then
 	SUFFIX="-${SUFFIX}"
 fi
 
-set -x
-docker build --build-arg ARCH="${ARCH}" --build-arg MICROARCH="${MICROARCH}" --build-arg BOOTSTRAP="${BOOTSTRAP}" --build-arg SUFFIX="${SUFFIX}"  -t "${ORG}/${TARGET}:${VERSION}" -f "${NAME}.Dockerfile" .
-docker-copyedit/docker-copyedit.py FROM "${ORG}/${TARGET}:${VERSION}" INTO "${ORG}/${TARGET}:${VERSION}" -vv \
-    set arch ${DOCKER_ARCH}
-docker tag "${ORG}/${TARGET}:${VERSION}" "${ORG}/${TARGET}:latest"
+docker buildx build \
+	--file "${NAME}.Dockerfile" \
+	--build-arg ARCH="${ARCH}" \
+	--build-arg MICROARCH="${MICROARCH}" \
+	--build-arg SUFFIX="${SUFFIX}" \
+	--tag "${ORG}/${TARGET}:latest" \
+	--tag "${ORG}/${TARGET}:${VERSION}" \
+	--platform "linux/${DOCKER_ARCH}" \
+	--progress plain \
+	--load \
+	.

diff --git a/docker-copyedit b/docker-copyedit
deleted file mode 160000
index ab6bd5d..0000000
--- a/docker-copyedit
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit ab6bd5d9f5ca3a9ba314e7124c2aac8ad7987a3b

diff --git a/portage.Dockerfile b/portage.Dockerfile
index 7f51c7e..23261d2 100644
--- a/portage.Dockerfile
+++ b/portage.Dockerfile
@@ -3,7 +3,7 @@
 # docker-17.05.0 or later. It fetches a daily snapshot from the official
 # sources and verifies its checksum as well as its gpg signature.
 
-FROM alpine:3.11 as builder
+FROM --platform=$BUILDPLATFORM alpine:3.11 as builder
 
 WORKDIR /portage
 

diff --git a/stage3.Dockerfile b/stage3.Dockerfile
index a9dc594..b9909ee 100644
--- a/stage3.Dockerfile
+++ b/stage3.Dockerfile
@@ -4,7 +4,7 @@
 # sources and verifies its checksum as well as its gpg signature.
 
 ARG BOOTSTRAP
-FROM ${BOOTSTRAP:-alpine:3.11} as builder
+FROM --platform=$BUILDPLATFORM ${BOOTSTRAP:-alpine:3.11} as builder
 
 WORKDIR /gentoo