feat(ci): introduce GH Actions for Portainer CE (#10419)

Co-authored-by: Chaim Lev-Ari <chaim.levi-ari@portainer.io>
2023-10-24 19:30:33 +09:00 · 2023-10-24 19:30:33 +09:00 · 08fdebfbd9
parent 860890046d
commit 08fdebfbd9
11 changed files with 194 additions and 26 deletions
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@ -0,0 +1,154 @@
+name: ci
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+    - "develop"
+    - "!release/*"
+  pull_request:
+    branches:
+    - "develop"
+    - "release/*"
+    - "feat/*"
+    - "fix/*"
+    - "refactor/*"
+
+env:
+  DOCKER_HUB_REPO: portainerci/portainer
+  NODE_ENV: testing
+  GO_VERSION: 1.21.0
+  NODE_VERSION: 18.x
+
+jobs:
+  build_images:
+    strategy:
+      matrix:
+        config:
+          - { platform: linux, arch: amd64 }
+          - { platform: linux, arch: arm64 }
+          - { platform: windows, arch: amd64, version: 1809 }
+          - { platform: windows, arch: amd64, version: ltsc2022 }
+    runs-on: arc-runner-set
+    steps:
+      - name: "[preparation] checkout the current branch"
+        uses: actions/checkout@v3.5.3
+        with:
+          ref: ${{ github.event.inputs.branch }}
+      - name: "[preparation] set up golang"
+        uses: actions/setup-go@v4.0.1
+        with:
+          go-version: ${{ env.GO_VERSION }}
+          cache: false
+      - name: "[preparation] cache paths"
+        id: cache-dir-path
+        run: |
+          echo "yarn-cache-dir=$(yarn cache dir)" >> "$GITHUB_OUTPUT"
+          echo "go-build-dir=$(go env GOCACHE)" >> "$GITHUB_OUTPUT"
+          echo "go-mod-dir=$(go env GOMODCACHE)" >> "$GITHUB_OUTPUT"
+      - name: "[preparation] cache go"
+        uses: actions/cache@v3
+        with:
+          path: |
+            ${{ steps.cache-dir-path.outputs.go-build-dir }}
+            ${{ steps.cache-dir-path.outputs.go-mod-dir }}
+          key: ${{ matrix.config.platform }}-${{ matrix.config.arch }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ matrix.config.platform }}-${{ matrix.config.arch }}-go-
+          enableCrossOsArchive: true
+      - name: "[preparation] set up node.js"
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ env.NODE_VERSION }}
+          cache: ''
+      - name: "[preparation] cache yarn"
+        uses: actions/cache@v3
+        with:
+          path: |
+            **/node_modules
+            ${{ steps.cache-dir-path.outputs.yarn-cache-dir }}
+          key: ${{ matrix.config.platform }}-${{ matrix.config.arch }}-yarn-${{ hashFiles('**/yarn.lock') }}
+          restore-keys: |
+            ${{ matrix.config.platform }}-${{ matrix.config.arch }}-yarn-
+          enableCrossOsArchive: true
+      - name: "[preparation] execute client test"
+        run: |
+          make client-deps && make test-client
+      - name: "[preparation] execute server test"
+        run: |
+          make test-server
+      - name: "[preparation] set up qemu"
+        uses: docker/setup-qemu-action@v2
+      - name: "[preparation] set up docker context for buildx"
+        run: docker context create builders
+      - name: "[preparation] set up docker buildx"
+        uses: docker/setup-buildx-action@v2
+        with:
+          endpoint: builders
+      - name: "[preparation] docker login"
+        uses: docker/login-action@v2.2.0
+        with:
+          username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          password: ${{ secrets.DOCKER_HUB_PASSWORD }}
+      - name: "[preparation] set the container image tag"
+        run: |
+          if [ "${GITHUB_EVENT_NAME}" == "pull_request" ]; then
+            CONTAINER_IMAGE_TAG="pr${{ github.event.number }}"
+          else
+            CONTAINER_IMAGE_TAG=$(echo $GITHUB_REF_NAME | sed 's/\//-/g')
+          fi
+          
+          if [ "${{ matrix.config.platform }}" == "windows" ]; then
+            CONTAINER_IMAGE_TAG="${CONTAINER_IMAGE_TAG}-${{ matrix.config.platform }}${{ matrix.config.version }}-${{ matrix.config.arch }}"
+          else 
+            CONTAINER_IMAGE_TAG="${CONTAINER_IMAGE_TAG}-${{ matrix.config.platform }}-${{ matrix.config.arch }}"
+          fi
+          
+          echo "CONTAINER_IMAGE_TAG=${CONTAINER_IMAGE_TAG}" >> $GITHUB_ENV
+      - name: "[execution] build linux & windows portainer binaries"
+        run: |
+          export YARN_VERSION=$(yarn --version)
+          export WEBPACK_VERSION=$(yarn list webpack --depth=0 | grep webpack | awk -F@ '{print $2}')
+          export BUILDNUMBER=${GITHUB_RUN_NUMBER}
+          make build-all PLATFORM=${{ matrix.config.platform }} ARCH=${{ matrix.config.arch }} ENV=${NODE_ENV}
+        env:
+          CONTAINER_IMAGE_TAG: ${{ env.CONTAINER_IMAGE_TAG }}
+      - name: "[execution] build and push docker images"
+        run: |          
+          if [ "${{ matrix.config.platform }}" == "windows" ]; then
+            mv dist/portainer dist/portainer.exe
+            docker buildx build --output=type=registry --platform ${{ matrix.config.platform }}/${{ matrix.config.arch }} --build-arg OSVERSION=${{ matrix.config.version }} -t "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}" -f build/${{ matrix.config.platform }}/Dockerfile .
+          else 
+            docker buildx build --output=type=registry --platform ${{ matrix.config.platform }}/${{ matrix.config.arch }} -t "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}" -f build/${{ matrix.config.platform }}/Dockerfile .
+            docker buildx build --output=type=registry --platform ${{ matrix.config.platform }}/${{ matrix.config.arch }} -t "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-alpine" -f build/${{ matrix.config.platform }}/alpine.Dockerfile .
+          fi
+        env:
+          CONTAINER_IMAGE_TAG: ${{ env.CONTAINER_IMAGE_TAG }}
+  build_manifests:
+    runs-on: arc-runner-set
+    needs: [build_images]
+    steps:
+      - name: "[preparation] docker login"
+        uses: docker/login-action@v2.2.0
+        with:
+          username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          password: ${{ secrets.DOCKER_HUB_PASSWORD }}
+      - name: "[preparation] set up docker context for buildx"
+        run: docker version && docker context create builders
+      - name: "[preparation] set up docker buildx"
+        uses: docker/setup-buildx-action@v2
+        with:
+          endpoint: builders
+      - name: "[execution] build and push manifests"
+        run: |
+          if [ "${GITHUB_EVENT_NAME}" == "pull_request" ]; then
+            CONTAINER_IMAGE_TAG="pr${{ github.event.number }}"
+          else
+            CONTAINER_IMAGE_TAG=$(echo $GITHUB_REF_NAME | sed 's/\//-/g')
+          fi
+          
+          docker buildx imagetools create -t "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}" \
+            "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-amd64" \
+            "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-linux-arm64" \
+            "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-windows1809-amd64" \
+            "${DOCKER_HUB_REPO}:${CONTAINER_IMAGE_TAG}-windowsltsc2022-amd64"
--- a/binary-version.json
+++ b/binary-version.json
@ -2,5 +2,6 @@
  "docker": "v20.10.21",
  "dockerCompose": "v2.20.3",
  "helm": "v3.12.3",
-  "kubectl": "v1.28.1"
+  "kubectl": "v1.28.1",
+  "mingit": "2.42.0.2"
 }
--- a/build/build_binary.sh
+++ b/build/build_binary.sh
@ -4,12 +4,12 @@ set -euo pipefail
 mkdir -p dist

 # populate tool versions
-BUILDNUMBER="N/A"
-CONTAINER_IMAGE_TAG="N/A"
-NODE_VERSION="0"
-YARN_VERSION="0"
-WEBPACK_VERSION="0"
-GO_VERSION="0"
+BUILDNUMBER=${BUILDNUMBER:-"N/A"}
+CONTAINER_IMAGE_TAG=${CONTAINER_IMAGE_TAG:-"N/A"}
+NODE_VERSION=${NODE_VERSION:-"N/A"}
+YARN_VERSION=${YARN_VERSION:-"N/A"}
+WEBPACK_VERSION=${WEBPACK_VERSION:-"N/A"}
+GO_VERSION=${GO_VERSION:-"N/A"}

 # copy templates
 cp -r "./mustache-templates" "./dist"
--- a/build/download_binaries.sh
+++ b/build/download_binaries.sh
@ -11,13 +11,15 @@ dockerVersion=$(jq -r '.docker' < "${BINARY_VERSION_FILE}")
 dockerComposeVersion=$(jq -r '.dockerCompose' < "${BINARY_VERSION_FILE}")
 helmVersion=$(jq -r '.helm' < "${BINARY_VERSION_FILE}")
 kubectlVersion=$(jq -r '.kubectl' < "${BINARY_VERSION_FILE}")
+mingitVersion=$(jq -r '.mingit' < "${BINARY_VERSION_FILE}")

 mkdir -p dist

-echo "Downloading binaries for docker ${dockerVersion}, docker-compose ${dockerComposeVersion}, helm ${helmVersion}, kubectl ${kubectlVersion}"
+echo "Downloading binaries for docker ${dockerVersion}, docker-compose ${dockerComposeVersion}, helm ${helmVersion}, kubectl ${kubectlVersion}, and mingit ${mingitVersion}"

 ./build/download_docker_binary.sh "$PLATFORM" "$ARCH" "$dockerVersion" &
 ./build/download_docker_compose_binary.sh "$PLATFORM" "$ARCH" "$dockerComposeVersion" &
 ./build/download_helm_binary.sh "$PLATFORM" "$ARCH" "$helmVersion" &
 ./build/download_kubectl_binary.sh "$PLATFORM" "$ARCH" "$kubectlVersion" &
+./build/download_mingit_binary.sh "$PLATFORM" "$ARCH" "$mingitVersion" &
 wait
--- a/build/download_docker_binary.sh
+++ b/build/download_docker_binary.sh
@ -31,11 +31,11 @@ rm -rf "${DOWNLOAD_FOLDER}"
 mkdir -pv "${DOWNLOAD_FOLDER}"

 if [[ ${PLATFORM} == "windows" ]]; then
-    wget -O "${DOWNLOAD_FOLDER}/docker-binaries.zip" "https://download.docker.com/win/static/stable/${ARCH}/docker-${DOCKER_VERSION}.zip"
+    wget --tries=3 --waitretry=30 --quiet  -O "${DOWNLOAD_FOLDER}/docker-binaries.zip" "https://download.docker.com/win/static/stable/${ARCH}/docker-${DOCKER_VERSION}.zip"
    unzip "${DOWNLOAD_FOLDER}/docker-binaries.zip" -d "${DOWNLOAD_FOLDER}"
    mv "${DOWNLOAD_FOLDER}/docker/docker.exe" dist/
 else
-    wget -O "${DOWNLOAD_FOLDER}/docker-binaries.tgz" "https://download.docker.com/${PLATFORM}/static/stable/${ARCH}/docker-${DOCKER_VERSION}.tgz"
+    wget --tries=3 --waitretry=30 --quiet  -O "${DOWNLOAD_FOLDER}/docker-binaries.tgz" "https://download.docker.com/${PLATFORM}/static/stable/${ARCH}/docker-${DOCKER_VERSION}.tgz"
    tar -xf "${DOWNLOAD_FOLDER}/docker-binaries.tgz" -C "${DOWNLOAD_FOLDER}"
    mv "${DOWNLOAD_FOLDER}/docker/docker" dist/
 fi
--- a/build/download_docker_compose_binary.sh
+++ b/build/download_docker_compose_binary.sh
@ -21,10 +21,10 @@ fi


 if [[ "$PLATFORM" == "windows" ]]; then
-    wget -O "dist/docker-compose.exe" "https://github.com/docker/compose/releases/download/$COMPOSE_VERSION/docker-compose-windows-${ARCH}.exe"
+    wget --tries=3 --waitretry=30 --quiet  -O "dist/docker-compose.exe" "https://github.com/docker/compose/releases/download/$COMPOSE_VERSION/docker-compose-windows-${ARCH}.exe"
    chmod +x "dist/docker-compose.exe"
 else
-    wget -O "dist/docker-compose" "https://github.com/docker/compose/releases/download/$COMPOSE_VERSION/docker-compose-${PLATFORM}-${ARCH}"
+    wget --tries=3 --waitretry=30 --quiet  -O "dist/docker-compose" "https://github.com/docker/compose/releases/download/$COMPOSE_VERSION/docker-compose-${PLATFORM}-${ARCH}"
    chmod +x "dist/docker-compose"
 fi

--- a/build/download_helm_binary.sh
+++ b/build/download_helm_binary.sh
@ -13,7 +13,7 @@ HELM_DIST="helm-$HELM_VERSION-$PLATFORM-$ARCH"


 if [[ ${PLATFORM} == "windows" ]]; then
-  wget -O tmp.zip "https://get.helm.sh/${HELM_DIST}.zip" && unzip -o -j tmp.zip "${PLATFORM}-${ARCH}/helm.exe" -d dist && rm -f tmp.zip
+  wget --tries=3 --waitretry=30 --quiet  -O tmp.zip "https://get.helm.sh/${HELM_DIST}.zip" && unzip -o -j tmp.zip "${PLATFORM}-${ARCH}/helm.exe" -d dist && rm -f tmp.zip
 else
  wget -qO- "https://get.helm.sh/${HELM_DIST}.tar.gz" | tar -x -z --strip-components 1 "${PLATFORM}-${ARCH}/helm"
  mv "helm" "dist/helm"
--- a/build/download_kubectl_binary.sh
+++ b/build/download_kubectl_binary.sh
@ -12,9 +12,9 @@ KUBECTL_VERSION=$3


 if [[ ${PLATFORM} == "windows" ]]; then
-  wget -O "dist/kubectl.exe" "https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VERSION}/bin/windows/amd64/kubectl.exe"
+  wget --tries=3 --waitretry=30 --quiet  -O "dist/kubectl.exe" "https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VERSION}/bin/windows/amd64/kubectl.exe"
  chmod +x "dist/kubectl.exe"
 else
-  wget -O "dist/kubectl" "https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VERSION}/bin/${PLATFORM}/${ARCH}/kubectl"
+  wget --tries=3 --waitretry=30 --quiet  -O "dist/kubectl" "https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VERSION}/bin/${PLATFORM}/${ARCH}/kubectl"
  chmod +x "dist/kubectl"
 fi
--- a/build/download_mingit_binary.sh
+++ b/build/download_mingit_binary.sh
@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+if [[ $# -ne 3 ]]; then
+    echo "Illegal number of parameters" >&2
+    exit 1
+fi
+
+PLATFORM=$1
+MINGIT_VERSION=$3
+
+if [[ ${PLATFORM} == "windows" ]]; then
+  GIT_VERSION=$(echo $MINGIT_VERSION | cut -d "." -f 1-3)
+  GIT_PATCH_VERSION=$(echo $MINGIT_VERSION | cut -d "." -f 4)
+
+  wget --tries=3 --waitretry=30 --quiet "https://github.com/git-for-windows/git/releases/download/v$GIT_VERSION.windows.$GIT_PATCH_VERSION/MinGit-$GIT_VERSION.$GIT_PATCH_VERSION-busybox-64-bit.zip"
+  unzip "MinGit-$GIT_VERSION.$GIT_PATCH_VERSION-busybox-64-bit.zip" -d dist/mingit
+fi
--- a/build/windows/Dockerfile
+++ b/build/windows/Dockerfile
@ -1,21 +1,14 @@
 ARG OSVERSION
 FROM --platform=linux/amd64 gcr.io/k8s-staging-e2e-test-images/windows-servercore-cache:1.0-linux-amd64-${OSVERSION} as core
-FROM --platform=linux/amd64 alpine:3.14 as downloader
-ENV GIT_VERSION 2.30.0
-ENV GIT_PATCH_VERSION 2
-
-RUN mkdir mingit/ \
-    && wget https://github.com/git-for-windows/git/releases/download/v$GIT_VERSION.windows.$GIT_PATCH_VERSION/MinGit-$GIT_VERSION.$GIT_PATCH_VERSION-busybox-64-bit.zip \
-    && unzip MinGit-$GIT_VERSION.$GIT_PATCH_VERSION-busybox-64-bit.zip -d mingit/

 FROM mcr.microsoft.com/windows/nanoserver:${OSVERSION} as production
 ENV PATH "C:\mingit\cmd;C:\Windows\system32;C:\Windows;"

-COPY --from=downloader /mingit mingit/
 COPY --from=core /Windows/System32/netapi32.dll /Windows/System32/netapi32.dll

 USER ContainerAdministrator

+COPY dist/mingit/ mingit/
 COPY dist/docker.exe /
 COPY dist/docker-compose.exe /
 COPY dist/helm.exe /
@ -33,4 +26,4 @@ EXPOSE 8000

 LABEL io.portainer.server true

-ENTRYPOINT ["/portainer.exe"]
+ENTRYPOINT ["/portainer.exe"]
--- a/pkg/libstack/compose/download.sh
+++ b/pkg/libstack/compose/download.sh
@ -16,10 +16,10 @@ fi


 if [[ "$PLATFORM" == "windows" ]]; then
-    wget -O "docker-compose.exe" "https://github.com/docker/compose/releases/download/$COMPOSE_VERSION/docker-compose-windows-${ARCH}.exe"
+    wget --tries=3 --waitretry=30 --quiet  -O "docker-compose.exe" "https://github.com/docker/compose/releases/download/$COMPOSE_VERSION/docker-compose-windows-${ARCH}.exe"
    chmod +x "docker-compose.exe"
 else
-    wget -O "docker-compose" "https://github.com/docker/compose/releases/download/$COMPOSE_VERSION/docker-compose-${PLATFORM}-${ARCH}"
+    wget --tries=3 --waitretry=30 --quiet  -O "docker-compose" "https://github.com/docker/compose/releases/download/$COMPOSE_VERSION/docker-compose-${PLATFORM}-${ARCH}"
    chmod +x "docker-compose"
 fi