Fix CI failures non-deterministic traefik chart repackaging (#3165)

* Fix CI failures non-deterministic traefik chart repackaging
* Update generated bindata

Signed-off-by: Brad Davidson <brad.davidson@rancher.com>

Dockerfile.dapper

@@ -8,30 +8,31 @@ ENV http_proxy=$http_proxy
 ENV https_proxy=$https_proxy
 ENV no_proxy=$no_proxy
 
-RUN apk -U --no-cache add bash git gcc musl-dev docker vim less file curl wget ca-certificates jq linux-headers zlib-dev tar zip squashfs-tools npm coreutils \
+RUN apk -U --no-cache add bash git gcc musl-dev docker vim less file curl wget ca-certificates jq linux-headers \
-    python2 openssl-dev libffi-dev libseccomp libseccomp-dev make libuv-static sqlite-dev sqlite-static libselinux libselinux-dev zlib-dev zlib-static zstd
+    zlib-dev tar zip squashfs-tools npm coreutils python2 openssl-dev libffi-dev libseccomp libseccomp-dev make \
+    libuv-static sqlite-dev sqlite-static libselinux libselinux-dev zlib-dev zlib-static zstd gzip alpine-sdk binutils-gold
 RUN if [ "$(go env GOARCH)" = "arm64" ]; then                                                               \
-    wget https://github.com/aquasecurity/trivy/releases/download/v0.11.0/trivy_0.11.0_Linux-ARM64.tar.gz && \
+    wget https://github.com/aquasecurity/trivy/releases/download/v0.16.0/trivy_0.16.0_Linux-ARM64.tar.gz && \
-    tar -zxvf trivy_0.11.0_Linux-ARM64.tar.gz                                                            && \
+    tar -zxvf trivy_0.16.0_Linux-ARM64.tar.gz                                                            && \
     mv trivy /usr/local/bin;                                                                                \
     elif [ "$(go env GOARCH)" = "arm" ]; then                                                               \
-    wget https://github.com/aquasecurity/trivy/releases/download/v0.11.0/trivy_0.11.0_Linux-ARM.tar.gz   && \
+    wget https://github.com/aquasecurity/trivy/releases/download/v0.16.0/trivy_0.16.0_Linux-ARM.tar.gz   && \
-    tar -zxvf trivy_0.11.0_Linux-ARM.tar.gz                                                              && \
+    tar -zxvf trivy_0.16.0_Linux-ARM.tar.gz                                                              && \
     mv trivy /usr/local/bin;                                                                                \
     else                                                                                                    \
-    wget https://github.com/aquasecurity/trivy/releases/download/v0.11.0/trivy_0.11.0_Linux-64bit.tar.gz && \
+    wget https://github.com/aquasecurity/trivy/releases/download/v0.16.0/trivy_0.16.0_Linux-64bit.tar.gz && \
-    tar -zxvf trivy_0.11.0_Linux-64bit.tar.gz                                                            && \
+    tar -zxvf trivy_0.16.0_Linux-64bit.tar.gz                                                            && \
     mv trivy /usr/local/bin;                                                                                \
     fi
 # this works for both go 1.15 and 1.16
-RUN GO111MODULE=on go get golang.org/x/tools/cmd/goimports@aa82965741a9fecd12b026fbb3d3c6ed3231b8f8
+RUN GO111MODULE=on GOPROXY=direct go get golang.org/x/tools/cmd/goimports@gopls/v0.6.9
 RUN rm -rf /go/src /go/pkg
 
 RUN if [ "$(go env GOARCH)" = "amd64" ]; then \
-    curl -sL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s v1.30.0; \
+    curl -sL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s v1.38.0; \
     fi
 
-ENV YQ_URL=https://github.com/mikefarah/yq/releases/download/3.4.1/yq_linux
+ENV YQ_URL=https://github.com/mikefarah/yq/releases/download/v4.6.2/yq_linux
 RUN wget -O - ${YQ_URL}_$(go env GOARCH) > /usr/bin/yq && chmod +x /usr/bin/yq
 
 ARG SELINUX=true

Dockerfile.test.dapper

@@ -3,7 +3,7 @@ FROM ${GOLANG}
 
 RUN apk -U --no-cache add bash git gcc musl-dev docker curl jq coreutils python2 openssl py-pip
 
-ENV SONOBUOY_VERSION 0.20.0
+ENV SONOBUOY_VERSION 0.50.0
 
 RUN OS=linux; \
     ARCH=$(go env GOARCH); \

scripts/download

@@ -7,8 +7,12 @@ cd $(dirname $0)/..
 ROOT_VERSION=v0.8.1
 TRAEFIK_VERSION=9.14.2  # appVersion: 2.4.2
 CHARTS_DIR=build/static/charts
+DATA_DIR=build/data
+export TZ=UTC
 
+umask 022
 mkdir -p ${CHARTS_DIR}
+mkdir -p ${DATA_DIR}
 
 curl --compressed -sfL https://github.com/k3s-io/k3s-root/releases/download/${ROOT_VERSION}/k3s-root-${ARCH}.tar | tar xf -
 
@@ -42,18 +46,19 @@ download_and_package_traefik () {
   TRAEFIK_TMP_CRD=${TRAEFIK_TMP_CHART}-crd
 
   # Collect information on chart
-  name=$(cat ${TRAEFIK_TMP_CHART}/Chart.yaml | yq r - 'name')
+  name=$(yq e '.name' ${TRAEFIK_TMP_CHART}/Chart.yaml)
-  api_version=$(cat ${TRAEFIK_TMP_CHART}/Chart.yaml | yq r - 'apiVersion')
+  api_version=$(yq e '.apiVersion' ${TRAEFIK_TMP_CHART}/Chart.yaml)
-  chart_version=$(cat ${TRAEFIK_TMP_CHART}/Chart.yaml | yq r - 'version')
+  chart_version=$(yq e '.version' ${TRAEFIK_TMP_CHART}/Chart.yaml)
 
   # Collect information on CRDs
   crd_apis=()
-  for crd_yaml in ${TRAEFIK_TMP_CHART}/crds/*; do
+  for crd_yaml in $(find ${TRAEFIK_TMP_CHART}/crds -type f | sort); do
-    crd_group=$(yq r ${crd_yaml} 'spec.group')
+    echo "Processing CRD at ${crd_yaml}"
-    crd_kind=$(yq r ${crd_yaml} 'spec.names.kind')
+    crd_group=$(yq e '.spec.group' ${crd_yaml})
-    crd_version=$(yq r ${crd_yaml} 'spec.version')
+    crd_kind=$(yq e '.spec.names.kind' ${crd_yaml})
+    crd_version=$(yq e '.spec.version' ${crd_yaml})
     if [[ -z "$crd_version" ]]; then
-      crd_version=$(yq r ${crd_yaml} 'spec.versions[0].name')
+      crd_version=$(yq e '.spec.versions[0].name' ${crd_yaml})
     fi
     crd_apis+=("${crd_group}/${crd_version}/${crd_kind}")
   done
@@ -67,8 +72,9 @@ download_and_package_traefik () {
   # Copy base template and apply variables to the template
   mkdir -p ${TRAEFIK_TMP_CRD}
   cp -R ./scripts/chart-templates/crd-base/* ${TRAEFIK_TMP_CRD}
-  for template_file in $(find ${TRAEFIK_TMP_CRD} -type f); do
+  for template_file in $(find ${TRAEFIK_TMP_CRD} -type f | sort); do
     # Applies any environment variables currently set onto your template file
+    echo "Templating ${template_file}"
     eval "echo \"$(sed 's/"/\\"/g' ${template_file})\"" > ${template_file}
   done
 
@@ -82,9 +88,14 @@ download_and_package_traefik () {
   rm -rf ${TRAEFIK_TMP_CHART}/crds
 
   # Package charts
-  OPTS="--format gnu --sort=name --owner=0 --group=0 --numeric-owner"
+  OPTS="--format=gnu --sort=name --owner=0 --group=0 --mode=gou-s --numeric-owner --no-acls --no-selinux --no-xattrs"
-  GZIP=-n tar ${OPTS} --mtime='UTC 2021-01-01' -cz -f ${CHARTS_DIR}/${TRAEFIK_FILE} -C ${TMP_DIR} $(basename ${TRAEFIK_TMP_CHART})
+  tar ${OPTS} --mtime='2021-01-01 00:00:00Z' -cf - -C ${TMP_DIR} $(basename ${TRAEFIK_TMP_CHART}) | gzip -n > ${CHARTS_DIR}/${TRAEFIK_FILE}
-  GZIP=-n tar ${OPTS} --mtime='UTC 2021-01-01' -cz -f ${CHARTS_DIR}/${TRAEFIK_CRD_FILE} -C ${TMP_DIR} $(basename ${TRAEFIK_TMP_CRD})
+  tar ${OPTS} --mtime='2021-01-01 00:00:00Z' -cf - -C ${TMP_DIR} $(basename ${TRAEFIK_TMP_CRD}) | gzip -n > ${CHARTS_DIR}/${TRAEFIK_CRD_FILE}
+  for TAR in ${CHARTS_DIR}/${TRAEFIK_FILE} ${CHARTS_DIR}/${TRAEFIK_CRD_FILE}; do
+    sha256sum ${TAR}
+    stat ${TAR}
+    tar -vtf ${TAR}
+  done
 }
 
 TRAEFIK_FILE=traefik-${TRAEFIK_VERSION}.tgz

scripts/validate

@@ -19,7 +19,7 @@ echo Running: "${GO}" generate
 "${GO}" generate
 
 echo Running: golangci-lint
-golangci-lint run
+golangci-lint run -v
 
 GO111MODULE=on go mod tidy
 GO111MODULE=on go mod vendor
@@ -29,6 +29,7 @@ GO111MODULE=on go mod vendor
 if [ -n "$DIRTY" ]; then
     echo Source dir is dirty
     git status --porcelain --untracked-files=no
+    git diff
     exit 1
 fi