diff --git a/.github/workflows/pr_verified.yaml b/.github/workflows/pr_verified.yaml
index 190800e523..7b674572ce 100644
--- a/.github/workflows/pr_verified.yaml
+++ b/.github/workflows/pr_verified.yaml
@@ -40,7 +40,7 @@ jobs:
         run: |
           sudo apt-get update
           sudo apt-get install -y libvirt-dev
-          make cross e2e-cross debs
+          MINIKUBE_BUILD_IN_DOCKER=y make cross e2e-cross debs
           cp -r test/integration/testdata ./out
           whoami
           echo github ref $GITHUB_REF
diff --git a/hack/jenkins/minikube_cross_build_and_upload.sh b/hack/jenkins/minikube_cross_build_and_upload.sh
index ed5394133f..94750f6996 100755
--- a/hack/jenkins/minikube_cross_build_and_upload.sh
+++ b/hack/jenkins/minikube_cross_build_and_upload.sh
@@ -46,6 +46,8 @@ make -j 16 \
   out/minikube_${DEB_VER}_amd64.deb \
   out/minikube_${DEB_VER}_arm64.deb \
   out/docker-machine-driver-kvm2_$(make deb_version_base).deb \
+  out/docker-machine-driver-kvm2_${DEB_VER}_amd64.deb \
+  out/docker-machine-driver-kvm2_${DEB_VER}_arm64.deb \
 && failed=$? || failed=$?
 
 BUILT_VERSION=$("out/minikube-$(go env GOOS)-$(go env GOARCH)" version)
@@ -70,7 +72,7 @@ fi
 cp -r test/integration/testdata out/
 
 # Don't upload the buildroot artifacts if they exist
-rm -r out/buildroot || true
+rm -rf out/buildroot
 
 # At this point, the out directory contains the jenkins scripts (populated by jenkins),
 # testdata, and our build output. Push the changes to GCS so that worker nodes can re-use them.