Migrate E2E SecretsEncryption to Docker test

Remove secrets encryption E2E from Drone

Signed-off-by: Derek Nola <derek.nola@suse.com>

.drone.yml

@@ -617,11 +617,6 @@ steps:
     vagrant destroy -f
     go test -v -timeout=45m ./validatecluster_test.go -ci -local
     cp ./coverage.out /tmp/artifacts/validate-coverage.out
-  - |
-    cd ../secretsencryption
-    vagrant destroy -f
-    go test -v -timeout=30m ./secretsencryption_test.go -ci -local
-    cp ./coverage.out /tmp/artifacts/se-coverage.out
   - |
     cd ../splitserver
     vagrant destroy -f
@@ -661,7 +656,6 @@ steps:
       from_secret: codecov_token
     files:
       - /tmp/artifacts/validate-coverage.out
-      - /tmp/artifacts/se-coverage.out
       - /tmp/artifacts/split-coverage.out
       - /tmp/artifacts/upgrade-coverage.out
     flags: 

.github/workflows/e2e.yaml

@@ -160,11 +160,13 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        dtest: [autoimport, basics, bootstraptoken, cacerts, etcd, hardened, lazypull, skew, snapshotrestore, token, upgrade]
+        dtest: [autoimport, basics, bootstraptoken, cacerts, etcd, hardened, lazypull, skew, secretsencryption, snapshotrestore, token, upgrade]
         arch: [amd64, arm64]
         exclude:
           - dtest: autoimport
             arch: arm64
+          - dtest: secretsencryption
+            arch: arm64
           - dtest: snapshotrestore
             arch: arm64
     runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }}
@@ -207,7 +209,7 @@ jobs:
         cd ./tests/docker/${{ matrix.dtest }}
         
         # These tests use rancher/systemd-node and have different flags.
-        CI_TESTS="autoimport hardened snapshotrestore token"
+        CI_TESTS="autoimport hardened secretsencryption snapshotrestore token"
         if [ ${{ matrix.dtest }} = "upgrade" ] || [ ${{ matrix.dtest }} = "skew" ]; then
           ./${{ matrix.dtest }}.test -k3sImage=$K3S_IMAGE -channel=$CHANNEL
         elif [[ $CI_TESTS =~ ${{ matrix.dtest }} ]]; then

tests/docker/resources/secrets.yaml

@@ -0,0 +1,49 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: docker-secret1
+type: Opaque
+stringData:
+  config.yaml: |
+    key: "hello"
+    val: "world"
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: docker-secret2
+type: Opaque
+stringData:
+  config.yaml: |
+    key: "good"
+    val: "day"
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: docker-secret3
+type: Opaque
+stringData:
+  config.yaml: |
+    key: "top-secret"
+    val: "information"
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: docker-secret4
+type: Opaque
+stringData:
+  config.yaml: |
+    key: "lock"
+    val: "key"
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: docker-secret5
+type: Opaque
+stringData:
+  config.yaml: |
+    key: "last"
+    val: "call"

tests/docker/secretsencryption/secretsencryption_test.go

@@ -0,0 +1,184 @@
+package secretsencryption
+
+import (
+	"flag"
+	"testing"
+
+	"github.com/k3s-io/k3s/tests"
+	"github.com/k3s-io/k3s/tests/docker"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var serverCount = flag.Int("serverCount", 3, "number of server nodes")
+var ci = flag.Bool("ci", false, "running on CI")
+
+func Test_DockerSecretsEncryption(t *testing.T) {
+	RegisterFailHandler(Fail)
+	flag.Parse()
+	suiteConfig, reporterConfig := GinkgoConfiguration()
+	RunSpecs(t, "Secrets Encryption Test Suite", suiteConfig, reporterConfig)
+}
+
+var tc *docker.TestConfig
+
+var _ = Describe("Verify Secrets Encryption Rotation", Ordered, func() {
+	Context("Setup Cluster", func() {
+		It("should provision servers and agents", func() {
+			var err error
+			tc, err = docker.NewTestConfig("rancher/systemd-node")
+			Expect(err).NotTo(HaveOccurred())
+			tc.ServerYaml = `secrets-encryption: true`
+			Expect(tc.ProvisionServers(*serverCount)).To(Succeed())
+			Eventually(func() error {
+				return tests.CheckDefaultDeployments(tc.KubeconfigFile)
+			}, "60s", "5s").Should(Succeed())
+			Eventually(func() error {
+				return tests.NodesReady(tc.KubeconfigFile, tc.GetNodeNames())
+			}, "40s", "5s").Should(Succeed())
+		})
+	})
+	Context("Secrets Keys are rotated:", func() {
+		It("Deploys several secrets", func() {
+			_, err := tc.DeployWorkload("secrets.yaml")
+			Expect(err).NotTo(HaveOccurred(), "Secrets not deployed")
+		})
+
+		It("Verifies encryption start stage", func() {
+			cmd := "k3s secrets-encrypt status"
+			for _, node := range tc.Servers {
+				res, err := node.RunCmdOnNode(cmd)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(res).Should(ContainSubstring("Encryption Status: Enabled"))
+				Expect(res).Should(ContainSubstring("Current Rotation Stage: start"))
+				Expect(res).Should(ContainSubstring("Server Encryption Hashes: All hashes match"))
+			}
+		})
+
+		It("Rotates the Secrets-Encryption Keys", func() {
+			cmd := "k3s secrets-encrypt rotate-keys"
+			res, err := tc.Servers[0].RunCmdOnNode(cmd)
+			Expect(err).NotTo(HaveOccurred(), res)
+			for i, node := range tc.Servers {
+				Eventually(func(g Gomega) {
+					cmd := "k3s secrets-encrypt status"
+					res, err := node.RunCmdOnNode(cmd)
+					g.Expect(err).NotTo(HaveOccurred(), res)
+					g.Expect(res).Should(ContainSubstring("Server Encryption Hashes: hash does not match"))
+					if i == 0 {
+						g.Expect(res).Should(ContainSubstring("Current Rotation Stage: reencrypt_finished"))
+					} else {
+						g.Expect(res).Should(ContainSubstring("Current Rotation Stage: start"))
+					}
+				}, "420s", "10s").Should(Succeed())
+			}
+		})
+
+		It("Restarts K3s servers", func() {
+			Expect(docker.RestartCluster(tc.Servers)).To(Succeed())
+		})
+
+		It("Verifies reencryption_finished stage", func() {
+			cmd := "k3s secrets-encrypt status"
+			for _, node := range tc.Servers {
+				Eventually(func(g Gomega) {
+					res, err := node.RunCmdOnNode(cmd)
+					g.Expect(err).NotTo(HaveOccurred())
+					g.Expect(res).Should(ContainSubstring("Encryption Status: Enabled"))
+					g.Expect(res).Should(ContainSubstring("Current Rotation Stage: reencrypt_finished"))
+					g.Expect(res).Should(ContainSubstring("Server Encryption Hashes: All hashes match"))
+				}, "420s", "2s").Should(Succeed())
+			}
+		})
+
+	})
+
+	Context("Disabling Secrets-Encryption", func() {
+		It("Disables encryption", func() {
+			cmd := "k3s secrets-encrypt disable"
+			res, err := tc.Servers[0].RunCmdOnNode(cmd)
+			Expect(err).NotTo(HaveOccurred(), res)
+
+			cmd = "k3s secrets-encrypt status"
+			Eventually(func() (string, error) {
+				return tc.Servers[0].RunCmdOnNode(cmd)
+			}, "240s", "10s").Should(ContainSubstring("Current Rotation Stage: reencrypt_finished"))
+
+			for i, node := range tc.Servers {
+				Eventually(func(g Gomega) {
+					res, err := node.RunCmdOnNode(cmd)
+					g.Expect(err).NotTo(HaveOccurred(), res)
+					if i == 0 {
+						g.Expect(res).Should(ContainSubstring("Encryption Status: Disabled"))
+					} else {
+						g.Expect(res).Should(ContainSubstring("Encryption Status: Enabled"))
+					}
+				}, "420s", "2s").Should(Succeed())
+			}
+		})
+
+		It("Restarts K3s servers", func() {
+			Expect(docker.RestartCluster(tc.Servers)).To(Succeed())
+		})
+
+		It("Verifies encryption disabled on all nodes", func() {
+			cmd := "k3s secrets-encrypt status"
+			for _, node := range tc.Servers {
+				Eventually(func(g Gomega) {
+					g.Expect(node.RunCmdOnNode(cmd)).Should(ContainSubstring("Encryption Status: Disabled"))
+				}, "420s", "2s").Should(Succeed())
+			}
+		})
+
+	})
+
+	Context("Enabling Secrets-Encryption", func() {
+		It("Enables encryption", func() {
+			cmd := "k3s secrets-encrypt enable"
+			res, err := tc.Servers[0].RunCmdOnNode(cmd)
+			Expect(err).NotTo(HaveOccurred(), res)
+
+			cmd = "k3s secrets-encrypt status"
+			Eventually(func() (string, error) {
+				return tc.Servers[0].RunCmdOnNode(cmd)
+			}, "180s", "5s").Should(ContainSubstring("Current Rotation Stage: reencrypt_finished"))
+
+			for i, node := range tc.Servers {
+				Eventually(func(g Gomega) {
+					res, err := node.RunCmdOnNode(cmd)
+					g.Expect(err).NotTo(HaveOccurred(), res)
+					if i == 0 {
+						g.Expect(res).Should(ContainSubstring("Encryption Status: Enabled"))
+					} else {
+						g.Expect(res).Should(ContainSubstring("Encryption Status: Disabled"))
+					}
+				}, "420s", "2s").Should(Succeed())
+			}
+		})
+
+		It("Restarts K3s servers", func() {
+			Expect(docker.RestartCluster(tc.Servers)).To(Succeed())
+		})
+
+		It("Verifies encryption enabled on all nodes", func() {
+			cmd := "k3s secrets-encrypt status"
+			for _, node := range tc.Servers {
+				Eventually(func(g Gomega) {
+					g.Expect(node.RunCmdOnNode(cmd)).Should(ContainSubstring("Encryption Status: Enabled"))
+				}, "420s", "2s").Should(Succeed())
+			}
+		})
+	})
+
+})
+
+var failed bool
+var _ = AfterEach(func() {
+	failed = failed || CurrentSpecReport().Failed()
+})
+
+var _ = AfterSuite(func() {
+	if *ci || (tc != nil && !failed) {
+		Expect(tc.Cleanup()).To(Succeed())
+	}
+})