Add journalctl logs to E2E tests

Signed-off-by: Derek Nola <derek.nola@suse.com>

.github/workflows/build-k3s.yaml

@@ -0,0 +1,39 @@
+name: Build K3s
+
+on: 
+  workflow_call:
+   inputs:
+    upload-repo:
+      type: boolean
+      required: false
+      default: false
+
+jobs:
+  build:
+    name: Build
+    runs-on: ubuntu-20.04
+    timeout-minutes: 20
+    steps:
+    - name: Checkout K3s
+      uses: actions/checkout@v3
+    - name: Build K3s binary
+      run: |
+        DOCKER_BUILDKIT=1 SKIP_AIRGAP=1 SKIP_VALIDATE=1 make
+      
+    - name: bundle repo
+      if: inputs.upload-repo == true
+      run: |
+        tar -czvf ../k3s-repo.tar.gz .
+        mv ../k3s-repo.tar.gz .
+    - name: "Upload K3s directory"
+      if: inputs.upload-repo == true
+      uses: actions/upload-artifact@v3
+      with:
+        name: k3s-repo.tar.gz
+        path: k3s-repo.tar.gz
+    - name: "Upload K3s binary"
+      if: inputs.upload-repo == false
+      uses: actions/upload-artifact@v3
+      with:
+        name: k3s
+        path: dist/artifacts/k3s

.github/workflows/integration.yaml

@@ -5,9 +5,8 @@ on:
       - "**.md"
       - "channel.yaml"
       - "install.sh"
-      - "tests/snapshotter/**"
+      - "tests/**"
-      - "tests/install/**"
+      - "!tests/integration**"
-      - "tests/cgroup/**"
       - ".github/**"
       - "!.github/workflows/integration.yaml"
   pull_request:
@@ -15,29 +14,14 @@ on:
       - "**.md"
       - "channel.yaml"
       - "install.sh"
-      - "tests/snapshotter/**"
+      - "tests/**"
-      - "tests/install/**"
+      - "!tests/integration**"
-      - "tests/cgroup/**"
       - ".github/**"
       - "!.github/workflows/integration.yaml"
   workflow_dispatch: {}
 jobs:
   build:
-    name: Build
+    uses: ./.github/workflows/build-k3s.yaml
-    runs-on: ubuntu-20.04
-    timeout-minutes: 20
-    steps:
-    - name: "Checkout"
-      uses: actions/checkout@v2
-      with:
-        fetch-depth: 1
-    - name: "Make"
-      run: DOCKER_BUILDKIT=1 SKIP_VALIDATE=1 make
-    - name: "Upload k3s binary"
-      uses: actions/upload-artifact@v2
-      with:
-        name: k3s
-        path: dist/artifacts/k3s
   test:
     needs: build
     name: Integration Tests

tests/e2e/clusterreset/clusterreset_test.go

@@ -47,7 +47,7 @@ var _ = Describe("Verify Create", Ordered, func() {
 			} else {
 				serverNodeNames, agentNodeNames, err = e2e.CreateCluster(*nodeOS, *serverCount, *agentCount)
 			}
-			Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog())
+			Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog(err))
 			fmt.Println("CLUSTER CONFIG")
 			fmt.Println("OS:", *nodeOS)
 			fmt.Println("Server Nodes:", serverNodeNames)

tests/e2e/docker/docker_test.go

@@ -37,7 +37,7 @@ var _ = Describe("Verify CRI-Dockerd", Ordered, func() {
 		It("Starts up with no issues", func() {
 			var err error
 			serverNodeNames, agentNodeNames, err = e2e.CreateCluster(*nodeOS, *serverCount, *agentCount)
-			Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog())
+			Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog(err))
 			fmt.Println("CLUSTER CONFIG")
 			fmt.Println("OS:", *nodeOS)
 			fmt.Println("Server Nodes:", serverNodeNames)

tests/e2e/dualstack/dualstack_test.go

@@ -75,7 +75,7 @@ var _ = Describe("Verify DualStack Configuration", Ordered, func() {
 	It("Starts up with no issues", func() {
 		var err error
 		serverNodeNames, agentNodeNames, err = e2e.CreateCluster(*nodeOS, *serverCount, *agentCount)
-		Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog())
+		Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog(err))
 		fmt.Println("CLUSTER CONFIG")
 		fmt.Println("OS:", *nodeOS)
 		fmt.Println("Server Nodes:", serverNodeNames)

tests/e2e/splitserver/splitserver_test.go

@@ -75,7 +75,7 @@ var _ = Describe("Verify Create", Ordered, func() {
 		It("Starts up with no issues", func() {
 			var err error
 			etcdNodeNames, cpNodeNames, agentNodeNames, err = createSplitCluster(*nodeOS, *etcdCount, *controlPlaneCount, *agentCount)
-			Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog())
+			Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog(err))
 			fmt.Println("CLUSTER CONFIG")
 			fmt.Println("OS:", *nodeOS)
 			fmt.Println("Etcd Server Nodes:", etcdNodeNames)

tests/e2e/testutils.go

@@ -2,6 +2,7 @@ package e2e
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"io/ioutil"
 	"os"
@@ -32,6 +33,28 @@ type Pod struct {
 	Node      string
 }
 
+type NodeError struct {
+	Node string
+	Cmd  string
+	Err  error
+}
+
+func (ne *NodeError) Error() string {
+	return fmt.Sprintf("failed creating cluster: %s: %v", ne.Cmd, ne.Err)
+}
+
+func (ne *NodeError) Unwrap() error {
+	return ne.Err
+}
+
+func newNodeError(cmd, node string, err error) *NodeError {
+	return &NodeError{
+		Cmd:  cmd,
+		Node: node,
+		Err:  err,
+	}
+}
+
 func CountOfStringInSlice(str string, pods []Pod) int {
 	count := 0
 	for _, pod := range pods {
@@ -79,7 +102,7 @@ func CreateCluster(nodeOS string, serverCount, agentCount int) ([]string, []stri
 
 	fmt.Println(cmd)
 	if _, err := RunCommand(cmd); err != nil {
-		return nil, nil, fmt.Errorf("failed creating cluster: %s: %v", cmd, err)
+		return nil, nil, newNodeError(cmd, serverNodeNames[0], err)
 	}
 	// Bring up the rest of the nodes in parallel
 	errg, _ := errgroup.WithContext(context.Background())
@@ -87,7 +110,7 @@ func CreateCluster(nodeOS string, serverCount, agentCount int) ([]string, []stri
 		cmd := fmt.Sprintf(`%s %s vagrant up %s &>> vagrant.log`, nodeEnvs, testOptions, node)
 		errg.Go(func() error {
 			if _, err := RunCommand(cmd); err != nil {
-				return fmt.Errorf("failed creating cluster: %s: %v", cmd, err)
+				return newNodeError(cmd, node, err)
 			}
 			return nil
 		})
@@ -128,7 +151,7 @@ func CreateLocalCluster(nodeOS string, serverCount, agentCount int) ([]string, [
 		}
 		errg.Go(func() error {
 			if _, err := RunCommand(cmd); err != nil {
-				return fmt.Errorf("failed creating cluster: %s: %v", cmd, err)
+				return fmt.Errorf("failed initializing nodes: %s: %v", cmd, err)
 			}
 			return nil
 		})
@@ -154,7 +177,7 @@ func CreateLocalCluster(nodeOS string, serverCount, agentCount int) ([]string, [
 		cmd = fmt.Sprintf(`%s %s vagrant provision %s &>> vagrant.log`, nodeEnvs, testOptions, node)
 		errg.Go(func() error {
 			if _, err := RunCommand(cmd); err != nil {
-				return fmt.Errorf("failed creating cluster: %s: %v", cmd, err)
+				return newNodeError(cmd, node, err)
 			}
 			return nil
 		})
@@ -251,7 +274,16 @@ func GenKubeConfigFile(serverName string) (string, error) {
 	return kubeConfigFile, nil
 }
 
-func GetVagrantLog() string {
+// GetVagrantLog returns the logs of on vagrant commands that initialize the nodes and provision K3s on each node.
+// It also attempts to fetch the systemctl logs of K3s on nodes where the k3s.service failed.
+func GetVagrantLog(cErr error) string {
+	var nodeErr *NodeError
+	nodeJournal := ""
+	if errors.As(cErr, &nodeErr) {
+		nodeJournal, _ = RunCommand("vagrant ssh " + nodeErr.Node + " -c \"sudo journalctl -u k3s* --no-pager\"")
+		nodeJournal = "\nNode Journal Logs:\n" + nodeJournal
+	}
+
 	log, err := os.Open("vagrant.log")
 	if err != nil {
 		return err.Error()
@@ -260,7 +292,7 @@ func GetVagrantLog() string {
 	if err != nil {
 		return err.Error()
 	}
-	return string(bytes)
+	return string(bytes) + nodeJournal
 }
 
 func ParseNodes(kubeConfig string, print bool) ([]Node, error) {

tests/e2e/upgradecluster/upgradecluster_test.go

@@ -45,7 +45,7 @@ var _ = Describe("Verify Upgrade", Ordered, func() {
 		It("Starts up with no issues", func() {
 			var err error
 			serverNodeNames, agentNodeNames, err = e2e.CreateCluster(*nodeOS, *serverCount, *agentCount)
-			Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog())
+			Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog(err))
 			fmt.Println("CLUSTER CONFIG")
 			fmt.Println("OS:", *nodeOS)
 			fmt.Println("Server Nodes:", serverNodeNames)

tests/e2e/validatecluster/validatecluster_test.go

@@ -48,7 +48,7 @@ var _ = Describe("Verify Create", Ordered, func() {
 			} else {
 				serverNodeNames, agentNodeNames, err = e2e.CreateCluster(*nodeOS, *serverCount, *agentCount)
 			}
-			Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog())
+			Expect(err).NotTo(HaveOccurred(), e2e.GetVagrantLog(err))
 			fmt.Println("CLUSTER CONFIG")
 			fmt.Println("OS:", *nodeOS)
 			fmt.Println("Server Nodes:", serverNodeNames)