diff --git a/test/e2e/framework.go b/test/e2e/framework.go index 892703f571..506d01f57c 100644 --- a/test/e2e/framework.go +++ b/test/e2e/framework.go @@ -102,6 +102,8 @@ func (f *Framework) afterEach() { // you may or may not see the killing/deletion/cleanup events. dumpAllPodInfo(f.Client) + + dumpAllNodeInfo(f.Client) } // Check whether all nodes are ready after the test. diff --git a/test/e2e/reboot.go b/test/e2e/reboot.go index 13cd85ff09..9e31863192 100644 --- a/test/e2e/reboot.go +++ b/test/e2e/reboot.go @@ -26,7 +26,6 @@ import ( "k8s.io/kubernetes/pkg/labels" . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" ) const ( @@ -44,13 +43,9 @@ const ( ) var _ = Describe("Reboot", func() { - var c *client.Client + f := NewFramework("reboot") BeforeEach(func() { - var err error - c, err = loadClient() - Expect(err).NotTo(HaveOccurred()) - // These tests requires SSH to nodes, so the provider check should be identical to there // (the limiting factor is the implementation of util.go's getSigner(...)). @@ -61,32 +56,32 @@ var _ = Describe("Reboot", func() { It("each node by ordering clean reboot and ensure they function upon restart", func() { // clean shutdown and restart // We sleep 10 seconds to give some time for ssh command to cleanly finish before the node is rebooted. - testReboot(c, "nohup sh -c 'sleep 10 && sudo reboot' >/dev/null 2>&1 &") + testReboot(f.Client, "nohup sh -c 'sleep 10 && sudo reboot' >/dev/null 2>&1 &") }) It("each node by ordering unclean reboot and ensure they function upon restart", func() { // unclean shutdown and restart // We sleep 10 seconds to give some time for ssh command to cleanly finish before the node is shutdown. - testReboot(c, "nohup sh -c 'sleep 10 && echo b | sudo tee /proc/sysrq-trigger' >/dev/null 2>&1 &") + testReboot(f.Client, "nohup sh -c 'sleep 10 && echo b | sudo tee /proc/sysrq-trigger' >/dev/null 2>&1 &") }) It("each node by triggering kernel panic and ensure they function upon restart", func() { // kernel panic // We sleep 10 seconds to give some time for ssh command to cleanly finish before kernel panic is triggered. - testReboot(c, "nohup sh -c 'sleep 10 && echo c | sudo tee /proc/sysrq-trigger' >/dev/null 2>&1 &") + testReboot(f.Client, "nohup sh -c 'sleep 10 && echo c | sudo tee /proc/sysrq-trigger' >/dev/null 2>&1 &") }) It("each node by switching off the network interface and ensure they function upon switch on", func() { // switch the network interface off for a while to simulate a network outage // We sleep 10 seconds to give some time for ssh command to cleanly finish before network is down. - testReboot(c, "nohup sh -c 'sleep 10 && sudo ifdown eth0 && sleep 120 && sudo ifup eth0' >/dev/null 2>&1 &") + testReboot(f.Client, "nohup sh -c 'sleep 10 && sudo ifdown eth0 && sleep 120 && sudo ifup eth0' >/dev/null 2>&1 &") }) It("each node by dropping all inbound packets for a while and ensure they function afterwards", func() { // tell the firewall to drop all inbound packets for a while // We sleep 10 seconds to give some time for ssh command to cleanly finish before starting dropping inbound packets. // We still accept packages send from localhost to prevent monit from restarting kubelet. - testReboot(c, "nohup sh -c 'sleep 10 && sudo iptables -I INPUT 1 -s 127.0.0.1 -j ACCEPT && sudo iptables -I INPUT 2 -j DROP && "+ + testReboot(f.Client, "nohup sh -c 'sleep 10 && sudo iptables -I INPUT 1 -s 127.0.0.1 -j ACCEPT && sudo iptables -I INPUT 2 -j DROP && "+ " sleep 120 && sudo iptables -D INPUT -j DROP && sudo iptables -D INPUT -s 127.0.0.1 -j ACCEPT' >/dev/null 2>&1 &") }) @@ -94,7 +89,7 @@ var _ = Describe("Reboot", func() { // tell the firewall to drop all outbound packets for a while // We sleep 10 seconds to give some time for ssh command to cleanly finish before starting dropping outbound packets. // We still accept packages send to localhost to prevent monit from restarting kubelet. - testReboot(c, "nohup sh -c 'sleep 10 && sudo iptables -I OUTPUT 1 -s 127.0.0.1 -j ACCEPT && sudo iptables -I OUTPUT 2 -j DROP && "+ + testReboot(f.Client, "nohup sh -c 'sleep 10 && sudo iptables -I OUTPUT 1 -s 127.0.0.1 -j ACCEPT && sudo iptables -I OUTPUT 2 -j DROP && "+ " sleep 120 && sudo iptables -D OUTPUT -j DROP && sudo iptables -D OUTPUT -s 127.0.0.1 -j ACCEPT' >/dev/null 2>&1 &") }) }) diff --git a/test/e2e/util.go b/test/e2e/util.go index 46f1e2e5a2..201df26262 100644 --- a/test/e2e/util.go +++ b/test/e2e/util.go @@ -1554,8 +1554,28 @@ func dumpAllPodInfo(c *client.Client) { logPodStates(pods.Items) } +func dumpAllNodeInfo(c *client.Client) { + nodes, err := c.Nodes().List(labels.Everything(), fields.Everything()) + if err != nil { + Logf("unable to fetch node list: %v", err) + return + } + names := make([]string, len(nodes.Items)) + for ix := range nodes.Items { + names[ix] = nodes.Items[ix].Name + } + dumpNodeDebugInfo(c, names) +} + func dumpNodeDebugInfo(c *client.Client, nodeNames []string) { for _, n := range nodeNames { + Logf("\nLogging node info for node %v", n) + node, err := c.Nodes().Get(n) + if err != nil { + Logf("Error getting node info %v", err) + } + Logf("Node Info: %v", node) + Logf("\nLogging kubelet events for node %v", n) for _, e := range getNodeEvents(c, n) { Logf("source %v message %v reason %v first ts %v last ts %v, involved obj %+v",