Merge pull request #8828 from tstromberg/kic-show-errs

kic: Show log excerpt for ErrExitedUnexpectedly, check state within prepareSSH

pkg/drivers/kic/kic.go

@@ -167,6 +167,18 @@ func (d *Driver) prepareSSH() error {
 	if err := cmder.Copy(f); err != nil {
 		return errors.Wrap(err, "copying pub key")
 	}
+
+	// Double-check that the container has not crashed so that we may give a better error message
+	s, err := oci.ContainerStatus(d.NodeConfig.OCIBinary, d.MachineName)
+	if err != nil {
+		return err
+	}
+
+	if s != state.Running {
+		excerpt := oci.LogContainerDebug(d.OCIBinary, d.MachineName)
+		return errors.Wrapf(oci.ErrExitedUnexpectedly, "container name %q state %s: log: %s", d.MachineName, s, excerpt)
+	}
+
 	if rr, err := cmder.RunCmd(exec.Command("chown", "docker:docker", "/home/docker/.ssh/authorized_keys")); err != nil {
 		return errors.Wrapf(err, "apply authorized_keys file ownership, output %s", rr.Output())
 	}
@@ -320,13 +332,13 @@ func (d *Driver) Start() error {
 	}
 
 	if err := retry.Expo(checkRunning, 500*time.Microsecond, time.Second*30); err != nil {
-		oci.LogContainerDebug(d.OCIBinary, d.MachineName)
+		excerpt := oci.LogContainerDebug(d.OCIBinary, d.MachineName)
 		_, err := oci.DaemonInfo(d.OCIBinary)
 		if err != nil {
 			return errors.Wrapf(oci.ErrDaemonInfo, "container name %q", d.MachineName)
 		}
 
-		return errors.Wrapf(oci.ErrExitedUnexpectedly, "container name %q", d.MachineName)
+		return errors.Wrapf(oci.ErrExitedUnexpectedly, "container name %q: log: %s", d.MachineName, excerpt)
 	}
 	return nil
 }

pkg/drivers/kic/oci/errors.go

@@ -19,6 +19,7 @@ package oci
 import (
 	"errors"
 	"os/exec"
+	"strings"
 
 	"github.com/golang/glog"
 )
@@ -45,7 +46,7 @@ var ErrExitedUnexpectedly = errors.New("container exited unexpectedly")
 var ErrDaemonInfo = errors.New("daemon info not responding")
 
 // LogContainerDebug will print relevant docker/podman infos after a container fails
-func LogContainerDebug(ociBin string, name string) {
+func LogContainerDebug(ociBin string, name string) string {
 	rr, err := containerInspect(ociBin, name)
 	if err != nil {
 		glog.Warningf("Filed to get postmortem inspect. %s :%v", rr.Command(), err)
@@ -74,6 +75,17 @@ func LogContainerDebug(ociBin string, name string) {
 			glog.Infof("postmortem podman info: %+v", pi)
 		}
 	}
+
+	if rr.Stdout.Len() == 0 {
+		return ""
+	}
+
+	// If available, return an excerpt of the post-mortem logs for inclusion in error message
+	excerpt := strings.Split(strings.TrimSpace(rr.Stdout.String()), "\n")
+	if len(excerpt) > 4 {
+		excerpt = excerpt[len(excerpt)-4:]
+	}
+	return strings.Join(excerpt, "\n")
 }
 
 // containerLogs will return out the logs for a container

pkg/drivers/kic/oci/oci.go

@@ -224,12 +224,13 @@ func CreateContainerNode(p CreateParams) error {
 	}
 
 	if err := retry.Expo(checkRunning, 15*time.Millisecond, 25*time.Second); err != nil {
-		LogContainerDebug(p.OCIBinary, p.Name)
+		excerpt := LogContainerDebug(p.OCIBinary, p.Name)
 		_, err := DaemonInfo(p.OCIBinary)
 		if err != nil {
 			return errors.Wrapf(ErrDaemonInfo, "container name %q", p.Name)
 		}
-		return errors.Wrapf(ErrExitedUnexpectedly, "container name %q", p.Name)
+
+		return errors.Wrapf(ErrExitedUnexpectedly, "container name %q: log: %s", p.Name, excerpt)
 	}
 
 	return nil