diff --git a/pkg/drivers/kic/kic.go b/pkg/drivers/kic/kic.go index aee0501166..10044ec268 100644 --- a/pkg/drivers/kic/kic.go +++ b/pkg/drivers/kic/kic.go @@ -167,6 +167,18 @@ func (d *Driver) prepareSSH() error { if err := cmder.Copy(f); err != nil { return errors.Wrap(err, "copying pub key") } + + // Double-check that the container has not crashed so that we may give a better error message + s, err := oci.ContainerStatus(d.NodeConfig.OCIBinary, d.MachineName) + if err != nil { + return err + } + + if s != state.Running { + excerpt := oci.LogContainerDebug(d.OCIBinary, d.MachineName) + return errors.Wrapf(oci.ErrExitedUnexpectedly, "container name %q state %s: log: %s", d.MachineName, s, excerpt) + } + if rr, err := cmder.RunCmd(exec.Command("chown", "docker:docker", "/home/docker/.ssh/authorized_keys")); err != nil { return errors.Wrapf(err, "apply authorized_keys file ownership, output %s", rr.Output()) } @@ -320,13 +332,13 @@ func (d *Driver) Start() error { } if err := retry.Expo(checkRunning, 500*time.Microsecond, time.Second*30); err != nil { - oci.LogContainerDebug(d.OCIBinary, d.MachineName) + excerpt := oci.LogContainerDebug(d.OCIBinary, d.MachineName) _, err := oci.DaemonInfo(d.OCIBinary) if err != nil { return errors.Wrapf(oci.ErrDaemonInfo, "container name %q", d.MachineName) } - return errors.Wrapf(oci.ErrExitedUnexpectedly, "container name %q", d.MachineName) + return errors.Wrapf(oci.ErrExitedUnexpectedly, "container name %q: log: %s", d.MachineName, excerpt) } return nil } diff --git a/pkg/drivers/kic/oci/errors.go b/pkg/drivers/kic/oci/errors.go index 0f5d7be0a1..8c30201f97 100644 --- a/pkg/drivers/kic/oci/errors.go +++ b/pkg/drivers/kic/oci/errors.go @@ -19,6 +19,7 @@ package oci import ( "errors" "os/exec" + "strings" "github.com/golang/glog" ) @@ -45,7 +46,7 @@ var ErrExitedUnexpectedly = errors.New("container exited unexpectedly") var ErrDaemonInfo = errors.New("daemon info not responding") // LogContainerDebug will print relevant docker/podman infos after a container fails -func LogContainerDebug(ociBin string, name string) { +func LogContainerDebug(ociBin string, name string) string { rr, err := containerInspect(ociBin, name) if err != nil { glog.Warningf("Filed to get postmortem inspect. %s :%v", rr.Command(), err) @@ -74,6 +75,17 @@ func LogContainerDebug(ociBin string, name string) { glog.Infof("postmortem podman info: %+v", pi) } } + + if rr.Stdout.Len() == 0 { + return "" + } + + // If available, return an excerpt of the post-mortem logs for inclusion in error message + excerpt := strings.Split(strings.TrimSpace(rr.Stdout.String()), "\n") + if len(excerpt) > 4 { + excerpt = excerpt[len(excerpt)-4:] + } + return strings.Join(excerpt, "\n") } // containerLogs will return out the logs for a container diff --git a/pkg/drivers/kic/oci/oci.go b/pkg/drivers/kic/oci/oci.go index 8756176592..116c61cd83 100644 --- a/pkg/drivers/kic/oci/oci.go +++ b/pkg/drivers/kic/oci/oci.go @@ -224,12 +224,13 @@ func CreateContainerNode(p CreateParams) error { } if err := retry.Expo(checkRunning, 15*time.Millisecond, 25*time.Second); err != nil { - LogContainerDebug(p.OCIBinary, p.Name) + excerpt := LogContainerDebug(p.OCIBinary, p.Name) _, err := DaemonInfo(p.OCIBinary) if err != nil { return errors.Wrapf(ErrDaemonInfo, "container name %q", p.Name) } - return errors.Wrapf(ErrExitedUnexpectedly, "container name %q", p.Name) + + return errors.Wrapf(ErrExitedUnexpectedly, "container name %q: log: %s", p.Name, excerpt) } return nil