unexport more funcs
Medya Gh
parent
006c781fbc
commit
968fe86869
|
|
@ -61,7 +61,7 @@ func NewDriver(c Config) *Driver {
|
|||
|
||||
// Create a host using the driver's config
|
||||
func (d *Driver) Create() error {
|
||||
params := createParams{
|
||||
params := oci.CreateParams{
|
||||
Name: d.NodeConfig.MachineName,
|
||||
Image: d.NodeConfig.ImageDigest,
|
||||
ClusterLabel: oci.ClusterLabelKey + "=" + d.MachineName,
|
||||
|
|
@ -87,7 +87,7 @@ func (d *Driver) Create() error {
|
|||
ContainerPort: constants.DockerDaemonPort,
|
||||
},
|
||||
)
|
||||
err := createNode(params)
|
||||
err := oci.CreateContainerNode(params)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "create kic node")
|
||||
}
|
||||
|
|
@ -296,56 +296,3 @@ func (d *Driver) nodeID(nameOrID string) (string, error) {
|
|||
}
|
||||
return string(id), err
|
||||
}
|
||||
|
||||
// createNode creates a new container node
|
||||
func createNode(p createParams) error {
|
||||
runArgs := []string{
|
||||
fmt.Sprintf("--cpus=%s", p.CPUs),
|
||||
fmt.Sprintf("--memory=%s", p.Memory),
|
||||
"-d", // run the container detached
|
||||
"-t", // allocate a tty for entrypoint logs
|
||||
// running containers in a container requires privileged
|
||||
// NOTE: we could try to replicate this with --cap-add, and use less
|
||||
// privileges, but this flag also changes some mounts that are necessary
|
||||
// including some ones docker would otherwise do by default.
|
||||
// for now this is what we want. in the future we may revisit this.
|
||||
"--privileged",
|
||||
"--security-opt", "seccomp=unconfined", // also ignore seccomp
|
||||
"--tmpfs", "/tmp", // various things depend on working /tmp
|
||||
"--tmpfs", "/run", // systemd wants a writable /run
|
||||
// logs,pods be stroed on filesystem vs inside container,
|
||||
"--volume", "/var",
|
||||
// some k8s things want /lib/modules
|
||||
"-v", "/lib/modules:/lib/modules:ro",
|
||||
"--hostname", p.Name, // make hostname match container name
|
||||
"--name", p.Name, // ... and set the container name
|
||||
// label the node with the cluster ID
|
||||
"--label", p.ClusterLabel,
|
||||
// label the node with the role ID
|
||||
"--label", fmt.Sprintf("%s=%s", oci.NodeRoleKey, p.Role),
|
||||
}
|
||||
|
||||
for key, val := range p.Envs {
|
||||
runArgs = append(runArgs, "-e", fmt.Sprintf("%s=%s", key, val))
|
||||
}
|
||||
|
||||
// adds node specific args
|
||||
runArgs = append(runArgs, p.ExtraArgs...)
|
||||
|
||||
if oci.UsernsRemap(p.OCIBinary) {
|
||||
// We need this argument in order to make this command work
|
||||
// in systems that have userns-remap enabled on the docker daemon
|
||||
runArgs = append(runArgs, "--userns=host")
|
||||
}
|
||||
|
||||
_, err := oci.CreateContainer(p.OCIBinary,
|
||||
p.Image,
|
||||
oci.WithRunArgs(runArgs...),
|
||||
oci.WithMounts(p.Mounts),
|
||||
oci.WithPortMappings(p.PortMappings),
|
||||
)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "create a kic node")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -30,8 +30,154 @@ import (
|
|||
"strings"
|
||||
)
|
||||
|
||||
// Inspect return low-level information on containers
|
||||
func Inspect(ociBinary string, containerNameOrID, format string) ([]string, error) {
|
||||
// CreateContainerNode creates a new container node
|
||||
func CreateContainerNode(p CreateParams) error {
|
||||
runArgs := []string{
|
||||
fmt.Sprintf("--cpus=%s", p.CPUs),
|
||||
fmt.Sprintf("--memory=%s", p.Memory),
|
||||
"-d", // run the container detached
|
||||
"-t", // allocate a tty for entrypoint logs
|
||||
// running containers in a container requires privileged
|
||||
// NOTE: we could try to replicate this with --cap-add, and use less
|
||||
// privileges, but this flag also changes some mounts that are necessary
|
||||
// including some ones docker would otherwise do by default.
|
||||
// for now this is what we want. in the future we may revisit this.
|
||||
"--privileged",
|
||||
"--security-opt", "seccomp=unconfined", // also ignore seccomp
|
||||
"--tmpfs", "/tmp", // various things depend on working /tmp
|
||||
"--tmpfs", "/run", // systemd wants a writable /run
|
||||
// logs,pods be stroed on filesystem vs inside container,
|
||||
"--volume", "/var",
|
||||
// some k8s things want /lib/modules
|
||||
"-v", "/lib/modules:/lib/modules:ro",
|
||||
"--hostname", p.Name, // make hostname match container name
|
||||
"--name", p.Name, // ... and set the container name
|
||||
// label the node with the cluster ID
|
||||
"--label", p.ClusterLabel,
|
||||
// label the node with the role ID
|
||||
"--label", fmt.Sprintf("%s=%s", NodeRoleKey, p.Role),
|
||||
}
|
||||
|
||||
for key, val := range p.Envs {
|
||||
runArgs = append(runArgs, "-e", fmt.Sprintf("%s=%s", key, val))
|
||||
}
|
||||
|
||||
// adds node specific args
|
||||
runArgs = append(runArgs, p.ExtraArgs...)
|
||||
|
||||
if isUsernsRemapEnabled(p.OCIBinary) {
|
||||
// We need this argument in order to make this command work
|
||||
// in systems that have userns-remap enabled on the docker daemon
|
||||
runArgs = append(runArgs, "--userns=host")
|
||||
}
|
||||
|
||||
_, err := createContainer(p.OCIBinary,
|
||||
p.Image,
|
||||
withRunArgs(runArgs...),
|
||||
withMounts(p.Mounts),
|
||||
withPortMappings(p.PortMappings),
|
||||
)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "create a kic node")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// CreateContainer creates a container with "docker/podman run"
|
||||
func createContainer(ociBinary string, image string, opts ...createOpt) ([]string, error) {
|
||||
o := &createOpts{}
|
||||
for _, opt := range opts {
|
||||
o = opt(o)
|
||||
}
|
||||
// convert mounts to container run args
|
||||
runArgs := o.RunArgs
|
||||
for _, mount := range o.Mounts {
|
||||
runArgs = append(runArgs, generateMountBindings(mount)...)
|
||||
}
|
||||
for _, portMapping := range o.PortMappings {
|
||||
runArgs = append(runArgs, generatePortMappings(portMapping)...)
|
||||
}
|
||||
// construct the actual docker run argv
|
||||
args := []string{"run"}
|
||||
args = append(args, runArgs...)
|
||||
args = append(args, image)
|
||||
args = append(args, o.ContainerArgs...)
|
||||
cmd := exec.Command(ociBinary, args...)
|
||||
var buff bytes.Buffer
|
||||
cmd.Stdout = &buff
|
||||
cmd.Stderr = &buff
|
||||
err := cmd.Run()
|
||||
scanner := bufio.NewScanner(&buff)
|
||||
var output []string
|
||||
for scanner.Scan() {
|
||||
output = append(output, scanner.Text())
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return output, errors.Wrapf(err, "args: %v output: %s ", args, output)
|
||||
}
|
||||
return output, nil
|
||||
}
|
||||
|
||||
// Copy copies a local asset into the container
|
||||
func Copy(ociBinary string, ociID string, targetDir string, fName string) error {
|
||||
if _, err := os.Stat(fName); os.IsNotExist(err) {
|
||||
return errors.Wrapf(err, "error source %s does not exist", fName)
|
||||
}
|
||||
destination := fmt.Sprintf("%s:%s", ociID, targetDir)
|
||||
cmd := exec.Command(ociBinary, "cp", fName, destination)
|
||||
err := cmd.Run()
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "error copying %s into node", fName)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// HostPortBinding will return port mapping for a container using cli.
|
||||
// example : HostPortBinding("docker", "minikube", "22")
|
||||
// will return the docker assigned port:
|
||||
// 32769, nil
|
||||
// only supports TCP ports
|
||||
func HostPortBinding(ociBinary string, ociID string, contPort int) (int, error) {
|
||||
cmd := exec.Command(ociBinary, "inspect", "-f", fmt.Sprintf("'{{(index (index .NetworkSettings.Ports \"%d/tcp\") 0).HostPort}}'", contPort), ociID)
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
return 0, errors.Wrapf(err, "getting host-bind port %d for container ID %q, output %s", contPort, ociID, out)
|
||||
}
|
||||
o := strings.Trim(string(out), "\n")
|
||||
o = strings.Trim(o, "'")
|
||||
p, err := strconv.Atoi(o)
|
||||
if err != nil {
|
||||
return p, errors.Wrapf(err, "convert host-port %q to number", p)
|
||||
}
|
||||
return p, nil
|
||||
}
|
||||
|
||||
// ContainerIPs returns ipv4,ipv6, error of a container by their name
|
||||
func ContainerIPs(ociBinary string, name string) (string, string, error) {
|
||||
// retrieve the IP address of the node using docker inspect
|
||||
lines, err := inspect(ociBinary, name, "{{range .NetworkSettings.Networks}}{{.IPAddress}},{{.GlobalIPv6Address}}{{end}}")
|
||||
if err != nil {
|
||||
return "", "", errors.Wrap(err, "inspecting NetworkSettings.Networks")
|
||||
}
|
||||
if len(lines) != 1 {
|
||||
return "", "", errors.Errorf("IPs output should only be one line, got %d lines", len(lines))
|
||||
}
|
||||
ips := strings.Split(lines[0], ",")
|
||||
if len(ips) != 2 {
|
||||
return "", "", errors.Errorf("container addresses should have 2 values, got %d values: %+v", len(ips), ips)
|
||||
}
|
||||
return ips[0], ips[1], nil
|
||||
|
||||
}
|
||||
|
||||
// ListOwnedContainers lists all the containres that kic driver created on user's machine using a label
|
||||
func ListOwnedContainers(ociBinary string) ([]string, error) {
|
||||
return listContainersByLabel(ociBinary, ClusterLabelKey)
|
||||
}
|
||||
|
||||
// inspect return low-level information on containers
|
||||
func inspect(ociBinary string, containerNameOrID, format string) ([]string, error) {
|
||||
cmd := exec.Command(ociBinary, "inspect",
|
||||
"-f", format,
|
||||
containerNameOrID) // ... against the "node" container
|
||||
|
|
@ -93,8 +239,8 @@ func generateMountBindings(mounts ...Mount) []string {
|
|||
return result
|
||||
}
|
||||
|
||||
// UsernsRemap checks if userns-remap is enabled in dockerd
|
||||
func UsernsRemap(ociBinary string) bool {
|
||||
// isUsernsRemapEnabled checks if userns-remap is enabled in docker
|
||||
func isUsernsRemapEnabled(ociBinary string) bool {
|
||||
cmd := exec.Command(ociBinary, "info", "--format", "'{{json .SecurityOptions}}'")
|
||||
var buff bytes.Buffer
|
||||
cmd.Stdout = &buff
|
||||
|
|
@ -127,112 +273,31 @@ func generatePortMappings(portMappings ...PortMapping) []string {
|
|||
return result
|
||||
}
|
||||
|
||||
// createOpt is an option for Create
|
||||
type createOpt func(*createOpts) *createOpts
|
||||
|
||||
// actual options struct
|
||||
type createOpts struct {
|
||||
RunArgs []string
|
||||
ContainerArgs []string
|
||||
Mounts []Mount
|
||||
PortMappings []PortMapping
|
||||
}
|
||||
|
||||
// CreateContainer creates a container with "docker/podman run"
|
||||
func CreateContainer(ociBinary string, image string, opts ...createOpt) ([]string, error) {
|
||||
o := &createOpts{}
|
||||
for _, opt := range opts {
|
||||
o = opt(o)
|
||||
}
|
||||
// convert mounts to container run args
|
||||
runArgs := o.RunArgs
|
||||
for _, mount := range o.Mounts {
|
||||
runArgs = append(runArgs, generateMountBindings(mount)...)
|
||||
}
|
||||
for _, portMapping := range o.PortMappings {
|
||||
runArgs = append(runArgs, generatePortMappings(portMapping)...)
|
||||
}
|
||||
// construct the actual docker run argv
|
||||
args := []string{"run"}
|
||||
args = append(args, runArgs...)
|
||||
args = append(args, image)
|
||||
args = append(args, o.ContainerArgs...)
|
||||
cmd := exec.Command(ociBinary, args...)
|
||||
var buff bytes.Buffer
|
||||
cmd.Stdout = &buff
|
||||
cmd.Stderr = &buff
|
||||
err := cmd.Run()
|
||||
scanner := bufio.NewScanner(&buff)
|
||||
var output []string
|
||||
for scanner.Scan() {
|
||||
output = append(output, scanner.Text())
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return output, errors.Wrapf(err, "args: %v output: %s ", args, output)
|
||||
}
|
||||
return output, nil
|
||||
}
|
||||
|
||||
// WithRunArgs sets the args for docker run
|
||||
// withRunArgs sets the args for docker run
|
||||
// as in the args portion of `docker run args... image containerArgs...`
|
||||
func WithRunArgs(args ...string) createOpt {
|
||||
func withRunArgs(args ...string) createOpt {
|
||||
return func(r *createOpts) *createOpts {
|
||||
r.RunArgs = args
|
||||
return r
|
||||
}
|
||||
}
|
||||
|
||||
// WithMounts sets the container mounts
|
||||
func WithMounts(mounts []Mount) createOpt {
|
||||
// withMounts sets the container mounts
|
||||
func withMounts(mounts []Mount) createOpt {
|
||||
return func(r *createOpts) *createOpts {
|
||||
r.Mounts = mounts
|
||||
return r
|
||||
}
|
||||
}
|
||||
|
||||
// WithPortMappings sets the container port mappings to the host
|
||||
func WithPortMappings(portMappings []PortMapping) createOpt {
|
||||
// withPortMappings sets the container port mappings to the host
|
||||
func withPortMappings(portMappings []PortMapping) createOpt {
|
||||
return func(r *createOpts) *createOpts {
|
||||
r.PortMappings = portMappings
|
||||
return r
|
||||
}
|
||||
}
|
||||
|
||||
// Copy copies a local asset into the container
|
||||
func Copy(ociBinary string, ociID string, targetDir string, fName string) error {
|
||||
if _, err := os.Stat(fName); os.IsNotExist(err) {
|
||||
return errors.Wrapf(err, "error source %s does not exist", fName)
|
||||
}
|
||||
destination := fmt.Sprintf("%s:%s", ociID, targetDir)
|
||||
cmd := exec.Command(ociBinary, "cp", fName, destination)
|
||||
err := cmd.Run()
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "error copying %s into node", fName)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// HostPortBinding will return port mapping for a container using cli.
|
||||
// example : HostPortBinding("docker", "minikube", "22")
|
||||
// will return the docker assigned port:
|
||||
// 32769, nil
|
||||
// only supports TCP ports
|
||||
func HostPortBinding(ociBinary string, ociID string, contPort int) (int, error) {
|
||||
cmd := exec.Command(ociBinary, "inspect", "-f", fmt.Sprintf("'{{(index (index .NetworkSettings.Ports \"%d/tcp\") 0).HostPort}}'", contPort), ociID)
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
return 0, errors.Wrapf(err, "getting host-bind port %d for container ID %q, output %s", contPort, ociID, out)
|
||||
}
|
||||
o := strings.Trim(string(out), "\n")
|
||||
o = strings.Trim(o, "'")
|
||||
p, err := strconv.Atoi(o)
|
||||
if err != nil {
|
||||
return p, errors.Wrapf(err, "convert host-port %q to number", p)
|
||||
}
|
||||
return p, nil
|
||||
}
|
||||
|
||||
// listContainersByLabel lists all the containres that kic driver created on user's machine using a label
|
||||
// io.x-k8s.kic.cluster
|
||||
func listContainersByLabel(ociBinary string, label string) ([]string, error) {
|
||||
|
|
@ -248,26 +313,3 @@ func listContainersByLabel(ociBinary string, label string) ([]string, error) {
|
|||
}
|
||||
return lines, err
|
||||
}
|
||||
|
||||
// ContainerIPs returns ipv4,ipv6, error of a container by their name
|
||||
func ContainerIPs(ociBinary string, name string) (string, string, error) {
|
||||
// retrieve the IP address of the node using docker inspect
|
||||
lines, err := Inspect(ociBinary, name, "{{range .NetworkSettings.Networks}}{{.IPAddress}},{{.GlobalIPv6Address}}{{end}}")
|
||||
if err != nil {
|
||||
return "", "", errors.Wrap(err, "inspecting NetworkSettings.Networks")
|
||||
}
|
||||
if len(lines) != 1 {
|
||||
return "", "", errors.Errorf("IPs output should only be one line, got %d lines", len(lines))
|
||||
}
|
||||
ips := strings.Split(lines[0], ",")
|
||||
if len(ips) != 2 {
|
||||
return "", "", errors.Errorf("container addresses should have 2 values, got %d values: %+v", len(ips), ips)
|
||||
}
|
||||
return ips[0], ips[1], nil
|
||||
|
||||
}
|
||||
|
||||
// ListOwnedContainers lists all the containres that kic driver created on user's machine using a label
|
||||
func ListOwnedContainers(ociBinary string) ([]string, error) {
|
||||
return listContainersByLabel(ociBinary, ClusterLabelKey)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -25,6 +25,32 @@ const (
|
|||
NodeRoleKey = "io.k8s.sigs.kic.role"
|
||||
)
|
||||
|
||||
type CreateParams struct {
|
||||
Name string // used for container name and hostname
|
||||
Image string // container image to use to create the node.
|
||||
ClusterLabel string // label the containers we create using minikube so we can clean up
|
||||
Role string // currently only role supported is control-plane
|
||||
Mounts []Mount // volume mounts
|
||||
APIServerPort int // kubernetes api server port
|
||||
PortMappings []PortMapping // ports to map to container from host
|
||||
CPUs string // number of cpu cores assign to container
|
||||
Memory string // memory (mbs) to assign to the container
|
||||
Envs map[string]string // environment variables to pass to the container
|
||||
ExtraArgs []string // a list of any extra option to pass to oci binary during creation time, for example --expose 8080...
|
||||
OCIBinary string // docker or podman
|
||||
}
|
||||
|
||||
// createOpt is an option for Create
|
||||
type createOpt func(*createOpts) *createOpts
|
||||
|
||||
// actual options struct
|
||||
type createOpts struct {
|
||||
RunArgs []string
|
||||
ContainerArgs []string
|
||||
Mounts []Mount
|
||||
PortMappings []PortMapping
|
||||
}
|
||||
|
||||
/*
|
||||
These types are from
|
||||
https://github.com/kubernetes/kubernetes/blob/063e7ff358fdc8b0916e6f39beedc0d025734cb1/pkg/kubelet/apis/cri/runtime/v1alpha2/api.pb.go#L183
|
||||
|
|
|
|||
|
|
@ -44,18 +44,3 @@ type Config struct {
|
|||
PortMappings []oci.PortMapping // container port mappings
|
||||
Envs map[string]string // key,value of environment variables passed to the node
|
||||
}
|
||||
|
||||
type createParams struct {
|
||||
Name string // used for container name and hostname
|
||||
Image string // container image to use to create the node.
|
||||
ClusterLabel string // label the containers we create using minikube so we can clean up
|
||||
Role string // currently only role supported is control-plane
|
||||
Mounts []oci.Mount // volume mounts
|
||||
APIServerPort int // kubernetes api server port
|
||||
PortMappings []oci.PortMapping // ports to map to container from host
|
||||
CPUs string // number of cpu cores assign to container
|
||||
Memory string // memory (mbs) to assign to the container
|
||||
Envs map[string]string // environment variables to pass to the container
|
||||
ExtraArgs []string // a list of any extra option to pass to oci binary during creation time, for example --expose 8080...
|
||||
OCIBinary string // docker or podman
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue