315 lines
9.7 KiB
Go
315 lines
9.7 KiB
Go
/*
|
|
Copyright 2025 The Kubernetes Authors All rights reserved.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package deployer
|
|
|
|
import (
|
|
"context"
|
|
"crypto/rand"
|
|
"crypto/rsa"
|
|
"crypto/x509"
|
|
"encoding/json"
|
|
"encoding/pem"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"strconv"
|
|
|
|
"github.com/docker/go-connections/nat"
|
|
"github.com/google/uuid"
|
|
"github.com/moby/moby/api/types/container"
|
|
"github.com/moby/moby/client"
|
|
"github.com/phayes/freeport"
|
|
gossh "golang.org/x/crypto/ssh"
|
|
"k8s.io/klog/v2"
|
|
)
|
|
|
|
var sshSetupScript = `
|
|
#!/bin/bash
|
|
USERNAME=%s
|
|
apt update
|
|
apt install -y openssh-server rsync
|
|
service ssh start
|
|
useradd -m -s /bin/bash ${USERNAME}
|
|
passwd -d ${USERNAME}
|
|
|
|
USER_HOME=$(eval echo "~${USERNAME}")
|
|
mkdir -p $USER_HOME/.ssh
|
|
chmod 700 $USER_HOME/.ssh
|
|
echo "%s" >> $USER_HOME/.ssh/authorized_keys
|
|
chmod 600 $USER_HOME/.ssh/authorized_keys
|
|
chown -R $USERNAME:$USERNAME "$USER_HOME/.ssh"
|
|
`
|
|
|
|
type MiniTestDockerDeployer struct {
|
|
ctx context.Context
|
|
config *MiniTestDockerConfig
|
|
isUp bool
|
|
|
|
dockerClient *client.Client
|
|
containerSHA string
|
|
|
|
sshPrivateKeyFile string
|
|
sshPublicKeyFile string
|
|
sshPublicKeyContent string
|
|
sshPort string
|
|
|
|
sshTempDir string
|
|
remoteUserName string
|
|
}
|
|
|
|
func NewMiniTestDockerDeployerFromConfigFile(path string) MiniTestDeployer {
|
|
config := MiniTestDockerConfig{}
|
|
data, err := os.ReadFile(path)
|
|
if err != nil {
|
|
klog.Fatalf("failed to read config file %s: %v", path, err)
|
|
}
|
|
if err := json.Unmarshal(data, &config); err != nil {
|
|
klog.Fatalf("failed to parse config file %s: %v", path, err)
|
|
}
|
|
return NewMiniTestDockerDeployer(&config)
|
|
|
|
}
|
|
|
|
func NewMiniTestDockerDeployer(config *MiniTestDockerConfig) MiniTestDeployer {
|
|
return &MiniTestDockerDeployer{
|
|
ctx: context.TODO(),
|
|
config: config,
|
|
isUp: false,
|
|
remoteUserName: remoteUserName,
|
|
}
|
|
}
|
|
|
|
func (m *MiniTestDockerDeployer) Up() error {
|
|
dockerClient, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
|
|
if err != nil {
|
|
return fmt.Errorf("failed to create docker api client: %v", err)
|
|
}
|
|
m.dockerClient = dockerClient
|
|
|
|
// pull the image
|
|
reader, err := dockerClient.ImagePull(m.ctx, m.config.Image, client.ImagePullOptions{})
|
|
if err != nil {
|
|
return fmt.Errorf("failed to pull image %s: %v", m.config.Image, err)
|
|
}
|
|
defer reader.Close()
|
|
_, err = io.Copy(os.Stdout, reader)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to read image pull response: %v", err)
|
|
}
|
|
// find a free port for ssh
|
|
port, err := freeport.GetFreePort()
|
|
if err != nil {
|
|
return fmt.Errorf("failed to get a free port for ssh: %v", err)
|
|
}
|
|
m.sshPort = strconv.Itoa(port)
|
|
klog.Infof("Using port %s for ssh", m.sshPort)
|
|
|
|
// start the container
|
|
id := uuid.New().String()[:8]
|
|
//todo: remove hard coded ports
|
|
exposedPorts, portBindings, _ := nat.ParsePortSpecs([]string{m.sshPort + ":22"})
|
|
response, err := dockerClient.ContainerCreate(
|
|
m.ctx,
|
|
&container.Config{
|
|
Image: m.config.Image,
|
|
Tty: true, //-t
|
|
OpenStdin: true, // -i
|
|
//Entrypoint: []string{"/usr/sbin/init"},
|
|
ExposedPorts: exposedPorts,
|
|
},
|
|
|
|
&container.HostConfig{
|
|
Privileged: true,
|
|
PortBindings: portBindings,
|
|
}, nil, nil, "minitest-"+id)
|
|
if err != nil {
|
|
klog.Errorf("failed to create container from image %s: %v", m.config.Image, err)
|
|
return fmt.Errorf("failed to create container from image %s: %v", m.config.Image, err)
|
|
}
|
|
m.containerSHA = response.ID
|
|
|
|
err = dockerClient.ContainerStart(m.ctx, m.containerSHA,
|
|
client.ContainerStartOptions{})
|
|
if err != nil {
|
|
klog.Errorf("failed to start container %s: %v", m.containerSHA, err)
|
|
return fmt.Errorf("failed to start container %s: %v", m.containerSHA, err)
|
|
}
|
|
|
|
// set up ssh keys
|
|
if err := m.sshSetUp(); err != nil {
|
|
klog.Errorf("failed to set up ssh: %v", err)
|
|
return fmt.Errorf("failed to set up ssh: %v", err)
|
|
}
|
|
|
|
// set up sshd server
|
|
err = m.executeDockerShellCommand("root", fmt.Sprintf(sshSetupScript, m.remoteUserName, m.sshPublicKeyContent))
|
|
if err != nil {
|
|
klog.Errorf("failed to set up sshd server in container %s: %v", m.containerSHA, err)
|
|
return fmt.Errorf("failed to set up sshd server in container %s: %v", m.containerSHA, err)
|
|
}
|
|
|
|
// check ssh connectivity
|
|
if err := sshConnectionCheck(m.ctx, m.remoteUserName, "localhost", m.sshAdditionalArgs()); err != nil {
|
|
klog.Errorf("Failed to conntect via ssh: %v", err)
|
|
return fmt.Errorf("Failed to conntect via ssh: %v", err)
|
|
}
|
|
|
|
m.isUp = true
|
|
return nil
|
|
}
|
|
|
|
func (m *MiniTestDockerDeployer) Down() error {
|
|
os.RemoveAll(m.sshTempDir)
|
|
|
|
if m.dockerClient == nil {
|
|
klog.Errorf("m.dockerClient not set")
|
|
return fmt.Errorf("m.dockerClient not set")
|
|
}
|
|
if err := m.dockerClient.ContainerRemove(m.ctx, m.containerSHA, client.ContainerRemoveOptions{Force: true}); err != nil {
|
|
klog.Errorf("failed to remove container %s: %v", m.containerSHA, err)
|
|
return fmt.Errorf("failed to remove container %s: %v", m.containerSHA, err)
|
|
}
|
|
// close the docker client
|
|
m.dockerClient.Close()
|
|
m.isUp = false
|
|
klog.Infof("Successfully removed container %s", m.containerSHA)
|
|
return nil
|
|
}
|
|
func (m *MiniTestDockerDeployer) IsUp() (bool, error) {
|
|
return m.isUp, nil
|
|
}
|
|
|
|
func (m *MiniTestDockerDeployer) Execute(args ...string) error {
|
|
return executeSSHCommand(m.ctx, m.remoteUserName, "localhost", m.sshAdditionalArgs(), args...)
|
|
}
|
|
|
|
func (m *MiniTestDockerDeployer) SyncToRemote(src string, dst string, excludedPattern []string) error {
|
|
excludedArgs := make([]string, 0, len(excludedPattern)*2)
|
|
for _, pattern := range excludedPattern {
|
|
excludedArgs = append(excludedArgs, "--exclude", pattern)
|
|
}
|
|
dstRemote := fmt.Sprintf("%s@%s:%s", m.remoteUserName, "localhost", dst)
|
|
return executeRsyncSSHCommand(m.ctx, m.sshAdditionalArgs(), src, dstRemote, excludedArgs)
|
|
}
|
|
|
|
func (m *MiniTestDockerDeployer) SyncToHost(src string, dst string, excludedPattern []string) error {
|
|
excludedArgs := make([]string, 0, len(excludedPattern)*2)
|
|
for _, pattern := range excludedPattern {
|
|
excludedArgs = append(excludedArgs, "--exclude", pattern)
|
|
}
|
|
srcRemote := fmt.Sprintf("%s@%s:%s", m.remoteUserName, "localhost", src)
|
|
return executeRsyncSSHCommand(m.ctx, m.sshAdditionalArgs(), srcRemote, dst, excludedArgs)
|
|
//return executeScpCommand(m.ctx, m.remoteUserName, "localhost", m.scpAdditionalArgs(), src, dst)
|
|
}
|
|
|
|
func (m *MiniTestDockerDeployer) executeDockerShellCommand(user string, args ...string) error {
|
|
execResp, err := m.dockerClient.ContainerExecCreate(m.ctx, m.containerSHA, container.ExecOptions{
|
|
Cmd: append([]string{"/bin/bash", "-c"}, args...),
|
|
AttachStdout: true,
|
|
AttachStderr: true,
|
|
Tty: true,
|
|
User: user,
|
|
})
|
|
if err != nil {
|
|
return fmt.Errorf("failed to create exec instance: %v", err)
|
|
}
|
|
|
|
attachResp, err := m.dockerClient.ContainerExecAttach(
|
|
m.ctx, execResp.ID,
|
|
container.ExecAttachOptions{Tty: true})
|
|
if err != nil {
|
|
return fmt.Errorf("failed to attach to exec instance: %v", err)
|
|
}
|
|
defer attachResp.Close()
|
|
if _, err = io.Copy(os.Stdout, attachResp.Reader); err != nil {
|
|
return fmt.Errorf("failed to read exec output: %v", err)
|
|
}
|
|
return nil
|
|
|
|
}
|
|
|
|
func (m *MiniTestDockerDeployer) sshSetUp() error {
|
|
// we generate a ssh key pair and add the public key to the container's authorized_keys
|
|
|
|
sshTempDir, err := os.MkdirTemp("", "minitest-ssh-")
|
|
if err != nil {
|
|
return fmt.Errorf("failed to create temp dir for ssh keys: %v", err)
|
|
}
|
|
m.sshTempDir = sshTempDir
|
|
klog.Info("Created temp dir for ssh keys: ", sshTempDir)
|
|
|
|
// create private key file
|
|
sshPrivateKeyFile, err := os.CreateTemp(sshTempDir, "id_rsa")
|
|
if err != nil {
|
|
return fmt.Errorf("failed to create temp file for private key: %v", err)
|
|
}
|
|
if sshPrivateKeyFile.Chmod(0600) != nil {
|
|
return fmt.Errorf("failed to chmod private key file: %v", err)
|
|
}
|
|
m.sshPrivateKeyFile = sshPrivateKeyFile.Name()
|
|
klog.Info("Created temp file for private key: ", m.sshPrivateKeyFile)
|
|
|
|
// create public key file
|
|
sshPublicKeyFile, err := os.CreateTemp(sshTempDir, "id_rsa.pub")
|
|
if err != nil {
|
|
return fmt.Errorf("failed to create temp file for public key: %v", err)
|
|
}
|
|
if sshPublicKeyFile.Chmod(0644) != nil {
|
|
return fmt.Errorf("failed to chmod public key file: %v", err)
|
|
}
|
|
m.sshPublicKeyFile = sshPublicKeyFile.Name()
|
|
klog.Info("Created temp file for public key: ", m.sshPublicKeyFile)
|
|
|
|
// generate private key and convert to PEM format
|
|
privateKey, err := rsa.GenerateKey(rand.Reader, 4096)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to generate private key: %v", err)
|
|
}
|
|
privDER := x509.MarshalPKCS1PrivateKey(privateKey)
|
|
privBlock := &pem.Block{
|
|
Type: "RSA PRIVATE KEY",
|
|
Bytes: privDER,
|
|
}
|
|
|
|
if err := pem.Encode(sshPrivateKeyFile, privBlock); err != nil {
|
|
sshPrivateKeyFile.Close()
|
|
return fmt.Errorf("failed to write private key to file: %v", err)
|
|
}
|
|
|
|
pub, err := gossh.NewPublicKey(&privateKey.PublicKey)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to generate public key: %v", err)
|
|
}
|
|
pubBytes := gossh.MarshalAuthorizedKey(pub)
|
|
|
|
if _, err := sshPublicKeyFile.Write(pubBytes); err != nil {
|
|
return fmt.Errorf("failed to write public key to file: %v", err)
|
|
}
|
|
sshPublicKeyFile.Close()
|
|
m.sshPublicKeyContent = string(pubBytes)
|
|
klog.Infof("Generated ssh public key:%s ", m.sshPublicKeyContent)
|
|
return nil
|
|
}
|
|
|
|
func (m *MiniTestDockerDeployer) sshAdditionalArgs() []string {
|
|
return []string{"-i", m.sshPrivateKeyFile, "-p", m.sshPort}
|
|
}
|
|
|
|
func (m *MiniTestDockerDeployer) scpAdditionalArgs() []string {
|
|
return []string{"-i", m.sshPrivateKeyFile, "-P", m.sshPort}
|
|
}
|