mirror of https://github.com/k3s-io/k3s.git
Merge pull request #73726 from wk8/wk8/gmsa_alpha
Kubelet changes for Windows GMSA supportpull/564/head
Kubernetes Prow Robot
GitHub
commit
44d13d3b77
|
|
@ -411,6 +411,12 @@ const (
|
|||
//
|
||||
// Implement support for limiting pids in nodes
|
||||
SupportNodePidsLimit utilfeature.Feature = "SupportNodePidsLimit"
|
||||
|
||||
// owner: @wk8
|
||||
// alpha: v1.14
|
||||
//
|
||||
// Enables GMSA support for Windows workloads.
|
||||
WindowsGMSA utilfeature.Feature = "WindowsGMSA"
|
||||
)
|
||||
|
||||
func init() {
|
||||
|
|
|
|||
|
|
@ -7,6 +7,8 @@ go_library(
|
|||
"doc.go",
|
||||
"docker_checkpoint.go",
|
||||
"docker_container.go",
|
||||
"docker_container_unsupported.go",
|
||||
"docker_container_windows.go",
|
||||
"docker_image.go",
|
||||
"docker_image_linux.go",
|
||||
"docker_image_unsupported.go",
|
||||
|
|
@ -73,6 +75,7 @@ go_library(
|
|||
"@io_bazel_rules_go//go/platform:windows": [
|
||||
"//pkg/kubelet/apis:go_default_library",
|
||||
"//pkg/kubelet/winstats:go_default_library",
|
||||
"//vendor/golang.org/x/sys/windows/registry:go_default_library",
|
||||
],
|
||||
"//conditions:default": [],
|
||||
}),
|
||||
|
|
@ -84,6 +87,7 @@ go_test(
|
|||
"convert_test.go",
|
||||
"docker_checkpoint_test.go",
|
||||
"docker_container_test.go",
|
||||
"docker_container_windows_test.go",
|
||||
"docker_image_test.go",
|
||||
"docker_sandbox_test.go",
|
||||
"docker_service_test.go",
|
||||
|
|
@ -118,6 +122,9 @@ go_test(
|
|||
"@io_bazel_rules_go//go/platform:linux": [
|
||||
"//staging/src/k8s.io/api/core/v1:go_default_library",
|
||||
],
|
||||
"@io_bazel_rules_go//go/platform:windows": [
|
||||
"//vendor/golang.org/x/sys/windows/registry:go_default_library",
|
||||
],
|
||||
"//conditions:default": [],
|
||||
}),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -114,8 +114,9 @@ func (ds *dockerService) CreateContainer(_ context.Context, r *runtimeapi.Create
|
|||
if iSpec := config.GetImage(); iSpec != nil {
|
||||
image = iSpec.Image
|
||||
}
|
||||
containerName := makeContainerName(sandboxConfig, config)
|
||||
createConfig := dockertypes.ContainerCreateConfig{
|
||||
Name: makeContainerName(sandboxConfig, config),
|
||||
Name: containerName,
|
||||
Config: &dockercontainer.Config{
|
||||
// TODO: set User.
|
||||
Entrypoint: dockerstrslice.StrSlice(config.Command),
|
||||
|
|
@ -162,15 +163,25 @@ func (ds *dockerService) CreateContainer(_ context.Context, r *runtimeapi.Create
|
|||
|
||||
hc.SecurityOpt = append(hc.SecurityOpt, securityOpts...)
|
||||
|
||||
createResp, err := ds.client.CreateContainer(createConfig)
|
||||
cleanupInfo, err := ds.applyPlatformSpecificDockerConfig(r, &createConfig)
|
||||
if err != nil {
|
||||
createResp, err = recoverFromCreationConflictIfNeeded(ds.client, createConfig, err)
|
||||
return nil, err
|
||||
}
|
||||
defer func() {
|
||||
for _, err := range ds.performPlatformSpecificContainerCreationCleanup(cleanupInfo) {
|
||||
klog.Warningf("error when cleaning up after container %v's creation: %v", containerName, err)
|
||||
}
|
||||
}()
|
||||
|
||||
createResp, createErr := ds.client.CreateContainer(createConfig)
|
||||
if createErr != nil {
|
||||
createResp, createErr = recoverFromCreationConflictIfNeeded(ds.client, createConfig, createErr)
|
||||
}
|
||||
|
||||
if createResp != nil {
|
||||
return &runtimeapi.CreateContainerResponse{ContainerId: createResp.ID}, nil
|
||||
}
|
||||
return nil, err
|
||||
return nil, createErr
|
||||
}
|
||||
|
||||
// getContainerLogPath returns the container log path specified by kubelet and the real
|
||||
|
|
|
|||
|
|
@ -0,0 +1,48 @@
|
|||
// +build !windows
|
||||
|
||||
/*
|
||||
Copyright 2019 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package dockershim
|
||||
|
||||
import (
|
||||
dockertypes "github.com/docker/docker/api/types"
|
||||
runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
|
||||
)
|
||||
|
||||
type containerCreationCleanupInfo struct{}
|
||||
|
||||
// applyPlatformSpecificDockerConfig applies platform-specific configurations to a dockertypes.ContainerCreateConfig struct.
|
||||
// The containerCreationCleanupInfo struct it returns will be passed as is to performPlatformSpecificContainerCreationCleanup
|
||||
// after the container has been created.
|
||||
func (ds *dockerService) applyPlatformSpecificDockerConfig(*runtimeapi.CreateContainerRequest, *dockertypes.ContainerCreateConfig) (*containerCreationCleanupInfo, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// performPlatformSpecificContainerCreationCleanup is responsible for doing any platform-specific cleanup
|
||||
// after a container creation. Any errors it returns are simply logged, but do not fail the container
|
||||
// creation.
|
||||
func (ds *dockerService) performPlatformSpecificContainerCreationCleanup(cleanupInfo *containerCreationCleanupInfo) (errors []error) {
|
||||
return
|
||||
}
|
||||
|
||||
// platformSpecificContainerCreationInitCleanup is called when dockershim
|
||||
// is starting, and is meant to clean up any cruft left by previous runs
|
||||
// creating containers.
|
||||
// Errors are simply logged, but don't prevent dockershim from starting.
|
||||
func (ds *dockerService) platformSpecificContainerCreationInitCleanup() (errors []error) {
|
||||
return
|
||||
}
|
||||
|
|
@ -0,0 +1,216 @@
|
|||
// +build windows
|
||||
|
||||
/*
|
||||
Copyright 2019 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package dockershim
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"regexp"
|
||||
|
||||
"golang.org/x/sys/windows/registry"
|
||||
|
||||
dockertypes "github.com/docker/docker/api/types"
|
||||
dockercontainer "github.com/docker/docker/api/types/container"
|
||||
|
||||
runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
|
||||
"k8s.io/kubernetes/pkg/kubelet/kuberuntime"
|
||||
)
|
||||
|
||||
type containerCreationCleanupInfo struct {
|
||||
gMSARegistryValueName string
|
||||
}
|
||||
|
||||
// applyPlatformSpecificDockerConfig applies platform-specific configurations to a dockertypes.ContainerCreateConfig struct.
|
||||
// The containerCreationCleanupInfo struct it returns will be passed as is to performPlatformSpecificContainerCreationCleanup
|
||||
// after the container has been created.
|
||||
func (ds *dockerService) applyPlatformSpecificDockerConfig(request *runtimeapi.CreateContainerRequest, createConfig *dockertypes.ContainerCreateConfig) (*containerCreationCleanupInfo, error) {
|
||||
cleanupInfo := &containerCreationCleanupInfo{}
|
||||
|
||||
if err := applyGMSAConfig(request.GetConfig(), createConfig, cleanupInfo); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return cleanupInfo, nil
|
||||
}
|
||||
|
||||
// applyGMSAConfig looks at the kuberuntime.GMSASpecContainerAnnotationKey container annotation; if present,
|
||||
// it copies its contents to a unique registry value, and sets a SecurityOpt on the config pointing to that registry value.
|
||||
// We use registry values instead of files since their location cannot change - as opposed to credential spec files,
|
||||
// whose location could potentially change down the line, or even be unknown (eg if docker is not installed on the
|
||||
// C: drive)
|
||||
// When docker supports passing a credential spec's contents directly, we should switch to using that
|
||||
// as it will avoid cluttering the registry - there is a moby PR out for this:
|
||||
// https://github.com/moby/moby/pull/38777
|
||||
func applyGMSAConfig(config *runtimeapi.ContainerConfig, createConfig *dockertypes.ContainerCreateConfig, cleanupInfo *containerCreationCleanupInfo) error {
|
||||
credSpec := config.Annotations[kuberuntime.GMSASpecContainerAnnotationKey]
|
||||
if credSpec == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
valueName, err := copyGMSACredSpecToRegistryValue(credSpec)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if createConfig.HostConfig == nil {
|
||||
createConfig.HostConfig = &dockercontainer.HostConfig{}
|
||||
}
|
||||
|
||||
createConfig.HostConfig.SecurityOpt = append(createConfig.HostConfig.SecurityOpt, "credentialspec=registry://"+valueName)
|
||||
cleanupInfo.gMSARegistryValueName = valueName
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
const (
|
||||
// same as https://github.com/moby/moby/blob/93d994e29c9cc8d81f1b0477e28d705fa7e2cd72/daemon/oci_windows.go#L23
|
||||
credentialSpecRegistryLocation = `SOFTWARE\Microsoft\Windows NT\CurrentVersion\Virtualization\Containers\CredentialSpecs`
|
||||
// the prefix for the registry values we write GMSA cred specs to
|
||||
gMSARegistryValueNamePrefix = "k8s-cred-spec-"
|
||||
// the number of random bytes to generate suffixes for registry value names
|
||||
gMSARegistryValueNameSuffixRandomBytes = 40
|
||||
)
|
||||
|
||||
// registryKey is an interface wrapper around `registry.Key`,
|
||||
// listing only the methods we care about here.
|
||||
// It's mainly useful to easily allow mocking the registry in tests.
|
||||
type registryKey interface {
|
||||
SetStringValue(name, value string) error
|
||||
DeleteValue(name string) error
|
||||
ReadValueNames(n int) ([]string, error)
|
||||
Close() error
|
||||
}
|
||||
|
||||
var registryCreateKeyFunc = func(baseKey registry.Key, path string, access uint32) (registryKey, bool, error) {
|
||||
return registry.CreateKey(baseKey, path, access)
|
||||
}
|
||||
|
||||
// randomReader is only meant to ever be overridden for testing purposes,
|
||||
// same idea as for `registryKey` above
|
||||
var randomReader = rand.Reader
|
||||
|
||||
// gMSARegistryValueNamesRegex is the regex used to detect gMSA cred spec
|
||||
// registry values in `removeAllGMSARegistryValues` below.
|
||||
var gMSARegistryValueNamesRegex = regexp.MustCompile(fmt.Sprintf("^%s[0-9a-f]{%d}$", gMSARegistryValueNamePrefix, 2*gMSARegistryValueNameSuffixRandomBytes))
|
||||
|
||||
// copyGMSACredSpecToRegistryKey copies the credential specs to a unique registry value, and returns its name.
|
||||
func copyGMSACredSpecToRegistryValue(credSpec string) (string, error) {
|
||||
valueName, err := gMSARegistryValueName()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
// write to the registry
|
||||
key, _, err := registryCreateKeyFunc(registry.LOCAL_MACHINE, credentialSpecRegistryLocation, registry.SET_VALUE)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("unable to open registry key %q: %v", credentialSpecRegistryLocation, err)
|
||||
}
|
||||
defer key.Close()
|
||||
if err = key.SetStringValue(valueName, credSpec); err != nil {
|
||||
return "", fmt.Errorf("unable to write into registry value %q/%q: %v", credentialSpecRegistryLocation, valueName, err)
|
||||
}
|
||||
|
||||
return valueName, nil
|
||||
}
|
||||
|
||||
// gMSARegistryValueName computes the name of the registry value where to store the GMSA cred spec contents.
|
||||
// The value's name is a purely random suffix appended to `gMSARegistryValueNamePrefix`.
|
||||
func gMSARegistryValueName() (string, error) {
|
||||
randomSuffix, err := randomString(gMSARegistryValueNameSuffixRandomBytes)
|
||||
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("error when generating gMSA registry value name: %v", err)
|
||||
}
|
||||
|
||||
return gMSARegistryValueNamePrefix + randomSuffix, nil
|
||||
}
|
||||
|
||||
// randomString returns a random hex string.
|
||||
func randomString(length int) (string, error) {
|
||||
randBytes := make([]byte, length)
|
||||
|
||||
if n, err := randomReader.Read(randBytes); err != nil || n != length {
|
||||
if err == nil {
|
||||
err = fmt.Errorf("only got %v random bytes, expected %v", n, length)
|
||||
}
|
||||
return "", fmt.Errorf("unable to generate random string: %v", err)
|
||||
}
|
||||
|
||||
return hex.EncodeToString(randBytes), nil
|
||||
}
|
||||
|
||||
// performPlatformSpecificContainerCreationCleanup is responsible for doing any platform-specific cleanup
|
||||
// after a container creation. Any errors it returns are simply logged, but do not fail the container
|
||||
// creation.
|
||||
func (ds *dockerService) performPlatformSpecificContainerCreationCleanup(cleanupInfo *containerCreationCleanupInfo) (errors []error) {
|
||||
if err := removeGMSARegistryValue(cleanupInfo); err != nil {
|
||||
errors = append(errors, err)
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func removeGMSARegistryValue(cleanupInfo *containerCreationCleanupInfo) error {
|
||||
if cleanupInfo == nil || cleanupInfo.gMSARegistryValueName == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
key, _, err := registryCreateKeyFunc(registry.LOCAL_MACHINE, credentialSpecRegistryLocation, registry.SET_VALUE)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to open registry key %q: %v", credentialSpecRegistryLocation, err)
|
||||
}
|
||||
defer key.Close()
|
||||
if err = key.DeleteValue(cleanupInfo.gMSARegistryValueName); err != nil {
|
||||
return fmt.Errorf("unable to remove registry value %q/%q: %v", credentialSpecRegistryLocation, cleanupInfo.gMSARegistryValueName, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// platformSpecificContainerCreationInitCleanup is called when dockershim
|
||||
// is starting, and is meant to clean up any cruft left by previous runs
|
||||
// creating containers.
|
||||
// Errors are simply logged, but don't prevent dockershim from starting.
|
||||
func (ds *dockerService) platformSpecificContainerCreationInitCleanup() (errors []error) {
|
||||
return removeAllGMSARegistryValues()
|
||||
}
|
||||
|
||||
func removeAllGMSARegistryValues() (errors []error) {
|
||||
key, _, err := registryCreateKeyFunc(registry.LOCAL_MACHINE, credentialSpecRegistryLocation, registry.SET_VALUE)
|
||||
if err != nil {
|
||||
return []error{fmt.Errorf("unable to open registry key %q: %v", credentialSpecRegistryLocation, err)}
|
||||
}
|
||||
defer key.Close()
|
||||
|
||||
valueNames, err := key.ReadValueNames(0)
|
||||
if err != nil {
|
||||
return []error{fmt.Errorf("unable to list values under registry key %q: %v", credentialSpecRegistryLocation, err)}
|
||||
}
|
||||
|
||||
for _, valueName := range valueNames {
|
||||
if gMSARegistryValueNamesRegex.MatchString(valueName) {
|
||||
if err = key.DeleteValue(valueName); err != nil {
|
||||
errors = append(errors, fmt.Errorf("unable to remove registry value %q/%q: %v", credentialSpecRegistryLocation, valueName, err))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
|
@ -0,0 +1,304 @@
|
|||
/*
|
||||
Copyright 2019 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package dockershim
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"testing"
|
||||
|
||||
dockertypes "github.com/docker/docker/api/types"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"golang.org/x/sys/windows/registry"
|
||||
runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
|
||||
)
|
||||
|
||||
type dummyRegistryKey struct {
|
||||
setStringValueError error
|
||||
setStringValueArgs [][]string
|
||||
|
||||
deleteValueFunc func(name string) error
|
||||
deleteValueArgs []string
|
||||
|
||||
readValueNamesError error
|
||||
readValueNamesReturn []string
|
||||
readValueNamesArgs []int
|
||||
|
||||
closed bool
|
||||
}
|
||||
|
||||
func (k *dummyRegistryKey) SetStringValue(name, value string) error {
|
||||
k.setStringValueArgs = append(k.setStringValueArgs, []string{name, value})
|
||||
return k.setStringValueError
|
||||
}
|
||||
|
||||
func (k *dummyRegistryKey) DeleteValue(name string) error {
|
||||
k.deleteValueArgs = append(k.deleteValueArgs, name)
|
||||
if k.deleteValueFunc == nil {
|
||||
return nil
|
||||
}
|
||||
return k.deleteValueFunc(name)
|
||||
}
|
||||
|
||||
func (k *dummyRegistryKey) ReadValueNames(n int) ([]string, error) {
|
||||
k.readValueNamesArgs = append(k.readValueNamesArgs, n)
|
||||
return k.readValueNamesReturn, k.readValueNamesError
|
||||
}
|
||||
|
||||
func (k *dummyRegistryKey) Close() error {
|
||||
k.closed = true
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestApplyGMSAConfig(t *testing.T) {
|
||||
dummyCredSpec := "test cred spec contents"
|
||||
randomBytes := []byte{0x19, 0x0, 0x25, 0x45, 0x18, 0x52, 0x9e, 0x2a, 0x3d, 0xed, 0xb8, 0x5c, 0xde, 0xc0, 0x3c, 0xe2, 0x70, 0x55, 0x96, 0x47, 0x45, 0x9a, 0xb5, 0x31, 0xf0, 0x7a, 0xf5, 0xeb, 0x1c, 0x54, 0x95, 0xfd, 0xa7, 0x9, 0x43, 0x5c, 0xe8, 0x2a, 0xb8, 0x9c}
|
||||
expectedHex := "1900254518529e2a3dedb85cdec03ce270559647459ab531f07af5eb1c5495fda709435ce82ab89c"
|
||||
expectedValueName := "k8s-cred-spec-" + expectedHex
|
||||
|
||||
containerConfigWithGMSAAnnotation := &runtimeapi.ContainerConfig{
|
||||
Annotations: map[string]string{"container.alpha.windows.kubernetes.io/gmsa-credential-spec": dummyCredSpec},
|
||||
}
|
||||
|
||||
t.Run("happy path", func(t *testing.T) {
|
||||
key := &dummyRegistryKey{}
|
||||
defer setRegistryCreateKeyFunc(t, key)()
|
||||
defer setRandomReader(randomBytes)()
|
||||
|
||||
createConfig := &dockertypes.ContainerCreateConfig{}
|
||||
cleanupInfo := &containerCreationCleanupInfo{}
|
||||
err := applyGMSAConfig(containerConfigWithGMSAAnnotation, createConfig, cleanupInfo)
|
||||
|
||||
assert.Nil(t, err)
|
||||
|
||||
// the registry key should have been properly created
|
||||
if assert.Equal(t, 1, len(key.setStringValueArgs)) {
|
||||
assert.Equal(t, []string{expectedValueName, dummyCredSpec}, key.setStringValueArgs[0])
|
||||
}
|
||||
assert.True(t, key.closed)
|
||||
|
||||
// the create config's security opt should have been populated
|
||||
if assert.NotNil(t, createConfig.HostConfig) {
|
||||
assert.Equal(t, createConfig.HostConfig.SecurityOpt, []string{"credentialspec=registry://" + expectedValueName})
|
||||
}
|
||||
|
||||
// and the name of that value should have been saved to the cleanup info
|
||||
assert.Equal(t, expectedValueName, cleanupInfo.gMSARegistryValueName)
|
||||
})
|
||||
t.Run("happy path with a truly random string", func(t *testing.T) {
|
||||
defer setRegistryCreateKeyFunc(t, &dummyRegistryKey{})()
|
||||
|
||||
createConfig := &dockertypes.ContainerCreateConfig{}
|
||||
cleanupInfo := &containerCreationCleanupInfo{}
|
||||
err := applyGMSAConfig(containerConfigWithGMSAAnnotation, createConfig, cleanupInfo)
|
||||
|
||||
assert.Nil(t, err)
|
||||
|
||||
if assert.NotNil(t, createConfig.HostConfig) && assert.Equal(t, 1, len(createConfig.HostConfig.SecurityOpt)) {
|
||||
secOpt := createConfig.HostConfig.SecurityOpt[0]
|
||||
|
||||
expectedPrefix := "credentialspec=registry://k8s-cred-spec-"
|
||||
assert.Equal(t, expectedPrefix, secOpt[:len(expectedPrefix)])
|
||||
|
||||
hex := secOpt[len(expectedPrefix):]
|
||||
hexRegex := regexp.MustCompile("^[0-9a-f]{80}$")
|
||||
assert.True(t, hexRegex.MatchString(hex))
|
||||
assert.NotEqual(t, expectedHex, hex)
|
||||
|
||||
assert.Equal(t, "k8s-cred-spec-"+hex, cleanupInfo.gMSARegistryValueName)
|
||||
}
|
||||
})
|
||||
t.Run("when there's an error generating the random value name", func(t *testing.T) {
|
||||
defer setRandomReader([]byte{})()
|
||||
|
||||
err := applyGMSAConfig(containerConfigWithGMSAAnnotation, &dockertypes.ContainerCreateConfig{}, &containerCreationCleanupInfo{})
|
||||
|
||||
require.NotNil(t, err)
|
||||
assert.Contains(t, err.Error(), "error when generating gMSA registry value name: unable to generate random string")
|
||||
})
|
||||
t.Run("if there's an error opening the registry key", func(t *testing.T) {
|
||||
defer setRegistryCreateKeyFunc(t, &dummyRegistryKey{}, fmt.Errorf("dummy error"))()
|
||||
|
||||
err := applyGMSAConfig(containerConfigWithGMSAAnnotation, &dockertypes.ContainerCreateConfig{}, &containerCreationCleanupInfo{})
|
||||
|
||||
require.NotNil(t, err)
|
||||
assert.Contains(t, err.Error(), "unable to open registry key")
|
||||
})
|
||||
t.Run("if there's an error writing to the registry key", func(t *testing.T) {
|
||||
key := &dummyRegistryKey{}
|
||||
key.setStringValueError = fmt.Errorf("dummy error")
|
||||
defer setRegistryCreateKeyFunc(t, key)()
|
||||
|
||||
err := applyGMSAConfig(containerConfigWithGMSAAnnotation, &dockertypes.ContainerCreateConfig{}, &containerCreationCleanupInfo{})
|
||||
|
||||
if assert.NotNil(t, err) {
|
||||
assert.Contains(t, err.Error(), "unable to write into registry value")
|
||||
}
|
||||
assert.True(t, key.closed)
|
||||
})
|
||||
t.Run("if there is no GMSA annotation", func(t *testing.T) {
|
||||
createConfig := &dockertypes.ContainerCreateConfig{}
|
||||
|
||||
err := applyGMSAConfig(&runtimeapi.ContainerConfig{}, createConfig, &containerCreationCleanupInfo{})
|
||||
|
||||
assert.Nil(t, err)
|
||||
assert.Nil(t, createConfig.HostConfig)
|
||||
})
|
||||
}
|
||||
|
||||
func TestRemoveGMSARegistryValue(t *testing.T) {
|
||||
valueName := "k8s-cred-spec-1900254518529e2a3dedb85cdec03ce270559647459ab531f07af5eb1c5495fda709435ce82ab89c"
|
||||
cleanupInfoWithValue := &containerCreationCleanupInfo{gMSARegistryValueName: valueName}
|
||||
|
||||
t.Run("it does remove the registry value", func(t *testing.T) {
|
||||
key := &dummyRegistryKey{}
|
||||
defer setRegistryCreateKeyFunc(t, key)()
|
||||
|
||||
err := removeGMSARegistryValue(cleanupInfoWithValue)
|
||||
|
||||
assert.Nil(t, err)
|
||||
|
||||
// the registry key should have been properly deleted
|
||||
if assert.Equal(t, 1, len(key.deleteValueArgs)) {
|
||||
assert.Equal(t, []string{valueName}, key.deleteValueArgs)
|
||||
}
|
||||
assert.True(t, key.closed)
|
||||
})
|
||||
t.Run("if there's an error opening the registry key", func(t *testing.T) {
|
||||
defer setRegistryCreateKeyFunc(t, &dummyRegistryKey{}, fmt.Errorf("dummy error"))()
|
||||
|
||||
err := removeGMSARegistryValue(cleanupInfoWithValue)
|
||||
|
||||
require.NotNil(t, err)
|
||||
assert.Contains(t, err.Error(), "unable to open registry key")
|
||||
})
|
||||
t.Run("if there's an error deleting from the registry key", func(t *testing.T) {
|
||||
key := &dummyRegistryKey{}
|
||||
key.deleteValueFunc = func(name string) error { return fmt.Errorf("dummy error") }
|
||||
defer setRegistryCreateKeyFunc(t, key)()
|
||||
|
||||
err := removeGMSARegistryValue(cleanupInfoWithValue)
|
||||
|
||||
if assert.NotNil(t, err) {
|
||||
assert.Contains(t, err.Error(), "unable to remove registry value")
|
||||
}
|
||||
assert.True(t, key.closed)
|
||||
})
|
||||
t.Run("if there's no registry value to be removed, it does nothing", func(t *testing.T) {
|
||||
key := &dummyRegistryKey{}
|
||||
defer setRegistryCreateKeyFunc(t, key)()
|
||||
|
||||
err := removeGMSARegistryValue(&containerCreationCleanupInfo{})
|
||||
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, 0, len(key.deleteValueArgs))
|
||||
})
|
||||
}
|
||||
|
||||
func TestRemoveAllGMSARegistryValues(t *testing.T) {
|
||||
cred1 := "k8s-cred-spec-1900254518529e2a3dedb85cdec03ce270559647459ab531f07af5eb1c5495fda709435ce82ab89c"
|
||||
cred2 := "k8s-cred-spec-8891436007c795a904fdf77b5348e94305e4c48c5f01c47e7f65e980dc7edda85f112715891d65fd"
|
||||
cred3 := "k8s-cred-spec-2f11f1c9e4f8182fe13caa708bd42b2098c8eefc489d6cc98806c058ccbe4cb3703b9ade61ce59a1"
|
||||
cred4 := "k8s-cred-spec-dc532f189598a8220a1e538f79081eee979f94fbdbf8d37e36959485dee57157c03742d691e1fae2"
|
||||
|
||||
t.Run("it removes the keys matching the k8s creds pattern", func(t *testing.T) {
|
||||
key := &dummyRegistryKey{readValueNamesReturn: []string{cred1, "other_creds", cred2}}
|
||||
defer setRegistryCreateKeyFunc(t, key)()
|
||||
|
||||
errors := removeAllGMSARegistryValues()
|
||||
|
||||
assert.Equal(t, 0, len(errors))
|
||||
assert.Equal(t, []string{cred1, cred2}, key.deleteValueArgs)
|
||||
assert.Equal(t, []int{0}, key.readValueNamesArgs)
|
||||
assert.True(t, key.closed)
|
||||
})
|
||||
t.Run("it ignores errors and does a best effort at removing all k8s creds", func(t *testing.T) {
|
||||
key := &dummyRegistryKey{
|
||||
readValueNamesReturn: []string{cred1, cred2, cred3, cred4},
|
||||
deleteValueFunc: func(name string) error {
|
||||
if name == cred1 || name == cred3 {
|
||||
return fmt.Errorf("dummy error")
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
defer setRegistryCreateKeyFunc(t, key)()
|
||||
|
||||
errors := removeAllGMSARegistryValues()
|
||||
|
||||
assert.Equal(t, 2, len(errors))
|
||||
for _, err := range errors {
|
||||
assert.Contains(t, err.Error(), "unable to remove registry value")
|
||||
}
|
||||
assert.Equal(t, []string{cred1, cred2, cred3, cred4}, key.deleteValueArgs)
|
||||
assert.Equal(t, []int{0}, key.readValueNamesArgs)
|
||||
assert.True(t, key.closed)
|
||||
})
|
||||
t.Run("if there's an error opening the registry key", func(t *testing.T) {
|
||||
defer setRegistryCreateKeyFunc(t, &dummyRegistryKey{}, fmt.Errorf("dummy error"))()
|
||||
|
||||
errors := removeAllGMSARegistryValues()
|
||||
|
||||
require.Equal(t, 1, len(errors))
|
||||
assert.Contains(t, errors[0].Error(), "unable to open registry key")
|
||||
})
|
||||
t.Run("if it's unable to list the registry values", func(t *testing.T) {
|
||||
key := &dummyRegistryKey{readValueNamesError: fmt.Errorf("dummy error")}
|
||||
defer setRegistryCreateKeyFunc(t, key)()
|
||||
|
||||
errors := removeAllGMSARegistryValues()
|
||||
|
||||
if assert.Equal(t, 1, len(errors)) {
|
||||
assert.Contains(t, errors[0].Error(), "unable to list values under registry key")
|
||||
}
|
||||
assert.True(t, key.closed)
|
||||
})
|
||||
}
|
||||
|
||||
// setRegistryCreateKeyFunc replaces the registryCreateKeyFunc package variable, and returns a function
|
||||
// to be called to revert the change when done with testing.
|
||||
func setRegistryCreateKeyFunc(t *testing.T, key *dummyRegistryKey, err ...error) func() {
|
||||
previousRegistryCreateKeyFunc := registryCreateKeyFunc
|
||||
|
||||
registryCreateKeyFunc = func(baseKey registry.Key, path string, access uint32) (registryKey, bool, error) {
|
||||
// this should always be called with exactly the same arguments
|
||||
assert.Equal(t, registry.LOCAL_MACHINE, baseKey)
|
||||
assert.Equal(t, credentialSpecRegistryLocation, path)
|
||||
assert.Equal(t, uint32(registry.SET_VALUE), access)
|
||||
|
||||
if len(err) > 0 {
|
||||
return nil, false, err[0]
|
||||
}
|
||||
return key, false, nil
|
||||
}
|
||||
|
||||
return func() {
|
||||
registryCreateKeyFunc = previousRegistryCreateKeyFunc
|
||||
}
|
||||
}
|
||||
|
||||
// setRandomReader replaces the randomReader package variable with a dummy reader that returns the provided
|
||||
// byte slice, and returns a function to be called to revert the change when done with testing.
|
||||
func setRandomReader(b []byte) func() {
|
||||
previousRandomReader := randomReader
|
||||
randomReader = bytes.NewReader(b)
|
||||
return func() {
|
||||
randomReader = previousRandomReader
|
||||
}
|
||||
}
|
||||
|
|
@ -395,6 +395,8 @@ func (ds *dockerService) GetPodPortMappings(podSandboxID string) ([]*hostport.Po
|
|||
|
||||
// Start initializes and starts components in dockerService.
|
||||
func (ds *dockerService) Start() error {
|
||||
ds.initCleanup()
|
||||
|
||||
// Initialize the legacy cleanup flag.
|
||||
if ds.startLocalStreamingServer {
|
||||
go func() {
|
||||
|
|
@ -406,6 +408,16 @@ func (ds *dockerService) Start() error {
|
|||
return ds.containerManager.Start()
|
||||
}
|
||||
|
||||
// initCleanup is responsible for cleaning up any crufts left by previous
|
||||
// runs. If there are any errros, it simply logs them.
|
||||
func (ds *dockerService) initCleanup() {
|
||||
errors := ds.platformSpecificContainerCreationInitCleanup()
|
||||
|
||||
for _, err := range errors {
|
||||
klog.Warningf("initialization error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Status returns the status of the runtime.
|
||||
func (ds *dockerService) Status(_ context.Context, r *runtimeapi.StatusRequest) (*runtimeapi.StatusResponse, error) {
|
||||
runtimeReady := &runtimeapi.RuntimeCondition{
|
||||
|
|
|
|||
|
|
@ -89,6 +89,7 @@ go_test(
|
|||
"instrumented_services_test.go",
|
||||
"kuberuntime_container_linux_test.go",
|
||||
"kuberuntime_container_test.go",
|
||||
"kuberuntime_container_windows_test.go",
|
||||
"kuberuntime_gc_test.go",
|
||||
"kuberuntime_image_test.go",
|
||||
"kuberuntime_manager_test.go",
|
||||
|
|
|
|||
|
|
@ -23,6 +23,8 @@ import (
|
|||
"github.com/docker/docker/pkg/sysinfo"
|
||||
|
||||
"k8s.io/api/core/v1"
|
||||
utilfeature "k8s.io/apiserver/pkg/util/feature"
|
||||
kubefeatures "k8s.io/kubernetes/pkg/features"
|
||||
kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis"
|
||||
runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
|
||||
"k8s.io/kubernetes/pkg/securitycontext"
|
||||
|
|
@ -35,6 +37,10 @@ func (m *kubeGenericRuntimeManager) applyPlatformSpecificContainerConfig(config
|
|||
return err
|
||||
}
|
||||
|
||||
if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.WindowsGMSA) {
|
||||
determineEffectiveSecurityContext(config, container, pod)
|
||||
}
|
||||
|
||||
config.Windows = windowsConfig
|
||||
return nil
|
||||
}
|
||||
|
|
@ -97,3 +103,40 @@ func (m *kubeGenericRuntimeManager) generateWindowsContainerConfig(container *v1
|
|||
|
||||
return wc, nil
|
||||
}
|
||||
|
||||
const (
|
||||
// GMSASpecContainerAnnotationKey is the container annotation where we store the contents of the GMSA credential spec to use.
|
||||
GMSASpecContainerAnnotationKey = "container.alpha.windows.kubernetes.io/gmsa-credential-spec"
|
||||
// gMSAContainerSpecPodAnnotationKeySuffix is the suffix of the pod annotation where the GMSA webhook admission controller
|
||||
// stores the contents of the GMSA credential spec for a given container (the full annotation being the container's name
|
||||
// with this suffix appended).
|
||||
gMSAContainerSpecPodAnnotationKeySuffix = "." + GMSASpecContainerAnnotationKey
|
||||
// gMSAPodSpecPodAnnotationKey is the pod annotation where the GMSA webhook admission controller stores the contents of the GMSA
|
||||
// credential spec to use for containers that do not have their own specific GMSA cred spec set via a
|
||||
// gMSAContainerSpecPodAnnotationKeySuffix annotation as explained above
|
||||
gMSAPodSpecPodAnnotationKey = "pod.alpha.windows.kubernetes.io/gmsa-credential-spec"
|
||||
)
|
||||
|
||||
// determineEffectiveSecurityContext determines the effective GMSA credential spec and, if any, copies it to the container's
|
||||
// GMSASpecContainerAnnotationKey annotation.
|
||||
func determineEffectiveSecurityContext(config *runtimeapi.ContainerConfig, container *v1.Container, pod *v1.Pod) {
|
||||
var containerCredSpec string
|
||||
|
||||
containerGMSAPodAnnotation := container.Name + gMSAContainerSpecPodAnnotationKeySuffix
|
||||
if pod.Annotations[containerGMSAPodAnnotation] != "" {
|
||||
containerCredSpec = pod.Annotations[containerGMSAPodAnnotation]
|
||||
} else if pod.Annotations[gMSAPodSpecPodAnnotationKey] != "" {
|
||||
containerCredSpec = pod.Annotations[gMSAPodSpecPodAnnotationKey]
|
||||
}
|
||||
|
||||
if containerCredSpec != "" {
|
||||
if config.Annotations == nil {
|
||||
config.Annotations = make(map[string]string)
|
||||
}
|
||||
config.Annotations[GMSASpecContainerAnnotationKey] = containerCredSpec
|
||||
} else {
|
||||
// the annotation shouldn't be present, but let's err on the side of caution:
|
||||
// it should only be set here and nowhere else
|
||||
delete(config.Annotations, GMSASpecContainerAnnotationKey)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,82 @@
|
|||
/*
|
||||
Copyright 2019 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package kuberuntime
|
||||
|
||||
import (
|
||||
"github.com/stretchr/testify/assert"
|
||||
"testing"
|
||||
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
|
||||
)
|
||||
|
||||
func TestDetermineEffectiveSecurityContext(t *testing.T) {
|
||||
containerName := "container_name"
|
||||
container := &corev1.Container{Name: containerName}
|
||||
dummyCredSpec := "test cred spec contents"
|
||||
|
||||
buildPod := func(annotations map[string]string) *corev1.Pod {
|
||||
return &corev1.Pod{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Annotations: annotations,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
t.Run("when there's a specific GMSA for that container, and no pod-wide GMSA", func(t *testing.T) {
|
||||
containerConfig := &runtimeapi.ContainerConfig{}
|
||||
|
||||
pod := buildPod(map[string]string{
|
||||
"container_name.container.alpha.windows.kubernetes.io/gmsa-credential-spec": dummyCredSpec,
|
||||
})
|
||||
|
||||
determineEffectiveSecurityContext(containerConfig, container, pod)
|
||||
|
||||
assert.Equal(t, dummyCredSpec, containerConfig.Annotations["container.alpha.windows.kubernetes.io/gmsa-credential-spec"])
|
||||
})
|
||||
t.Run("when there's a specific GMSA for that container, and a pod-wide GMSA", func(t *testing.T) {
|
||||
containerConfig := &runtimeapi.ContainerConfig{}
|
||||
|
||||
pod := buildPod(map[string]string{
|
||||
"container_name.container.alpha.windows.kubernetes.io/gmsa-credential-spec": dummyCredSpec,
|
||||
"pod.alpha.windows.kubernetes.io/gmsa-credential-spec": "should be ignored",
|
||||
})
|
||||
|
||||
determineEffectiveSecurityContext(containerConfig, container, pod)
|
||||
|
||||
assert.Equal(t, dummyCredSpec, containerConfig.Annotations["container.alpha.windows.kubernetes.io/gmsa-credential-spec"])
|
||||
})
|
||||
t.Run("when there's no specific GMSA for that container, and a pod-wide GMSA", func(t *testing.T) {
|
||||
containerConfig := &runtimeapi.ContainerConfig{}
|
||||
|
||||
pod := buildPod(map[string]string{
|
||||
"pod.alpha.windows.kubernetes.io/gmsa-credential-spec": dummyCredSpec,
|
||||
})
|
||||
|
||||
determineEffectiveSecurityContext(containerConfig, container, pod)
|
||||
|
||||
assert.Equal(t, dummyCredSpec, containerConfig.Annotations["container.alpha.windows.kubernetes.io/gmsa-credential-spec"])
|
||||
})
|
||||
t.Run("when there's no specific GMSA for that container, and no pod-wide GMSA", func(t *testing.T) {
|
||||
containerConfig := &runtimeapi.ContainerConfig{}
|
||||
|
||||
determineEffectiveSecurityContext(containerConfig, container, &corev1.Pod{})
|
||||
|
||||
assert.Nil(t, containerConfig.Annotations)
|
||||
})
|
||||
}
|
||||
Loading…
Reference in New Issue