k3s-io
/
k3s
mirror of https://github.com/k3s-io/k3s.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Delete nsenter and containerized flag

k3s-v1.14.0
Darren Shepherd 2018-10-08 00:38:07 -07:00
parent e68c8ca61c
commit f39734f827
10 changed files with 0 additions and 1404 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
cmd/kubelet/app/options/options.go
View File

@ -132,8 +132,6 @@ type KubeletFlags struct {
// Whitelist of unsafe sysctls or sysctl patterns (ending in *).
// +optional
AllowedUnsafeSysctls []string
// containerized should be set to true if kubelet is running in a container.
Containerized bool
// remoteRuntimeEndpoint is the endpoint of remote runtime service
RemoteRuntimeEndpoint string
// remoteImageEndpoint is the endpoint of remote image service
@ -425,7 +423,6 @@ func (f *KubeletFlags) AddFlags(mainfs *pflag.FlagSet) {
fs.Int32Var(&f.NodeStatusMaxImages, "node-status-max-images", f.NodeStatusMaxImages, "<Warning: Alpha feature> The maximum number of images to report in Node.Status.Images. If -1 is specified, no cap will be applied.")
// DEPRECATED FLAGS
fs.BoolVar(&f.Containerized, "containerized", f.Containerized, "Running kubelet in a container.")
fs.MarkDeprecated("containerized", "This feature will be removed in a later release.")
fs.StringVar(&f.BootstrapKubeconfig, "experimental-bootstrap-kubeconfig", f.BootstrapKubeconfig, "")
fs.MarkDeprecated("experimental-bootstrap-kubeconfig", "Use --bootstrap-kubeconfig")

16
cmd/kubelet/app/server.go
View File

@ -94,7 +94,6 @@ import (
"k8s.io/kubernetes/pkg/version/verflag"
"k8s.io/kubernetes/pkg/volume/util/subpath"
"k8s.io/utils/exec"
"k8s.io/utils/nsenter"
)
const (
@ -363,21 +362,6 @@ func UnsecuredDependencies(s *options.KubeletServer) (*kubelet.Dependencies, err
mounter := mount.New(s.ExperimentalMounterPath)
subpather := subpath.New(mounter)
var pluginRunner = exec.New()
if s.Containerized {
klog.V(2).Info("Running kubelet in containerized mode")
ne, err := nsenter.NewNsenter(nsenter.DefaultHostRootFsPath, exec.New())
if err != nil {
return nil, err
}
mounter = mount.NewNsenterMounter(s.RootDirectory, ne)
// NSenter only valid on Linux
subpather = subpath.NewNSEnter(mounter, ne, s.RootDirectory)
// an exec interface which can use nsenter for flex plugin calls
pluginRunner, err = nsenter.NewNsenter(nsenter.DefaultHostRootFsPath, exec.New())
if err != nil {
return nil, err
}
}
var dockerClientConfig *dockershim.ClientConfig
if s.ContainerRuntime == kubetypes.DockerContainerRuntime {

333
pkg/util/mount/nsenter_mount.go
View File

@ -1,333 +0,0 @@
// +build linux
/*
Copyright 2014 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package mount
import (
"fmt"
"os"
"path/filepath"
"strings"
"k8s.io/klog"
"k8s.io/utils/nsenter"
utilpath "k8s.io/utils/path"
)
const (
// hostProcMountsPath is the default mount path for rootfs
hostProcMountsPath = "/rootfs/proc/1/mounts"
// hostProcMountinfoPath is the default mount info path for rootfs
hostProcMountinfoPath = "/rootfs/proc/1/mountinfo"
)
// Currently, all docker containers receive their own mount namespaces.
// NsenterMounter works by executing nsenter to run commands in
// the host's mount namespace.
type NsenterMounter struct {
ne *nsenter.Nsenter
// rootDir is location of /var/lib/kubelet directory.
rootDir string
}
// NewNsenterMounter creates a new mounter for kubelet that runs as a container.
func NewNsenterMounter(rootDir string, ne *nsenter.Nsenter) *NsenterMounter {
return &NsenterMounter{
rootDir: rootDir,
ne: ne,
}
}
// NsenterMounter implements mount.Interface
var _ = Interface(&NsenterMounter{})
// Mount runs mount(8) in the host's root mount namespace. Aside from this
// aspect, Mount has the same semantics as the mounter returned by mount.New()
func (n *NsenterMounter) Mount(source string, target string, fstype string, options []string) error {
bind, bindOpts, bindRemountOpts := isBind(options)
if bind {
err := n.doNsenterMount(source, target, fstype, bindOpts)
if err != nil {
return err
}
return n.doNsenterMount(source, target, fstype, bindRemountOpts)
}
return n.doNsenterMount(source, target, fstype, options)
}
// doNsenterMount nsenters the host's mount namespace and performs the
// requested mount.
func (n *NsenterMounter) doNsenterMount(source, target, fstype string, options []string) error {
klog.V(5).Infof("nsenter mount %s %s %s %v", source, target, fstype, options)
cmd, args := n.makeNsenterArgs(source, target, fstype, options)
outputBytes, err := n.ne.Exec(cmd, args).CombinedOutput()
if len(outputBytes) != 0 {
klog.V(5).Infof("Output of mounting %s to %s: %v", source, target, string(outputBytes))
}
return err
}
// makeNsenterArgs makes a list of argument to nsenter in order to do the
// requested mount.
func (n *NsenterMounter) makeNsenterArgs(source, target, fstype string, options []string) (string, []string) {
mountCmd := n.ne.AbsHostPath("mount")
mountArgs := makeMountArgs(source, target, fstype, options)
if systemdRunPath, hasSystemd := n.ne.SupportsSystemd(); hasSystemd {
// Complete command line:
// nsenter --mount=/rootfs/proc/1/ns/mnt -- /bin/systemd-run --description=... --scope -- /bin/mount -t <type> <what> <where>
// Expected flow is:
// * nsenter breaks out of container's mount namespace and executes
// host's systemd-run.
// * systemd-run creates a transient scope (=~ cgroup) and executes its
// argument (/bin/mount) there.
// * mount does its job, forks a fuse daemon if necessary and finishes.
// (systemd-run --scope finishes at this point, returning mount's exit
// code and stdout/stderr - thats one of --scope benefits).
// * systemd keeps the fuse daemon running in the scope (i.e. in its own
// cgroup) until the fuse daemon dies (another --scope benefit).
// Kubelet container can be restarted and the fuse daemon survives.
// * When the daemon dies (e.g. during unmount) systemd removes the
// scope automatically.
mountCmd, mountArgs = addSystemdScope(systemdRunPath, target, mountCmd, mountArgs)
} else {
// Fall back to simple mount when the host has no systemd.
// Complete command line:
// nsenter --mount=/rootfs/proc/1/ns/mnt -- /bin/mount -t <type> <what> <where>
// Expected flow is:
// * nsenter breaks out of container's mount namespace and executes host's /bin/mount.
// * mount does its job, forks a fuse daemon if necessary and finishes.
// * Any fuse daemon runs in cgroup of kubelet docker container,
// restart of kubelet container will kill it!
// No code here, mountCmd and mountArgs use /bin/mount
}
return mountCmd, mountArgs
}
// Unmount runs umount(8) in the host's mount namespace.
func (n *NsenterMounter) Unmount(target string) error {
args := []string{target}
// No need to execute systemd-run here, it's enough that unmount is executed
// in the host's mount namespace. It will finish appropriate fuse daemon(s)
// running in any scope.
klog.V(5).Infof("nsenter unmount args: %v", args)
outputBytes, err := n.ne.Exec("umount", args).CombinedOutput()
if len(outputBytes) != 0 {
klog.V(5).Infof("Output of unmounting %s: %v", target, string(outputBytes))
}
return err
}
// List returns a list of all mounted filesystems in the host's mount namespace.
func (*NsenterMounter) List() ([]MountPoint, error) {
return listProcMounts(hostProcMountsPath)
}
func (m *NsenterMounter) IsNotMountPoint(dir string) (bool, error) {
return isNotMountPoint(m, dir)
}
func (*NsenterMounter) IsMountPointMatch(mp MountPoint, dir string) bool {
deletedDir := fmt.Sprintf("%s\\040(deleted)", dir)
return (mp.Path == dir) || (mp.Path == deletedDir)
}
// IsLikelyNotMountPoint determines whether a path is a mountpoint by calling findmnt
// in the host's root mount namespace.
func (n *NsenterMounter) IsLikelyNotMountPoint(file string) (bool, error) {
file, err := filepath.Abs(file)
if err != nil {
return true, err
}
// Check the directory exists
if _, err = os.Stat(file); os.IsNotExist(err) {
klog.V(5).Infof("findmnt: directory %s does not exist", file)
return true, err
}
// Resolve any symlinks in file, kernel would do the same and use the resolved path in /proc/mounts
resolvedFile, err := n.EvalHostSymlinks(file)
if err != nil {
return true, err
}
// Add --first-only option: since we are testing for the absence of a mountpoint, it is sufficient to get only
// the first of multiple possible mountpoints using --first-only.
// Also add fstype output to make sure that the output of target file will give the full path
// TODO: Need more refactoring for this function. Track the solution with issue #26996
args := []string{"-o", "target,fstype", "--noheadings", "--first-only", "--target", resolvedFile}
klog.V(5).Infof("nsenter findmnt args: %v", args)
out, err := n.ne.Exec("findmnt", args).CombinedOutput()
if err != nil {
klog.V(2).Infof("Failed findmnt command for path %s: %s %v", resolvedFile, out, err)
// Different operating systems behave differently for paths which are not mount points.
// On older versions (e.g. 2.20.1) we'd get error, on newer ones (e.g. 2.26.2) we'd get "/".
// It's safer to assume that it's not a mount point.
return true, nil
}
mountTarget, err := parseFindMnt(string(out))
if err != nil {
return false, err
}
klog.V(5).Infof("IsLikelyNotMountPoint findmnt output for path %s: %v:", resolvedFile, mountTarget)
if mountTarget == resolvedFile {
klog.V(5).Infof("IsLikelyNotMountPoint: %s is a mount point", resolvedFile)
return false, nil
}
klog.V(5).Infof("IsLikelyNotMountPoint: %s is not a mount point", resolvedFile)
return true, nil
}
// parse output of "findmnt -o target,fstype" and return just the target
func parseFindMnt(out string) (string, error) {
// cut trailing newline
out = strings.TrimSuffix(out, "\n")
// cut everything after the last space - it's the filesystem type
i := strings.LastIndex(out, " ")
if i == -1 {
return "", fmt.Errorf("error parsing findmnt output, expected at least one space: %q", out)
}
return out[:i], nil
}
// DeviceOpened checks if block device in use by calling Open with O_EXCL flag.
// Returns true if open returns errno EBUSY, and false if errno is nil.
// Returns an error if errno is any error other than EBUSY.
// Returns with error if pathname is not a device.
func (n *NsenterMounter) DeviceOpened(pathname string) (bool, error) {
return exclusiveOpenFailsOnDevice(pathname)
}
// PathIsDevice uses FileInfo returned from os.Stat to check if path refers
// to a device.
func (n *NsenterMounter) PathIsDevice(pathname string) (bool, error) {
pathType, err := n.GetFileType(pathname)
isDevice := pathType == FileTypeCharDev || pathType == FileTypeBlockDev
return isDevice, err
}
//GetDeviceNameFromMount given a mount point, find the volume id from checking /proc/mounts
func (n *NsenterMounter) GetDeviceNameFromMount(mountPath, pluginDir string) (string, error) {
return getDeviceNameFromMount(n, mountPath, pluginDir)
}
func (n *NsenterMounter) MakeRShared(path string) error {
return doMakeRShared(path, hostProcMountinfoPath)
}
func (mounter *NsenterMounter) GetFileType(pathname string) (FileType, error) {
var pathType FileType
outputBytes, err := mounter.ne.Exec("stat", []string{"-L", "--printf=%F", pathname}).CombinedOutput()
if err != nil {
if strings.Contains(string(outputBytes), "No such file") {
err = fmt.Errorf("%s does not exist", pathname)
} else {
err = fmt.Errorf("stat %s error: %v", pathname, string(outputBytes))
}
return pathType, err
}
switch string(outputBytes) {
case "socket":
return FileTypeSocket, nil
case "character special file":
return FileTypeCharDev, nil
case "block special file":
return FileTypeBlockDev, nil
case "directory":
return FileTypeDirectory, nil
case "regular file":
return FileTypeFile, nil
}
return pathType, fmt.Errorf("only recognise file, directory, socket, block device and character device")
}
func (mounter *NsenterMounter) MakeDir(pathname string) error {
args := []string{"-p", pathname}
if _, err := mounter.ne.Exec("mkdir", args).CombinedOutput(); err != nil {
return err
}
return nil
}
func (mounter *NsenterMounter) MakeFile(pathname string) error {
args := []string{pathname}
if _, err := mounter.ne.Exec("touch", args).CombinedOutput(); err != nil {
return err
}
return nil
}
func (mounter *NsenterMounter) ExistsPath(pathname string) (bool, error) {
// Resolve the symlinks but allow the target not to exist. EvalSymlinks
// would return an generic error when the target does not exist.
hostPath, err := mounter.ne.EvalSymlinks(pathname, false /* mustExist */)
if err != nil {
return false, err
}
kubeletpath := mounter.ne.KubeletPath(hostPath)
return utilpath.Exists(utilpath.CheckFollowSymlink, kubeletpath)
}
func (mounter *NsenterMounter) EvalHostSymlinks(pathname string) (string, error) {
return mounter.ne.EvalSymlinks(pathname, true)
}
func (mounter *NsenterMounter) GetMountRefs(pathname string) ([]string, error) {
pathExists, pathErr := PathExists(pathname)
if !pathExists || IsCorruptedMnt(pathErr) {
return []string{}, nil
} else if pathErr != nil {
return nil, fmt.Errorf("Error checking path %s: %v", pathname, pathErr)
}
hostpath, err := mounter.ne.EvalSymlinks(pathname, true /* mustExist */)
if err != nil {
return nil, err
}
return searchMountPoints(hostpath, hostProcMountinfoPath)
}
func (mounter *NsenterMounter) GetFSGroup(pathname string) (int64, error) {
hostPath, err := mounter.ne.EvalSymlinks(pathname, true /* mustExist */)
if err != nil {
return -1, err
}
kubeletpath := mounter.ne.KubeletPath(hostPath)
return getFSGroup(kubeletpath)
}
func (mounter *NsenterMounter) GetSELinuxSupport(pathname string) (bool, error) {
return getSELinuxSupport(pathname, hostProcMountsPath)
}
func (mounter *NsenterMounter) GetMode(pathname string) (os.FileMode, error) {
hostPath, err := mounter.ne.EvalSymlinks(pathname, true /* mustExist */)
if err != nil {
return 0, err
}
kubeletpath := mounter.ne.KubeletPath(hostPath)
return getMode(kubeletpath)
}

110
pkg/util/mount/nsenter_mount_unsupported.go
View File

@ -1,110 +0,0 @@
// +build !linux
/*
Copyright 2014 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package mount
import (
"errors"
"os"
"k8s.io/utils/nsenter"
)
type NsenterMounter struct{}
func NewNsenterMounter(rootDir string, ne *nsenter.Nsenter) *NsenterMounter {
return &NsenterMounter{}
}
var _ = Interface(&NsenterMounter{})
func (*NsenterMounter) Mount(source string, target string, fstype string, options []string) error {
return nil
}
func (*NsenterMounter) Unmount(target string) error {
return nil
}
func (*NsenterMounter) List() ([]MountPoint, error) {
return []MountPoint{}, nil
}
func (m *NsenterMounter) IsNotMountPoint(dir string) (bool, error) {
return isNotMountPoint(m, dir)
}
func (*NsenterMounter) IsMountPointMatch(mp MountPoint, dir string) bool {
return (mp.Path == dir)
}
func (*NsenterMounter) IsLikelyNotMountPoint(file string) (bool, error) {
return true, nil
}
func (*NsenterMounter) DeviceOpened(pathname string) (bool, error) {
return false, nil
}
func (*NsenterMounter) PathIsDevice(pathname string) (bool, error) {
return true, nil
}
func (*NsenterMounter) GetDeviceNameFromMount(mountPath, pluginDir string) (string, error) {
return "", nil
}
func (*NsenterMounter) MakeRShared(path string) error {
return nil
}
func (*NsenterMounter) GetFileType(_ string) (FileType, error) {
return FileType("fake"), errors.New("not implemented")
}
func (*NsenterMounter) MakeDir(pathname string) error {
return nil
}
func (*NsenterMounter) MakeFile(pathname string) error {
return nil
}
func (*NsenterMounter) ExistsPath(pathname string) (bool, error) {
return true, errors.New("not implemented")
}
func (*NsenterMounter) EvalHostSymlinks(pathname string) (string, error) {
return "", errors.New("not implemented")
}
func (*NsenterMounter) GetMountRefs(pathname string) ([]string, error) {
return nil, errors.New("not implemented")
}
func (*NsenterMounter) GetFSGroup(pathname string) (int64, error) {
return -1, errors.New("not implemented")
}
func (*NsenterMounter) GetSELinuxSupport(pathname string) (bool, error) {
return false, errors.New("not implemented")
}
func (*NsenterMounter) GetMode(pathname string) (os.FileMode, error) {
return 0, errors.New("not implemented")
}

186
pkg/volume/util/subpath/subpath_nsenter.go
View File

@ -1,186 +0,0 @@
// +build linux
/*
Copyright 2014 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package subpath
import (
"fmt"
"os"
"path/filepath"
"syscall"
"golang.org/x/sys/unix"
"k8s.io/klog"
"k8s.io/utils/nsenter"
"k8s.io/kubernetes/pkg/util/mount"
)
type subpathNSE struct {
mounter mount.Interface
ne *nsenter.Nsenter
rootDir string
}
// Compile time-check for all implementers of subpath interface
var _ Interface = &subpathNSE{}
// NewNSEnter returns a subpath.Interface that is to be used with the NsenterMounter
// It is only valid on Linux systems
func NewNSEnter(mounter mount.Interface, ne *nsenter.Nsenter, rootDir string) Interface {
return &subpathNSE{
mounter: mounter,
ne: ne,
rootDir: rootDir,
}
}
func (sp *subpathNSE) CleanSubPaths(podDir string, volumeName string) error {
return doCleanSubPaths(sp.mounter, podDir, volumeName)
}
func (sp *subpathNSE) PrepareSafeSubpath(subPath Subpath) (newHostPath string, cleanupAction func(), err error) {
// Bind-mount the subpath to avoid using symlinks in subpaths.
newHostPath, err = sp.doNsEnterBindSubPath(subPath)
// There is no action when the container starts. Bind-mount will be cleaned
// when container stops by CleanSubPaths.
cleanupAction = nil
return newHostPath, cleanupAction, err
}
func (sp *subpathNSE) SafeMakeDir(subdir string, base string, perm os.FileMode) error {
fullSubdirPath := filepath.Join(base, subdir)
evaluatedSubdirPath, err := sp.ne.EvalSymlinks(fullSubdirPath, false /* mustExist */)
if err != nil {
return fmt.Errorf("error resolving symlinks in %s: %s", fullSubdirPath, err)
}
evaluatedSubdirPath = filepath.Clean(evaluatedSubdirPath)
evaluatedBase, err := sp.ne.EvalSymlinks(base, true /* mustExist */)
if err != nil {
return fmt.Errorf("error resolving symlinks in %s: %s", base, err)
}
evaluatedBase = filepath.Clean(evaluatedBase)
rootDir := filepath.Clean(sp.rootDir)
if mount.PathWithinBase(evaluatedBase, rootDir) {
// Base is in /var/lib/kubelet. This directory is shared between the
// container with kubelet and the host. We don't need to add '/rootfs'.
// This is useful when /rootfs is mounted as read-only - we can still
// create subpaths for paths in /var/lib/kubelet.
return doSafeMakeDir(evaluatedSubdirPath, evaluatedBase, perm)
}
// Base is somewhere on the host's filesystem. Add /rootfs and try to make
// the directory there.
// This requires /rootfs to be writable.
kubeletSubdirPath := sp.ne.KubeletPath(evaluatedSubdirPath)
kubeletBase := sp.ne.KubeletPath(evaluatedBase)
return doSafeMakeDir(kubeletSubdirPath, kubeletBase, perm)
}
func (sp *subpathNSE) doNsEnterBindSubPath(subpath Subpath) (hostPath string, err error) {
// Linux, kubelet runs in a container:
// - safely open the subpath
// - bind-mount the subpath to target (this can be unsafe)
// - check that we mounted the right thing by comparing device ID and inode
// of the subpath (via safely opened fd) and the target (that's under our
// control)
// Evaluate all symlinks here once for all subsequent functions.
evaluatedHostVolumePath, err := sp.ne.EvalSymlinks(subpath.VolumePath, true /*mustExist*/)
if err != nil {
return "", fmt.Errorf("error resolving symlinks in %q: %v", subpath.VolumePath, err)
}
evaluatedHostSubpath, err := sp.ne.EvalSymlinks(subpath.Path, true /*mustExist*/)
if err != nil {
return "", fmt.Errorf("error resolving symlinks in %q: %v", subpath.Path, err)
}
klog.V(5).Infof("doBindSubPath %q (%q) for volumepath %q", subpath.Path, evaluatedHostSubpath, subpath.VolumePath)
subpath.VolumePath = sp.ne.KubeletPath(evaluatedHostVolumePath)
subpath.Path = sp.ne.KubeletPath(evaluatedHostSubpath)
// Check the subpath is correct and open it
fd, err := safeOpenSubPath(sp.mounter, subpath)
if err != nil {
return "", err
}
defer syscall.Close(fd)
alreadyMounted, bindPathTarget, err := prepareSubpathTarget(sp.mounter, subpath)
if err != nil {
return "", err
}
if alreadyMounted {
return bindPathTarget, nil
}
success := false
defer func() {
// Cleanup subpath on error
if !success {
klog.V(4).Infof("doNsEnterBindSubPath() failed for %q, cleaning up subpath", bindPathTarget)
if cleanErr := cleanSubPath(sp.mounter, subpath); cleanErr != nil {
klog.Errorf("Failed to clean subpath %q: %v", bindPathTarget, cleanErr)
}
}
}()
// Leap of faith: optimistically expect that nobody has modified previously
// expanded evalSubPath with evil symlinks and bind-mount it.
// Mount is done on the host! don't use kubelet path!
klog.V(5).Infof("bind mounting %q at %q", evaluatedHostSubpath, bindPathTarget)
if err = sp.mounter.Mount(evaluatedHostSubpath, bindPathTarget, "" /*fstype*/, []string{"bind"}); err != nil {
return "", fmt.Errorf("error mounting %s: %s", evaluatedHostSubpath, err)
}
// Check that the bind-mount target is the same inode and device as the
// source that we keept open, i.e. we mounted the right thing.
err = checkDeviceInode(fd, bindPathTarget)
if err != nil {
return "", fmt.Errorf("error checking bind mount for subpath %s: %s", subpath.VolumePath, err)
}
success = true
klog.V(3).Infof("Bound SubPath %s into %s", subpath.Path, bindPathTarget)
return bindPathTarget, nil
}
// checkDeviceInode checks that opened file and path represent the same file.
func checkDeviceInode(fd int, path string) error {
var srcStat, dstStat unix.Stat_t
err := unix.Fstat(fd, &srcStat)
if err != nil {
return fmt.Errorf("error running fstat on subpath FD: %v", err)
}
err = unix.Stat(path, &dstStat)
if err != nil {
return fmt.Errorf("error running fstat on %s: %v", path, err)
}
if srcStat.Dev != dstStat.Dev {
return fmt.Errorf("different device number")
}
if srcStat.Ino != dstStat.Ino {
return fmt.Errorf("different inode")
}
return nil
}

346
pkg/volume/util/subpath/subpath_nsenter_test.go
View File

@ -1,346 +0,0 @@
// +build linux
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package subpath
import (
"io/ioutil"
"os"
"path/filepath"
"strings"
"testing"
"golang.org/x/sys/unix"
"k8s.io/utils/nsenter"
"k8s.io/kubernetes/pkg/util/mount"
)
func TestCheckDeviceInode(t *testing.T) {
testDir, err := ioutil.TempDir("", "nsenter-mounter-device-")
if err != nil {
t.Fatalf("Cannot create temporary directory: %s", err)
}
defer os.RemoveAll(testDir)
tests := []struct {
name string
srcPath string
dstPath string
expectError string
}{
{
name: "the same file",
srcPath: filepath.Join(testDir, "1"),
dstPath: filepath.Join(testDir, "1"),
expectError: "",
},
{
name: "different file on the same FS",
srcPath: filepath.Join(testDir, "2.1"),
dstPath: filepath.Join(testDir, "2.2"),
expectError: "different inode",
},
{
name: "different file on different device",
srcPath: filepath.Join(testDir, "3"),
// /proc is always on a different "device" than /tmp (or $TEMP)
dstPath: "/proc/self/status",
expectError: "different device",
},
}
for _, test := range tests {
if err := ioutil.WriteFile(test.srcPath, []byte{}, 0644); err != nil {
t.Errorf("Test %q: cannot create srcPath %s: %s", test.name, test.srcPath, err)
continue
}
// Don't create dst if it exists
if _, err := os.Stat(test.dstPath); os.IsNotExist(err) {
if err := ioutil.WriteFile(test.dstPath, []byte{}, 0644); err != nil {
t.Errorf("Test %q: cannot create dstPath %s: %s", test.name, test.dstPath, err)
continue
}
} else if err != nil {
t.Errorf("Test %q: cannot check existence of dstPath %s: %s", test.name, test.dstPath, err)
continue
}
fd, err := unix.Open(test.srcPath, unix.O_CREAT, 0644)
if err != nil {
t.Errorf("Test %q: cannot open srcPath %s: %s", test.name, test.srcPath, err)
continue
}
err = checkDeviceInode(fd, test.dstPath)
if test.expectError == "" && err != nil {
t.Errorf("Test %q: expected no error, got %s", test.name, err)
}
if test.expectError != "" {
if err == nil {
t.Errorf("Test %q: expected error, got none", test.name)
} else {
if !strings.Contains(err.Error(), test.expectError) {
t.Errorf("Test %q: expected error %q, got %q", test.name, test.expectError, err)
}
}
}
}
}
func newFakeNsenterMounter(tmpdir string, t *testing.T) (*mount.NsenterMounter, string, string, *nsenter.Nsenter, error) {
rootfsPath := filepath.Join(tmpdir, "rootfs")
if err := os.Mkdir(rootfsPath, 0755); err != nil {
return nil, "", "", nil, err
}
ne, err := nsenter.NewFakeNsenter(rootfsPath)
if err != nil {
return nil, "", "", nil, err
}
varlibPath := filepath.Join(tmpdir, "/var/lib/kubelet")
if err := os.MkdirAll(varlibPath, 0755); err != nil {
return nil, "", "", nil, err
}
return mount.NewNsenterMounter(varlibPath, ne), rootfsPath, varlibPath, ne, nil
}
func TestNsenterSafeMakeDir(t *testing.T) {
tests := []struct {
name string
prepare func(base, rootfs, varlib string) (expectedDir string, err error)
subdir string
expectError bool
// If true, "base" directory for SafeMakeDir will be /var/lib/kubelet
baseIsVarLib bool
}{
{
name: "simple directory",
// evaluated in base
subdir: "some/subdirectory/structure",
prepare: func(base, rootfs, varlib string) (expectedDir string, err error) {
// expected to be created in /roots/
expectedDir = filepath.Join(rootfs, base, "some/subdirectory/structure")
return expectedDir, nil
},
},
{
name: "simple existing directory",
// evaluated in base
subdir: "some/subdirectory/structure",
prepare: func(base, rootfs, varlib string) (expectedDir string, err error) {
// On the host: directory exists
hostPath := filepath.Join(base, "some/subdirectory/structure")
if err := os.MkdirAll(hostPath, 0755); err != nil {
return "", err
}
// In rootfs: directory exists
kubeletPath := filepath.Join(rootfs, hostPath)
if err := os.MkdirAll(kubeletPath, 0755); err != nil {
return "", err
}
// expected to be created in /roots/
expectedDir = kubeletPath
return expectedDir, nil
},
},
{
name: "absolute symlink into safe place",
// evaluated in base
subdir: "some/subdirectory/structure",
prepare: func(base, rootfs, varlib string) (expectedDir string, err error) {
// On the host: /base/other/subdirectory exists, /base/some is link to /base/other
hostPath := filepath.Join(base, "other/subdirectory")
if err := os.MkdirAll(hostPath, 0755); err != nil {
return "", err
}
somePath := filepath.Join(base, "some")
otherPath := filepath.Join(base, "other")
if err := os.Symlink(otherPath, somePath); err != nil {
return "", err
}
// In rootfs: /base/other/subdirectory exists
kubeletPath := filepath.Join(rootfs, hostPath)
if err := os.MkdirAll(kubeletPath, 0755); err != nil {
return "", err
}
// expected 'structure' to be created
expectedDir = filepath.Join(rootfs, hostPath, "structure")
return expectedDir, nil
},
},
{
name: "relative symlink into safe place",
// evaluated in base
subdir: "some/subdirectory/structure",
prepare: func(base, rootfs, varlib string) (expectedDir string, err error) {
// On the host: /base/other/subdirectory exists, /base/some is link to other
hostPath := filepath.Join(base, "other/subdirectory")
if err := os.MkdirAll(hostPath, 0755); err != nil {
return "", err
}
somePath := filepath.Join(base, "some")
if err := os.Symlink("other", somePath); err != nil {
return "", err
}
// In rootfs: /base/other/subdirectory exists
kubeletPath := filepath.Join(rootfs, hostPath)
if err := os.MkdirAll(kubeletPath, 0755); err != nil {
return "", err
}
// expected 'structure' to be created
expectedDir = filepath.Join(rootfs, hostPath, "structure")
return expectedDir, nil
},
},
{
name: "symlink into unsafe place",
// evaluated in base
subdir: "some/subdirectory/structure",
prepare: func(base, rootfs, varlib string) (expectedDir string, err error) {
// On the host: /base/some is link to /bin/other
somePath := filepath.Join(base, "some")
if err := os.Symlink("/bin", somePath); err != nil {
return "", err
}
return "", nil
},
expectError: true,
},
{
name: "simple directory in /var/lib/kubelet",
// evaluated in varlib
subdir: "some/subdirectory/structure",
baseIsVarLib: true,
prepare: func(base, rootfs, varlib string) (expectedDir string, err error) {
// expected to be created in /base/var/lib/kubelet, not in /rootfs!
expectedDir = filepath.Join(varlib, "some/subdirectory/structure")
return expectedDir, nil
},
},
{
name: "safe symlink in /var/lib/kubelet",
// evaluated in varlib
subdir: "some/subdirectory/structure",
baseIsVarLib: true,
prepare: func(base, rootfs, varlib string) (expectedDir string, err error) {
// On the host: /varlib/kubelet/other/subdirectory exists, /varlib/some is link to other
hostPath := filepath.Join(varlib, "other/subdirectory")
if err := os.MkdirAll(hostPath, 0755); err != nil {
return "", err
}
somePath := filepath.Join(varlib, "some")
if err := os.Symlink("other", somePath); err != nil {
return "", err
}
// expected to be created in /base/var/lib/kubelet, not in /rootfs!
expectedDir = filepath.Join(varlib, "other/subdirectory/structure")
return expectedDir, nil
},
},
{
name: "unsafe symlink in /var/lib/kubelet",
// evaluated in varlib
subdir: "some/subdirectory/structure",
baseIsVarLib: true,
prepare: func(base, rootfs, varlib string) (expectedDir string, err error) {
// On the host: /varlib/some is link to /bin
somePath := filepath.Join(varlib, "some")
if err := os.Symlink("/bin", somePath); err != nil {
return "", err
}
return "", nil
},
expectError: true,
},
}
for _, test := range tests {
tmpdir, err := ioutil.TempDir("", "nsenter-get-safedir-")
if err != nil {
t.Error(err)
continue
}
defer os.RemoveAll(tmpdir)
mounter, rootfs, varlib, ne, err := newFakeNsenterMounter(tmpdir, t)
if err != nil {
t.Error(err)
continue
}
fsp := NewNSEnter(mounter, ne, varlib)
// Prepare base directory for the test
testBase := filepath.Join(tmpdir, "base")
if err := os.Mkdir(testBase, 0755); err != nil {
t.Error(err)
continue
}
// Prepare base directory also in /rootfs
rootfsBase := filepath.Join(rootfs, testBase)
if err := os.MkdirAll(rootfsBase, 0755); err != nil {
t.Error(err)
continue
}
expectedDir := ""
if test.prepare != nil {
expectedDir, err = test.prepare(testBase, rootfs, varlib)
if err != nil {
t.Error(err)
continue
}
}
if test.baseIsVarLib {
// use /var/lib/kubelet as the test base so we can test creating
// subdirs there directly in /var/lib/kubenet and not in
// /rootfs/var/lib/kubelet
testBase = varlib
}
err = fsp.SafeMakeDir(test.subdir, testBase, 0755)
if err != nil && !test.expectError {
t.Errorf("Test %q: unexpected error: %s", test.name, err)
}
if test.expectError {
if err == nil {
t.Errorf("Test %q: expected error, got none", test.name)
} else {
if !strings.Contains(err.Error(), "is outside of allowed base") {
t.Errorf("Test %q: expected error to contain \"is outside of allowed base\", got this one instead: %s", test.name, err)
}
}
}
if expectedDir != "" {
_, err := os.Stat(expectedDir)
if err != nil {
t.Errorf("Test %q: expected %q to exist, got error: %s", test.name, expectedDir, err)
}
}
}
}

63
vendor/k8s.io/utils/nsenter/BUILD generated vendored
View File

@ -1,63 +0,0 @@
load("@io_bazel_rules_go//go:def.bzl", "go_library")
go_library(
name = "go_default_library",
srcs = [
"nsenter.go",
"nsenter_unsupported.go",
],
importmap = "k8s.io/kubernetes/vendor/k8s.io/utils/nsenter",
importpath = "k8s.io/utils/nsenter",
visibility = ["//visibility:public"],
deps = select({
"@io_bazel_rules_go//go/platform:android": [
"//vendor/k8s.io/utils/exec:go_default_library",
],
"@io_bazel_rules_go//go/platform:darwin": [
"//vendor/k8s.io/utils/exec:go_default_library",
],
"@io_bazel_rules_go//go/platform:dragonfly": [
"//vendor/k8s.io/utils/exec:go_default_library",
],
"@io_bazel_rules_go//go/platform:freebsd": [
"//vendor/k8s.io/utils/exec:go_default_library",
],
"@io_bazel_rules_go//go/platform:linux": [
"//vendor/k8s.io/klog:go_default_library",
"//vendor/k8s.io/utils/exec:go_default_library",
],
"@io_bazel_rules_go//go/platform:nacl": [
"//vendor/k8s.io/utils/exec:go_default_library",
],
"@io_bazel_rules_go//go/platform:netbsd": [
"//vendor/k8s.io/utils/exec:go_default_library",
],
"@io_bazel_rules_go//go/platform:openbsd": [
"//vendor/k8s.io/utils/exec:go_default_library",
],
"@io_bazel_rules_go//go/platform:plan9": [
"//vendor/k8s.io/utils/exec:go_default_library",
],
"@io_bazel_rules_go//go/platform:solaris": [
"//vendor/k8s.io/utils/exec:go_default_library",
],
"@io_bazel_rules_go//go/platform:windows": [
"//vendor/k8s.io/utils/exec:go_default_library",
],
"//conditions:default": [],
}),
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

10
vendor/k8s.io/utils/nsenter/OWNERS generated vendored
View File

@ -1,10 +0,0 @@
# See the OWNERS docs at https://go.k8s.io/owners
reviewers:
- jsafrane
- msau42
- cofyc
approvers:
- jsafrane
- msau42
- cofyc

258
vendor/k8s.io/utils/nsenter/nsenter.go generated vendored
View File

@ -1,258 +0,0 @@
// +build linux
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package nsenter
import (
"context"
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"k8s.io/klog"
"k8s.io/utils/exec"
)
const (
// DefaultHostRootFsPath is path to host's filesystem mounted into container
// with kubelet.
DefaultHostRootFsPath = "/rootfs"
// mountNsPath is the default mount namespace of the host
mountNsPath = "/proc/1/ns/mnt"
// nsenterPath is the default nsenter command
nsenterPath = "nsenter"
)
// Nsenter is a type alias for backward compatibility
type Nsenter = NSEnter
// NSEnter is part of experimental support for running the kubelet
// in a container.
//
// NSEnter requires:
//
// 1. Docker >= 1.6 due to the dependency on the slave propagation mode
// of the bind-mount of the kubelet root directory in the container.
// Docker 1.5 used a private propagation mode for bind-mounts, so mounts
// performed in the host's mount namespace do not propagate out to the
// bind-mount in this docker version.
// 2. The host's root filesystem must be available at /rootfs
// 3. The nsenter binary must be on the Kubelet process' PATH in the container's
// filesystem.
// 4. The Kubelet process must have CAP_SYS_ADMIN (required by nsenter); at
// the present, this effectively means that the kubelet is running in a
// privileged container.
// 5. The volume path used by the Kubelet must be the same inside and outside
// the container and be writable by the container (to initialize volume)
// contents. TODO: remove this requirement.
// 6. The host image must have "mount", "findmnt", "umount", "stat", "touch",
// "mkdir", "ls", "sh" and "chmod" binaries in /bin, /usr/sbin, or /usr/bin
// 7. The host image should have systemd-run in /bin, /usr/sbin, or /usr/bin if
// systemd is installed/enabled in the operating system.
// For more information about mount propagation modes, see:
// https://www.kernel.org/doc/Documentation/filesystems/sharedsubtree.txt
type NSEnter struct {
// a map of commands to their paths on the host filesystem
paths map[string]string
// Path to the host filesystem, typically "/rootfs". Used only for testing.
hostRootFsPath string
// Exec implementation
executor exec.Interface
}
// NewNsenter constructs a new instance of NSEnter
func NewNsenter(hostRootFsPath string, executor exec.Interface) (*NSEnter, error) {
ne := &NSEnter{
hostRootFsPath: hostRootFsPath,
executor: executor,
}
if err := ne.initPaths(); err != nil {
return nil, err
}
return ne, nil
}
func (ne *NSEnter) initPaths() error {
ne.paths = map[string]string{}
binaries := []string{
"mount",
"findmnt",
"umount",
"systemd-run",
"stat",
"touch",
"mkdir",
"sh",
"chmod",
"realpath",
}
// search for the required commands in other locations besides /usr/bin
for _, binary := range binaries {
// check for binary under the following directories
for _, path := range []string{"/", "/bin", "/usr/sbin", "/usr/bin"} {
binPath := filepath.Join(path, binary)
if _, err := os.Stat(filepath.Join(ne.hostRootFsPath, binPath)); err != nil {
continue
}
ne.paths[binary] = binPath
break
}
// systemd-run is optional, bailout if we don't find any of the other binaries
if ne.paths[binary] == "" && binary != "systemd-run" {
return fmt.Errorf("unable to find %v", binary)
}
}
return nil
}
// Exec executes nsenter commands in hostProcMountNsPath mount namespace
func (ne *NSEnter) Exec(cmd string, args []string) exec.Cmd {
hostProcMountNsPath := filepath.Join(ne.hostRootFsPath, mountNsPath)
fullArgs := append([]string{fmt.Sprintf("--mount=%s", hostProcMountNsPath), "--"},
append([]string{ne.AbsHostPath(cmd)}, args...)...)
klog.V(5).Infof("Running nsenter command: %v %v", nsenterPath, fullArgs)
return ne.executor.Command(nsenterPath, fullArgs...)
}
// Command returns a command wrapped with nsenter
func (ne *NSEnter) Command(cmd string, args ...string) exec.Cmd {
return ne.Exec(cmd, args)
}
// CommandContext returns a CommandContext wrapped with nsenter
func (ne *NSEnter) CommandContext(ctx context.Context, cmd string, args ...string) exec.Cmd {
hostProcMountNsPath := filepath.Join(ne.hostRootFsPath, mountNsPath)
fullArgs := append([]string{fmt.Sprintf("--mount=%s", hostProcMountNsPath), "--"},
append([]string{ne.AbsHostPath(cmd)}, args...)...)
klog.V(5).Infof("Running nsenter command: %v %v", nsenterPath, fullArgs)
return ne.executor.CommandContext(ctx, nsenterPath, fullArgs...)
}
// LookPath returns a LookPath wrapped with nsenter
func (ne *NSEnter) LookPath(file string) (string, error) {
return "", fmt.Errorf("not implemented, error looking up : %s", file)
}
// AbsHostPath returns the absolute runnable path for a specified command
func (ne *NSEnter) AbsHostPath(command string) string {
path, ok := ne.paths[command]
if !ok {
return command
}
return path
}
// SupportsSystemd checks whether command systemd-run exists
func (ne *NSEnter) SupportsSystemd() (string, bool) {
systemdRunPath, ok := ne.paths["systemd-run"]
return systemdRunPath, ok && systemdRunPath != ""
}
// EvalSymlinks returns the path name on the host after evaluating symlinks on the
// host.
// mustExist makes EvalSymlinks to return error when the path does not
// exist. When it's false, it evaluates symlinks of the existing part and
// blindly adds the non-existing part:
// pathname: /mnt/volume/non/existing/directory
// /mnt/volume exists
// non/existing/directory does not exist
// -> It resolves symlinks in /mnt/volume to say /mnt/foo and returns
// /mnt/foo/non/existing/directory.
//
// BEWARE! EvalSymlinks is not able to detect symlink looks with mustExist=false!
// If /tmp/link is symlink to /tmp/link, EvalSymlinks(/tmp/link/foo) returns /tmp/link/foo.
func (ne *NSEnter) EvalSymlinks(pathname string, mustExist bool) (string, error) {
var args []string
if mustExist {
// "realpath -e: all components of the path must exist"
args = []string{"-e", pathname}
} else {
// "realpath -m: no path components need exist or be a directory"
args = []string{"-m", pathname}
}
outBytes, err := ne.Exec("realpath", args).CombinedOutput()
if err != nil {
klog.Infof("failed to resolve symbolic links on %s: %v", pathname, err)
return "", err
}
return strings.TrimSpace(string(outBytes)), nil
}
// KubeletPath returns the path name that can be accessed by containerized
// kubelet. It is recommended to resolve symlinks on the host by EvalSymlinks
// before calling this function
func (ne *NSEnter) KubeletPath(pathname string) string {
return filepath.Join(ne.hostRootFsPath, pathname)
}
// NewFakeNsenter returns a NSEnter that does not run "nsenter --mount=... --",
// but runs everything in the same mount namespace as the unit test binary.
// rootfsPath is supposed to be a symlink, e.g. /tmp/xyz/rootfs -> /.
// This fake NSEnter is enough for most operations, e.g. to resolve symlinks,
// but it's not enough to call /bin/mount - unit tests don't run as root.
func NewFakeNsenter(rootfsPath string) (*NSEnter, error) {
executor := &fakeExec{
rootfsPath: rootfsPath,
}
// prepare /rootfs/bin, usr/bin and usr/sbin
bin := filepath.Join(rootfsPath, "bin")
if err := os.Symlink("/bin", bin); err != nil {
return nil, err
}
usr := filepath.Join(rootfsPath, "usr")
if err := os.Mkdir(usr, 0755); err != nil {
return nil, err
}
usrbin := filepath.Join(usr, "bin")
if err := os.Symlink("/usr/bin", usrbin); err != nil {
return nil, err
}
usrsbin := filepath.Join(usr, "sbin")
if err := os.Symlink("/usr/sbin", usrsbin); err != nil {
return nil, err
}
return NewNsenter(rootfsPath, executor)
}
type fakeExec struct {
rootfsPath string
}
func (f fakeExec) Command(cmd string, args ...string) exec.Cmd {
// This will intentionaly panic if NSEnter does not provide enough arguments.
realCmd := args[2]
realArgs := args[3:]
return exec.New().Command(realCmd, realArgs...)
}
func (fakeExec) LookPath(file string) (string, error) {
return "", errors.New("not implemented")
}
func (fakeExec) CommandContext(ctx context.Context, cmd string, args ...string) exec.Cmd {
return nil
}
var _ exec.Interface = fakeExec{}
var _ exec.Interface = &NSEnter{}

79
vendor/k8s.io/utils/nsenter/nsenter_unsupported.go generated vendored
View File

@ -1,79 +0,0 @@
// +build !linux
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package nsenter
import (
"context"
"fmt"
"k8s.io/utils/exec"
)
const (
// DefaultHostRootFsPath is path to host's filesystem mounted into container
// with kubelet.
DefaultHostRootFsPath = "/rootfs"
)
// Nsenter is a type alias for backward compatibility
type Nsenter = NSEnter
// NSEnter is part of experimental support for running the kubelet
// in a container.
type NSEnter struct {
// a map of commands to their paths on the host filesystem
Paths map[string]string
}
// NewNsenter constructs a new instance of NSEnter
func NewNsenter(hostRootFsPath string, executor exec.Interface) (*Nsenter, error) {
return &Nsenter{}, nil
}
// Exec executes nsenter commands in hostProcMountNsPath mount namespace
func (ne *NSEnter) Exec(cmd string, args []string) exec.Cmd {
return nil
}
// AbsHostPath returns the absolute runnable path for a specified command
func (ne *NSEnter) AbsHostPath(command string) string {
return ""
}
// SupportsSystemd checks whether command systemd-run exists
func (ne *NSEnter) SupportsSystemd() (string, bool) {
return "", false
}
// Command returns a command wrapped with nenter
func (ne *NSEnter) Command(cmd string, args ...string) exec.Cmd {
return nil
}
// CommandContext returns a CommandContext wrapped with nsenter
func (ne *NSEnter) CommandContext(ctx context.Context, cmd string, args ...string) exec.Cmd {
return nil
}
// LookPath returns a LookPath wrapped with nsenter
func (ne *NSEnter) LookPath(file string) (string, error) {
return "", fmt.Errorf("not implemented, error looking up : %s", file)
}
var _ exec.Interface = &NSEnter{}