vmware-tanzu
/
velero
mirror of https://github.com/vmware-tanzu/velero.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #4864 from sseago/vsl-creds-main 

Add credentials to volume snapshot locations.
pull/5335/head
Shubham Pampattiwar 2022-09-12 09:56:49 -04:00 committed by GitHub
parent 325b8c0d05 596114b427
commit be40d7eb19
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 189 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
changelogs/unreleased/4864-sseago Normal file
View File

@ -0,0 +1 @@
Add credentials to volume snapshot locations

18
config/crd/v1/bases/velero.io_volumesnapshotlocations.yaml
View File

@ -45,6 +45,24 @@ spec:
type: string
description: Config is for provider-specific configuration fields.
type: object
credential:
description: Credential contains the credential information intended
to be used with this location
properties:
key:
description: The key of the secret to select from. Must be a
valid secret key.
type: string
name:
description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
TODO: Add other useful fields. apiVersion, kind, uid?'
type: string
optional:
description: Specify whether the Secret or its key must be defined
type: boolean
required:
- key
type: object
provider:
description: Provider is the provider of the volume storage.
type: string

2
config/crd/v1/crds/crds.go
View File

File diff suppressed because one or more lines are too long

9
pkg/apis/velero/v1/volume_snapshot_location_type.go
View File

@ -16,7 +16,10 @@ limitations under the License.
package v1
import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
import (
corev1api "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
// +genclient
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
@ -61,6 +64,10 @@ type VolumeSnapshotLocationSpec struct {
// Config is for provider-specific configuration fields.
// +optional
Config map[string]string `json:"config,omitempty"`
// Credential contains the credential information intended to be used with this location
// +optional
Credential *corev1api.SecretKeySelector `json:"credential,omitempty"`
}
// VolumeSnapshotLocationPhase is the lifecycle phase of a Velero VolumeSnapshotLocation.

5
pkg/apis/velero/v1/zz_generated.deepcopy.go
View File

@ -1656,6 +1656,11 @@ func (in *VolumeSnapshotLocationSpec) DeepCopyInto(out *VolumeSnapshotLocationSp
(*out)[key] = val
}
}
if in.Credential != nil {
in, out := &in.Credential, &out.Credential
*out = new(corev1.SecretKeySelector)
(*in).DeepCopyInto(*out)
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VolumeSnapshotLocationSpec.

10
pkg/builder/volume_snapshot_location_builder.go
View File

@ -1,5 +1,5 @@
/*
Copyright 2019 the Velero contributors.
Copyright the Velero contributors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
@ -19,6 +19,8 @@ package builder
import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
corev1api "k8s.io/api/core/v1"
velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
)
@ -62,3 +64,9 @@ func (b *VolumeSnapshotLocationBuilder) Provider(name string) *VolumeSnapshotLoc
b.object.Spec.Provider = name
return b
}
// Credential sets the VolumeSnapshotLocation's credential selector.
func (b *VolumeSnapshotLocationBuilder) Credential(selector *corev1api.SecretKeySelector) *VolumeSnapshotLocationBuilder {
b.object.Spec.Credential = selector
return b
}

35
pkg/cmd/cli/snapshotlocation/create.go
View File

@ -1,5 +1,5 @@
/*
Copyright 2020 the Velero contributors.
Copyright the Velero contributors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
@ -26,6 +26,7 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
"github.com/vmware-tanzu/velero/pkg/builder"
"github.com/vmware-tanzu/velero/pkg/client"
"github.com/vmware-tanzu/velero/pkg/cmd"
"github.com/vmware-tanzu/velero/pkg/cmd/util/flag"
@ -54,16 +55,18 @@ func NewCreateCommand(f client.Factory, use string) *cobra.Command {
}
type CreateOptions struct {
Name string
Provider string
Config flag.Map
Labels flag.Map
Name string
Provider string
Config flag.Map
Labels flag.Map
Credential flag.Map
}
func NewCreateOptions() *CreateOptions {
return &CreateOptions{
Config: flag.NewMap(),
Labels: flag.NewMap(),
Config: flag.NewMap(),
Labels: flag.NewMap(),
Credential: flag.NewMap(),
}
}
@ -71,6 +74,7 @@ func (o *CreateOptions) BindFlags(flags *pflag.FlagSet) {
flags.StringVar(&o.Provider, "provider", o.Provider, "Name of the volume snapshot provider (e.g. aws, azure, gcp).")
flags.Var(&o.Config, "config", "Configuration key-value pairs.")
flags.Var(&o.Labels, "labels", "Labels to apply to the volume snapshot location.")
flags.Var(&o.Credential, "credential", "The credential to be used by this location as a key-value pair, where the key is the Kubernetes Secret name, and the value is the data key name within the Secret. Optional, one value only.")
}
func (o *CreateOptions) Validate(c *cobra.Command, args []string, f client.Factory) error {
@ -82,6 +86,10 @@ func (o *CreateOptions) Validate(c *cobra.Command, args []string, f client.Facto
return errors.New("--provider is required")
}
if len(o.Credential.Data()) > 1 {
return errors.New("--credential can only contain 1 key/value pair")
}
return nil
}
@ -90,10 +98,10 @@ func (o *CreateOptions) Complete(args []string, f client.Factory) error {
return nil
}
func (o *CreateOptions) Run(c *cobra.Command, f client.Factory) error {
func (o *CreateOptions) BuildVolumeSnapshotLocation(namespace string) *api.VolumeSnapshotLocation {
volumeSnapshotLocation := &api.VolumeSnapshotLocation{
ObjectMeta: metav1.ObjectMeta{
Namespace: f.Namespace(),
Namespace: namespace,
Name: o.Name,
Labels: o.Labels.Data(),
},
@ -102,6 +110,15 @@ func (o *CreateOptions) Run(c *cobra.Command, f client.Factory) error {
Config: o.Config.Data(),
},
}
for secretName, secretKey := range o.Credential.Data() {
volumeSnapshotLocation.Spec.Credential = builder.ForSecretKeySelector(secretName, secretKey).Result()
break
}
return volumeSnapshotLocation
}
func (o *CreateOptions) Run(c *cobra.Command, f client.Factory) error {
volumeSnapshotLocation := o.BuildVolumeSnapshotLocation(f.Namespace())
if printed, err := output.PrintWithFormat(c, volumeSnapshotLocation); printed || err != nil {
return err

24
pkg/cmd/cli/snapshotlocation/set.go
View File

@ -27,8 +27,10 @@ import (
kbclient "sigs.k8s.io/controller-runtime/pkg/client"
velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
"github.com/vmware-tanzu/velero/pkg/builder"
"github.com/vmware-tanzu/velero/pkg/client"
"github.com/vmware-tanzu/velero/pkg/cmd"
"github.com/vmware-tanzu/velero/pkg/cmd/util/flag"
"github.com/vmware-tanzu/velero/pkg/cmd/util/output"
)
@ -39,9 +41,6 @@ func NewSetCommand(f client.Factory, use string) *cobra.Command {
Use: use + " NAME",
Short: "Set specific features for a snapshot location",
Args: cobra.ExactArgs(1),
// Mark this command as hidden until more functionality is added
// as part of https://github.com/vmware-tanzu/velero/issues/2426
Hidden: true,
Run: func(c *cobra.Command, args []string) {
cmd.CheckError(o.Complete(args, f))
cmd.CheckError(o.Validate(c, args, f))
@ -54,14 +53,18 @@ func NewSetCommand(f client.Factory, use string) *cobra.Command {
}
type SetOptions struct {
Name string
Name string
Credential flag.Map
}
func NewSetOptions() *SetOptions {
return &SetOptions{}
return &SetOptions{
Credential: flag.NewMap(),
}
}
func (o *SetOptions) BindFlags(*pflag.FlagSet) {
func (o *SetOptions) BindFlags(flags *pflag.FlagSet) {
flags.Var(&o.Credential, "credential", "Sets the credential to be used by this location as a key-value pair, where the key is the Kubernetes Secret name, and the value is the data key name within the Secret. Optional, one value only.")
}
func (o *SetOptions) Validate(c *cobra.Command, args []string, f client.Factory) error {
@ -69,6 +72,10 @@ func (o *SetOptions) Validate(c *cobra.Command, args []string, f client.Factory)
return err
}
if len(o.Credential.Data()) > 1 {
return errors.New("--credential can only contain 1 key/value pair")
}
return nil
}
@ -92,6 +99,11 @@ func (o *SetOptions) Run(c *cobra.Command, f client.Factory) error {
return errors.WithStack(err)
}
for name, key := range o.Credential.Data() {
location.Spec.Credential = builder.ForSecretKeySelector(name, key).Result()
break
}
if err := kbClient.Update(context.Background(), location, &kbclient.UpdateOptions{}); err != nil {
return errors.WithStack(err)
}

2
pkg/cmd/server/server.go
View File

@ -650,6 +650,7 @@ func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string
csiVSCLister,
csiVSClassLister,
backupStoreGetter,
s.credentialFileStore,
)
return controllerRunInfo{
@ -672,6 +673,7 @@ func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string
s.logger,
podexec.NewPodCommandExecutor(s.kubeClientConfig, s.kubeClient.CoreV1().RESTClient()),
s.kubeClient.CoreV1().RESTClient(),
s.credentialFileStore,
)
cmd.CheckError(err)

13
pkg/controller/backup_controller.go
View File

@ -50,6 +50,7 @@ import (
snapshotterClientSet "github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned"
snapshotv1listers "github.com/kubernetes-csi/external-snapshotter/client/v4/listers/volumesnapshot/v1"
"github.com/vmware-tanzu/velero/internal/credentials"
"github.com/vmware-tanzu/velero/internal/storage"
velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
pkgbackup "github.com/vmware-tanzu/velero/pkg/backup"
@ -98,6 +99,7 @@ type backupController struct {
volumeSnapshotClient *snapshotterClientSet.Clientset
volumeSnapshotContentLister snapshotv1listers.VolumeSnapshotContentLister
volumeSnapshotClassLister snapshotv1listers.VolumeSnapshotClassLister
credentialFileStore credentials.FileStore
}
func NewBackupController(
@ -123,6 +125,7 @@ func NewBackupController(
volumeSnapshotContentLister snapshotv1listers.VolumeSnapshotContentLister,
volumesnapshotClassLister snapshotv1listers.VolumeSnapshotClassLister,
backupStoreGetter persistence.ObjectBackupStoreGetter,
credentialStore credentials.FileStore,
) Interface {
c := &backupController{
genericController: newGenericController(Backup, logger),
@ -148,6 +151,7 @@ func NewBackupController(
volumeSnapshotContentLister: volumeSnapshotContentLister,
volumeSnapshotClassLister: volumesnapshotClassLister,
backupStoreGetter: backupStoreGetter,
credentialFileStore: credentialStore,
}
c.syncHandler = c.processBackup
@ -566,6 +570,15 @@ func (c *backupController) validateAndGetSnapshotLocations(backup *velerov1api.B
return nil, errors
}
// add credential to config for each location
for _, location := range providerLocations {
err = volume.UpdateVolumeSnapshotLocationWithCredentialConfig(location, c.credentialFileStore, c.logger)
if err != nil {
errors = append(errors, fmt.Sprintf("error adding credentials to volume snapshot location named %s: %v", location.Name, err))
continue
}
}
return providerLocations, nil
}

3
pkg/controller/backup_controller_test.go
View File

@ -1007,12 +1007,15 @@ func TestValidateAndGetSnapshotLocations(t *testing.T) {
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
formatFlag := logging.FormatText
var (
client = fake.NewSimpleClientset()
sharedInformers = informers.NewSharedInformerFactory(client, 0)
logger = logging.DefaultLogger(logrus.DebugLevel, formatFlag)
)
c := &backupController{
genericController: newGenericController("backup-test", logger),
snapshotLocationLister: sharedInformers.Velero().V1().VolumeSnapshotLocations().Lister(),
defaultSnapshotLocations: test.defaultLocations,
}

11
pkg/restore/pv_restorer.go
View File

@ -21,6 +21,7 @@ import (
"github.com/sirupsen/logrus"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"github.com/vmware-tanzu/velero/internal/credentials"
api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
listers "github.com/vmware-tanzu/velero/pkg/generated/listers/velero/v1"
"github.com/vmware-tanzu/velero/pkg/util/boolptr"
@ -39,6 +40,7 @@ type pvRestorer struct {
volumeSnapshots []*volume.Snapshot
volumeSnapshotterGetter VolumeSnapshotterGetter
snapshotLocationLister listers.VolumeSnapshotLocationLister
credentialFileStore credentials.FileStore
}
func (r *pvRestorer) executePVAction(obj *unstructured.Unstructured) (*unstructured.Unstructured, error) {
@ -59,7 +61,7 @@ func (r *pvRestorer) executePVAction(obj *unstructured.Unstructured) (*unstructu
log := r.logger.WithFields(logrus.Fields{"persistentVolume": pvName})
snapshotInfo, err := getSnapshotInfo(pvName, r.backup, r.volumeSnapshots, r.snapshotLocationLister)
snapshotInfo, err := getSnapshotInfo(pvName, r.backup, r.volumeSnapshots, r.snapshotLocationLister, r.credentialFileStore, r.logger)
if err != nil {
return nil, err
}
@ -103,7 +105,7 @@ type snapshotInfo struct {
location *api.VolumeSnapshotLocation
}
func getSnapshotInfo(pvName string, backup *api.Backup, volumeSnapshots []*volume.Snapshot, snapshotLocationLister listers.VolumeSnapshotLocationLister) (*snapshotInfo, error) {
func getSnapshotInfo(pvName string, backup *api.Backup, volumeSnapshots []*volume.Snapshot, snapshotLocationLister listers.VolumeSnapshotLocationLister, credentialStore credentials.FileStore, logger logrus.FieldLogger) (*snapshotInfo, error) {
var pvSnapshot *volume.Snapshot
for _, snapshot := range volumeSnapshots {
if snapshot.Spec.PersistentVolumeName == pvName {
@ -120,6 +122,11 @@ func getSnapshotInfo(pvName string, backup *api.Backup, volumeSnapshots []*volum
if err != nil {
return nil, errors.WithStack(err)
}
// add credential to config
err = volume.UpdateVolumeSnapshotLocationWithCredentialConfig(loc, credentialStore, logger)
if err != nil {
return nil, errors.WithStack(err)
}
return &snapshotInfo{
providerSnapshotID: pvSnapshot.Status.ProviderSnapshotID,

11
pkg/restore/restore.go
View File

@ -46,6 +46,7 @@ import (
corev1 "k8s.io/client-go/kubernetes/typed/core/v1"
"k8s.io/client-go/tools/cache"
"github.com/vmware-tanzu/velero/internal/credentials"
"github.com/vmware-tanzu/velero/internal/hook"
velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
"github.com/vmware-tanzu/velero/pkg/archive"
@ -113,6 +114,7 @@ type kubernetesRestorer struct {
logger logrus.FieldLogger
podCommandExecutor podexec.PodCommandExecutor
podGetter cache.Getter
credentialFileStore credentials.FileStore
}
// NewKubernetesRestorer creates a new kubernetesRestorer.
@ -128,6 +130,7 @@ func NewKubernetesRestorer(
logger logrus.FieldLogger,
podCommandExecutor podexec.PodCommandExecutor,
podGetter cache.Getter,
credentialStore credentials.FileStore,
) (Restorer, error) {
return &kubernetesRestorer{
restoreClient: restoreClient,
@ -147,9 +150,10 @@ func NewKubernetesRestorer(
veleroCloneName := "velero-clone-" + veleroCloneUuid.String()
return veleroCloneName, nil
},
fileSystem: filesystem.NewFileSystem(),
podCommandExecutor: podCommandExecutor,
podGetter: podGetter,
fileSystem: filesystem.NewFileSystem(),
podCommandExecutor: podCommandExecutor,
podGetter: podGetter,
credentialFileStore: credentialStore,
}, nil
}
@ -276,6 +280,7 @@ func (kr *kubernetesRestorer) RestoreWithResolvers(
volumeSnapshots: req.VolumeSnapshots,
volumeSnapshotterGetter: volumeSnapshotterGetter,
snapshotLocationLister: snapshotLocationLister,
credentialFileStore: kr.credentialFileStore,
}
restoreCtx := &restoreContext{

44
pkg/volume/snapshotlocation.go Normal file
View File

@ -0,0 +1,44 @@
/*
Copyright the Velero contributors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package volume
import (
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"github.com/vmware-tanzu/velero/internal/credentials"
velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
)
// UpdateVolumeSnapshotLocationWithCredentialConfig adds the credentials file path to the config
// if the VSL specifies a credential
func UpdateVolumeSnapshotLocationWithCredentialConfig(location *velerov1api.VolumeSnapshotLocation, credentialStore credentials.FileStore, logger logrus.FieldLogger) error {
if location.Spec.Config == nil {
location.Spec.Config = make(map[string]string)
}
// If the VSL specifies a credential, fetch its path on disk and pass to
// plugin via the config.
if location.Spec.Credential != nil && credentialStore != nil {
credsFile, err := credentialStore.Path(location.Spec.Credential)
if err != nil {
return errors.Wrap(err, "unable to get credentials")
}
location.Spec.Config["credentialsFile"] = credsFile
}
return nil
}

6
site/content/docs/main/api-types/volumesnapshotlocation.md
View File

@ -21,6 +21,9 @@ metadata:
namespace: velero
spec:
provider: aws
credential:
name: secret-name
key: key-in-secret
config:
region: us-west-2
profile: "default"
@ -37,4 +40,7 @@ The configurable parameters are as follows:
| --- | --- | --- | --- |
| `provider` | String | Required Field | The name for whichever storage provider will be used to create/store the volume snapshots. See [your volume snapshot provider's plugin documentation](../supported-providers) for the appropriate value to use. |
| `config` | map string string | None (Optional) | Provider-specific configuration keys/values to be passed to the volume snapshotter plugin. See [your volume snapshot provider's plugin documentation](../supported-providers) for details. |
| `credential` | [corev1.SecretKeySelector](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#secretkeyselector-v1-core) | Optional Field | The credential information to be used with this location. |
| `credential/name` | String | Optional Field | The name of the secret within the Velero namespace which contains the credential information. |
| `credential/key` | String | Optional Field | The key to use within the secret. |
{{< /table >}}

6
site/content/docs/main/locations.md
View File

@ -26,8 +26,10 @@ This configuration design enables a number of different use cases, including:
All [plugins maintained by the Velero team][5] support this feature.
If you are using a plugin from another provider, please check their documentation to determine if this feature is supported.
- Velero only supports a single set of credentials for `VolumeSnapshotLocations`.
Velero will always use the credentials provided at install time (stored in the `cloud-credentials` secret) for volume snapshots.
- Velero supports multiple credentials for `VolumeSnapshotLocations`, allowing you to specify the credentials to use with any `VolumeSnapshotLocation`.
However, use of this feature requires support within the plugin for the object storage provider you wish to use.
All [plugins maintained by the Velero team][5] support this feature.
If you are using a plugin from another provider, please check their documentation to determine if this feature is supported.
- Volume snapshots are still limited by where your provider allows you to create snapshots. For example, AWS and Azure do not allow you to create a volume snapshot in a different region than where the volume is. If you try to take a Velero backup using a volume snapshot location with a different region than where your cluster's volumes are, the backup will fail.

18
site/content/docs/main/troubleshooting.md
View File

@ -175,9 +175,9 @@ Follow the below troubleshooting steps to confirm that Velero is using the corre
<Output should be your credentials>
```
### Troubleshooting `BackupStorageLocation` credentials
### Troubleshooting `BackupStorageLocation` and `VolumeSnapshotLocation` credentials
Follow the below troubleshooting steps to confirm that Velero is using the correct credentials if using credentials specific to a [`BackupStorageLocation`][10]:
Follow the below troubleshooting steps to confirm that Velero is using the correct credentials if using credentials specific to a [`BackupStorageLocation` or `VolumeSnapshotLocation`][10]:
1. Confirm that the object storage provider plugin being used supports multiple credentials.
If the logs from the Velero deployment contain the error message `"config has invalid keys credentialsFile"`, the version of your object storage plugin does not yet support multiple credentials.
@ -186,7 +186,7 @@ Follow the below troubleshooting steps to confirm that Velero is using the corre
If you are using a plugin from a different provider, please contact them for further advice.
1. Confirm that the secret and key referenced by the `BackupStorageLocation` exists in the Velero namespace and has the correct content:
1. Confirm that the secret and key referenced by the `BackupStorageLocation` or `VolumeSnapshotLocation` exists in the Velero namespace and has the correct content:
```bash
# Determine which secret and key the BackupStorageLocation is using
BSL_SECRET=$(kubectl get backupstoragelocations.velero.io -n velero <bsl-name> -o yaml -o jsonpath={.spec.credential.name})
@ -197,11 +197,21 @@ Follow the below troubleshooting steps to confirm that Velero is using the corre
# Print the content of the secret and ensure it is correct
kubectl -n velero get secret $BSL_SECRET -ojsonpath={.data.$BSL_SECRET_KEY} | base64 --decode
# Determine which secret and key the VolumeSnapshotLocation is using
VSL_SECRET=$(kubectl get volumesnapshotlocations.velero.io -n velero <vsl-name> -o yaml -o jsonpath={.spec.credential.name})
VSL_SECRET_KEY=$(kubectl get volumesnapshotlocations.velero.io -n velero <vsl-name> -o yaml -o jsonpath={.spec.credential.key})
# Confirm that the secret exists
kubectl -n velero get secret $VSL_SECRET
# Print the content of the secret and ensure it is correct
kubectl -n velero get secret $VSL_SECRET -ojsonpath={.data.$VSL_SECRET_KEY} | base64 --decode
```
If the secret can't be found, the secret does not exist within the Velero namespace and must be created.
If no output is produced when printing the contents of the secret, the key within the secret may not exist or may have no content.
Ensure that the key exists within the secret's data by checking the output from `kubectl -n velero describe secret $BSL_SECRET`.
Ensure that the key exists within the secret's data by checking the output from `kubectl -n velero describe secret $BSL_SECRET` or `kubectl -n velero describe secret $VSL_SECRET`.
If it does not exist, follow the instructions for [editing a Kubernetes secret][12] to add the base64 encoded credentials data.