vmware-tanzu
/
velero
mirror of https://github.com/vmware-tanzu/velero.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Support IRSA for data mover pipeline 

Signed-off-by: danfengl <danfengl@vmware.com>
pull/7297/head
danfengl 2024-01-24 06:21:04 +00:00
parent df585053e7
commit 72438b7319
11 changed files with 44 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
test/e2e/Makefile
View File

@ -67,7 +67,8 @@ MIGRATE_FROM_VELERO_CLI ?=
VELERO_NAMESPACE ?= velero
CREDS_FILE ?=
SERVICE_ACCOUNT_NAME ?=
DEFAULT_CLS_SERVICE_ACCOUNT_NAME ?=
STANDBY_CLS_SERVICE_ACCOUNT_NAME ?=
BSL_BUCKET ?=
BSL_PREFIX ?=
BSL_CONFIG ?=
@ -153,8 +154,8 @@ run: ginkgo
-kibishii-directory=$(KIBISHII_DIRECTORY) \
-debug-e2e-test=$(DEBUG_E2E_TEST) \
-velero-server-debug-mode=$(VELERO_SERVER_DEBUG_MODE) \
-default-cluster=$(DEFAULT_CLUSTER) \
-standby-cluster=$(STANDBY_CLUSTER) \
-default-cluster-context=$(DEFAULT_CLUSTER) \
-standby-cluster-context=$(STANDBY_CLUSTER) \
-uploader-type=$(UPLOADER_TYPE) \
-snapshot-move-data=$(SNAPSHOT_MOVE_DATA) \
-data-mover-plugin=$(DATA_MOVER_PLUGIN) \
@ -166,7 +167,8 @@ run: ginkgo
-default-cluster-name=$(DEFAULT_CLUSTER_NAME) \
-standby-cluster-name=$(STANDBY_CLUSTER_NAME) \
-eks-policy-arn=$(EKS_POLICY_ARN) \
-service-account-name=$(SERVICE_ACCOUNT_NAME)
-default-cls-service-account-name=$(DEFAULT_CLS_SERVICE_ACCOUNT_NAME) \
-standby-cls-service-account-name=$(STANDBY_CLS_SERVICE_ACCOUNT_NAME)
build: ginkgo
mkdir -p $(OUTPUT_DIR)

8
test/e2e/README.md
View File

@ -69,8 +69,8 @@ the object-store-provider to be specified.
1. `-debug-e2e-test`: A Switch for enable or disable test data cleaning action.
1. `-garbage-collection-frequency`: frequency of garbage collection. It is a parameter for Velero installation. Optional.
1. `-velero-server-debug-mode`: A switch for enable or disable having debug log of Velero server.
1. `-default-cluster`: Default (source) cluster's kube config context, it's for migration test.
1. `-standby-cluster`: Standby (destination) cluster's kube config context, it's for migration test.
1. `-default-cluster-context`: Default (source) cluster's kube config context, it's for migration test.
1. `-standby-cluster-context`: Standby (destination) cluster's kube config context, it's for migration test.
1. `-uploader-type`: Type of uploader for persistent volume backup.
1. `-snapshot-move-data`: A Switch for taking backup with Velero's data mover, if data-mover-plugin is not provided, using built-in plugin.
1. `-data-mover-plugin`: Customized plugin for data mover.
@ -118,8 +118,8 @@ Below is a mapping between `make` variables to E2E configuration flags.
1. `KIBISHII_DIRECTORY`: `-kibishii-directory`. Optional.
1. `DEBUG_E2E_TEST`: `-debug-e2e-test`. Optional.
1. `VELERO_SERVER_DEBUG_MODE`: `-velero-server-debug-mode`. Optional.
1. `DEFAULT_CLUSTER`: `-default-cluster`. Optional.
1. `STANDBY_CLUSTER`: `-standby-cluster`. Optional.
1. `DEFAULT_CLUSTER`: `-default-cluster-context`. Optional.
1. `STANDBY_CLUSTER`: `-standby-cluster-context`. Optional.
1. `UPLOADER_TYPE`: `-uploader-type`. Optional.
1. `SNAPSHOT_MOVE_DATA`: `-snapshot-move-data`. Optional.
1. `DATA_MOVER_plugin`: `-data-mover-plugin`. Optional.

2
test/e2e/bsl-mgmt/deletion.go
View File

@ -229,7 +229,7 @@ func BslDeletionTest(useVolumeSnapshots bool) {
})
By(fmt.Sprintf("Snapshot of bsl %s should be created in cloud object store", backupLocation_2), func() {
snapshotCheckPoint, err = GetSnapshotCheckPoint(*veleroCfg.ClientToInstallVelero, veleroCfg, 1, bslDeletionTestNs, backupName_2, []string{podName_2})
Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint")
Expect(err).NotTo(HaveOccurred(), "Fail to get snapshot checkpoint")
var BSLCredentials, BSLConfig string
if veleroCfg.CloudProvider == "vsphere" {
BSLCredentials = veleroCfg.AdditionalBSLCredentials

8
test/e2e/e2e_suite_test.go
View File

@ -84,8 +84,8 @@ func init() {
flag.StringVar(&VeleroCfg.Features, "features", "", "comma-separated list of features to enable for this Velero process.")
flag.BoolVar(&VeleroCfg.Debug, "debug-e2e-test", false, "A Switch for enable or disable test data cleaning action.")
flag.StringVar(&VeleroCfg.GCFrequency, "garbage-collection-frequency", "", "frequency of garbage collection.")
flag.StringVar(&VeleroCfg.DefaultClusterContext, "default-cluster", "", "default cluster's kube config context, it's for migration test.")
flag.StringVar(&VeleroCfg.StandbyClusterContext, "standby-cluster", "", "standby cluster's kube config context, it's for migration test.")
flag.StringVar(&VeleroCfg.DefaultClusterContext, "default-cluster-context", "", "default cluster's kube config context, it's for migration test.")
flag.StringVar(&VeleroCfg.StandbyClusterContext, "standby-cluster-context", "", "standby cluster's kube config context, it's for migration test.")
flag.StringVar(&VeleroCfg.UploaderType, "uploader-type", "", "type of uploader for persistent volume backup.")
flag.BoolVar(&VeleroCfg.VeleroServerDebugMode, "velero-server-debug-mode", false, "a switch for enable or disable having debug log of Velero server.")
flag.BoolVar(&VeleroCfg.SnapshotMoveData, "snapshot-move-data", false, "a Switch for taking backup with Velero's data mover, if data-mover-plugin is not provided, using built-in plugin")
@ -98,7 +98,8 @@ func init() {
flag.StringVar(&VeleroCfg.DefaultClusterName, "default-cluster-name", "", "default cluster's name in kube config file, it's for EKS IRSA test.")
flag.StringVar(&VeleroCfg.StandbyClusterName, "standby-cluster-name", "", "standby cluster's name in kube config file, it's for EKS IRSA test.")
flag.StringVar(&VeleroCfg.EKSPolicyARN, "eks-policy-arn", "", "EKS plicy ARN for creating AWS IAM service account.")
flag.StringVar(&VeleroCfg.ServiceAccountName, "service-account-name", "", "service account name.")
flag.StringVar(&VeleroCfg.DefaultCLSServiceAccountName, "default-cls-service-account-name", "", "default cluster service account name.")
flag.StringVar(&VeleroCfg.StandbyCLSServiceAccountName, "standby-cls-service-account-name", "", "standby cluster service account name.")
}
@ -173,6 +174,7 @@ func GetKubeconfigContext() error {
VeleroCfg.DefaultClient = &tcDefault
VeleroCfg.ClientToInstallVelero = VeleroCfg.DefaultClient
VeleroCfg.ClusterToInstallVelero = VeleroCfg.DefaultClusterName
VeleroCfg.ServiceAccountNameToInstall = VeleroCfg.DefaultCLSServiceAccountName
if err != nil {
return err
}

2
test/e2e/migration/migration.go
View File

@ -149,6 +149,7 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version)
OriginVeleroCfg.VeleroCLI = veleroCLI2Version.VeleroCLI
OriginVeleroCfg.ClientToInstallVelero = OriginVeleroCfg.DefaultClient
OriginVeleroCfg.ClusterToInstallVelero = veleroCfg.DefaultClusterName
OriginVeleroCfg.ServiceAccountNameToInstall = veleroCfg.DefaultCLSServiceAccountName
OriginVeleroCfg.UseVolumeSnapshots = useVolumeSnapshots
OriginVeleroCfg.UseNodeAgent = !useVolumeSnapshots
@ -289,6 +290,7 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version)
veleroCfg.ClientToInstallVelero = veleroCfg.StandbyClient
veleroCfg.ClusterToInstallVelero = veleroCfg.StandbyClusterName
veleroCfg.ServiceAccountNameToInstall = veleroCfg.StandbyCLSServiceAccountName
veleroCfg.UseNodeAgent = !useVolumeSnapshots
veleroCfg.UseRestic = false
if veleroCfg.SnapshotMoveData {

4
test/types.go
View File

@ -91,7 +91,9 @@ type VeleroConfig struct {
WithoutDisableInformerCacheParam bool
DisableInformerCache bool
CreateClusterRoleBinding bool
ServiceAccountName string
DefaultCLSServiceAccountName string
StandbyCLSServiceAccountName string
ServiceAccountNameToInstall string
EKSPolicyARN string
}

6
test/util/eks/eks.go
View File

@ -40,11 +40,10 @@ func KubectlDeleteIAMServiceAcount(ctx context.Context, name, namespace, cluster
if strings.Contains(stderr, "NotFound") {
err = nil
}
fmt.Printf("err: %v\n", err)
return err
}
func KubectlCreateIAMServiceAcount(ctx context.Context, name, namespace, policyARN, cluster string) error {
func EksctlCreateIAMServiceAcount(ctx context.Context, name, namespace, policyARN, cluster string) error {
args := []string{"create", "iamserviceaccount", name,
"--namespace", namespace, "--cluster", cluster, "--attach-policy-arn", policyARN,
"--approve", "--override-existing-serviceaccounts"}
@ -55,9 +54,8 @@ func KubectlCreateIAMServiceAcount(ctx context.Context, name, namespace, policyA
cmd := exec.CommandContext(ctx, "eksctl", args...)
fmt.Println(cmd)
stdout, stderr, err := veleroexec.RunCommand(cmd)
fmt.Printf("Output: %v|%v|%v\n", stdout, stderr, err)
if err != nil {
fmt.Printf("err: %v|%v|%v\n", stdout, stderr, err)
fmt.Printf("eksctl return stdout: %v, stderr: %v, err: %v\n", stdout, stderr, err)
return false, nil
}
return true, nil

1
test/util/k8s/clusterrolebinding.go
View File

@ -32,7 +32,6 @@ func KubectlDeleteClusterRoleBinding(ctx context.Context, name string) error {
cmd := exec.CommandContext(ctx, "kubectl", args...)
fmt.Println(cmd)
_, stderr, err := veleroexec.RunCommand(cmd)
fmt.Printf("Ignore error: %v\n", stderr)
if strings.Contains(stderr, "NotFound") {
fmt.Printf("Ignore error: %v\n", stderr)
err = nil

12
test/util/providers/common.go
View File

@ -48,8 +48,7 @@ func ObjectsShouldNotBeInBucket(cloudProvider, cloudCredentialsFile, bslBucket,
var exist bool
fmt.Printf("|| VERIFICATION || - %s %s should not exist in object store %s\n", subPrefix, backupName, bslPrefix)
if cloudCredentialsFile == "" {
fmt.Printf("|| SKIPPED || - Skipping object storebackup checkpoint %s\n", backupName)
return nil
return errors.New(fmt.Sprintf("|| ERROR || - Please provide credential file of cloud %s \n", cloudProvider))
}
for i := 0; i < retryTimes; i++ {
exist, err = IsObjectsInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix)
@ -103,8 +102,7 @@ func DeleteObjectsInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPr
bslPrefix = getFullPrefix(bslPrefix, subPrefix)
fmt.Printf("|| VERIFICATION || - Delete backup %s in storage %s\n", backupName, bslPrefix)
if cloudCredentialsFile == "" {
fmt.Printf("|| SKIPPED || - Skipping snapshots checkpoint %s\n", backupName)
return nil
return errors.New(fmt.Sprintf("|| ERROR || - Please provide credential file of cloud %s \n", cloudProvider))
}
s, err := getProvider(cloudProvider)
if err != nil {
@ -120,8 +118,7 @@ func DeleteObjectsInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPr
func SnapshotsShouldNotExistInCloud(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName string, snapshotCheckPoint SnapshotCheckPoint) error {
fmt.Printf("|| VERIFICATION || - Snapshots should not exist in cloud, backup %s\n", backupName)
if cloudCredentialsFile == "" {
fmt.Printf("|| SKIPPED || - Skipping snapshots checkpoint %s\n", backupName)
return nil
return errors.New(fmt.Sprintf("|| ERROR || - Please provide credential file of cloud %s \n", cloudProvider))
}
snapshotCheckPoint.ExpectCount = 0
err := IsSnapshotExisted(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName, snapshotCheckPoint)
@ -135,8 +132,7 @@ func SnapshotsShouldNotExistInCloud(cloudProvider, cloudCredentialsFile, bslBuck
func SnapshotsShouldBeCreatedInCloud(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName string, snapshotCheckPoint SnapshotCheckPoint) error {
fmt.Printf("|| VERIFICATION || - Snapshots should exist in cloud, backup %s\n", backupName)
if cloudCredentialsFile == "" {
fmt.Printf("|| SKIPPED || - Skipping snapshots checkpoint %s\n", backupName)
return nil
return errors.New(fmt.Sprintf("|| ERROR || - Please provide credential file of cloud %s \n", cloudProvider))
}
err := IsSnapshotExisted(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName, snapshotCheckPoint)
if err != nil {

26
test/util/velero/install.go
View File

@ -76,7 +76,7 @@ func VeleroInstall(ctx context.Context, veleroCfg *VeleroConfig, isStandbyCluste
veleroCfg.CloudProvider = veleroCfg.StandbyClusterCloudProvider
}
if veleroCfg.CloudProvider != "kind" {
fmt.Printf("For cloud platforms, object store plugin provider will be set as cloud provider\n")
fmt.Println("For cloud platforms, object store plugin provider will be set as cloud provider")
// If ObjectStoreProvider is not provided, then using the value same as CloudProvider
if veleroCfg.ObjectStoreProvider == "" {
veleroCfg.ObjectStoreProvider = veleroCfg.CloudProvider
@ -114,32 +114,34 @@ func VeleroInstall(ctx context.Context, veleroCfg *VeleroConfig, isStandbyCluste
return errors.WithMessagef(err, "Failed to get Velero InstallOptions for plugin provider %s", veleroCfg.ObjectStoreProvider)
}
// For AWS IRSA credential test, AWS IAM service account is required, so if ServiceAccountName and EKSPolicyARN
// are both provided, we assume IRSA test is running, otherwise skip this IAM service account creation part.
if veleroCfg.CloudProvider == "aws" && veleroInstallOptions.ServiceAccountName != "" {
if veleroCfg.EKSPolicyARN == "" {
return errors.New("Please provide EKSPolicyARN for IRSA test.")
}
_, err = GetNamespace(ctx, *veleroCfg.ClientToInstallVelero, veleroCfg.VeleroNamespace)
// We should uninstall Velero for a new service account creation.
if !apierrors.IsNotFound(err) {
if err := VeleroUninstall(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace); err != nil {
return errors.Wrapf(err, "Failed to uninstall velero %s", veleroCfg.VeleroNamespace)
}
}
// If velero namespace does not exist, we should create it for service account creation
if err := KubectlCreateNamespace(ctx, veleroCfg.VeleroNamespace); err != nil {
return errors.Wrapf(err, "Failed to create namespace %s to install Velero", veleroCfg.VeleroNamespace)
}
if err := KubectlDeleteClusterRoleBinding(ctx, "velero-cluster-role"); err != nil {
fmt.Println(err)
return errors.Wrapf(err, "Failed to delete clusterrolebinding %s to %s namesapce", "velero-cluster-role", veleroCfg.VeleroNamespace)
return errors.Wrapf(err, "Failed to delete clusterrolebinding %s to %s namespace", "velero-cluster-role", veleroCfg.VeleroNamespace)
}
if err := KubectlCreateClusterRoleBinding(ctx, "velero-cluster-role", "cluster-admin", veleroCfg.VeleroNamespace, veleroInstallOptions.ServiceAccountName); err != nil {
fmt.Println(err)
return errors.Wrapf(err, "Failed to create clusterrolebinding %s to %s namesapce", "velero-cluster-role", veleroCfg.VeleroNamespace)
return errors.Wrapf(err, "Failed to create clusterrolebinding %s to %s namespace", "velero-cluster-role", veleroCfg.VeleroNamespace)
}
if err := KubectlDeleteIAMServiceAcount(ctx, veleroCfg.ServiceAccountName, veleroCfg.VeleroNamespace, veleroCfg.ClusterToInstallVelero); err != nil {
fmt.Println(err)
return errors.Wrapf(err, "Failed to delete service account %s to %s namesapce", veleroCfg.ServiceAccountName, veleroCfg.VeleroNamespace)
if err := KubectlDeleteIAMServiceAcount(ctx, veleroInstallOptions.ServiceAccountName, veleroCfg.VeleroNamespace, veleroCfg.ClusterToInstallVelero); err != nil {
return errors.Wrapf(err, "Failed to delete service account %s to %s namespace", veleroInstallOptions.ServiceAccountName, veleroCfg.VeleroNamespace)
}
time.Sleep(10 * time.Second)
if err := KubectlCreateIAMServiceAcount(ctx, veleroCfg.ServiceAccountName, veleroCfg.VeleroNamespace, veleroCfg.EKSPolicyARN, veleroCfg.ClusterToInstallVelero); err != nil {
fmt.Println(err)
return errors.Wrapf(err, "Failed to create service account %s to %s namesapce", veleroCfg.ServiceAccountName, veleroCfg.VeleroNamespace)
if err := EksctlCreateIAMServiceAcount(ctx, veleroInstallOptions.ServiceAccountName, veleroCfg.VeleroNamespace, veleroCfg.EKSPolicyARN, veleroCfg.ClusterToInstallVelero); err != nil {
return errors.Wrapf(err, "Failed to create service account %s to %s namespace", veleroInstallOptions.ServiceAccountName, veleroCfg.VeleroNamespace)
}
}
err = installVeleroServer(ctx, veleroCfg.VeleroCLI, veleroCfg.CloudProvider, &installOptions{

6
test/util/velero/velero_utils.go
View File

@ -174,7 +174,7 @@ func getPluginsByVersion(version, cloudProvider, objectStoreProvider, feature st
func getProviderVeleroInstallOptions(veleroCfg *VeleroConfig,
plugins []string) (*cliinstall.Options, error) {
if veleroCfg.CloudCredentialsFile == "" && veleroCfg.ServiceAccountName == "" {
if veleroCfg.CloudCredentialsFile == "" && veleroCfg.ServiceAccountNameToInstall == "" {
return nil, errors.Errorf("No credentials were supplied to use for E2E tests")
}
@ -191,8 +191,8 @@ func getProviderVeleroInstallOptions(veleroCfg *VeleroConfig,
io.SecretFile = realPath
}
if veleroCfg.ServiceAccountName != "" {
io.ServiceAccountName = veleroCfg.ServiceAccountName
if veleroCfg.ServiceAccountNameToInstall != "" {
io.ServiceAccountName = veleroCfg.ServiceAccountNameToInstall
io.NoSecret = true
}