diff --git a/hack/build-image/Dockerfile b/hack/build-image/Dockerfile index f4767d705..904b6dabe 100644 --- a/hack/build-image/Dockerfile +++ b/hack/build-image/Dockerfile @@ -34,7 +34,7 @@ RUN go install sigs.k8s.io/controller-tools/cmd/controller-gen@v0.16.5 # get goimports (the revision is pinned so we don't indiscriminately update, but the particular commit # is not important) -RUN go install golang.org/x/tools/cmd/goimports@11e9d9cc0042e6bd10337d4d2c3e5d9295508e7d +RUN go install golang.org/x/tools/cmd/goimports@v0.33.0 # get protoc compiler and golang plugin WORKDIR /root diff --git a/test/Makefile b/test/Makefile index 1f8f3a0aa..10fd75da6 100644 --- a/test/Makefile +++ b/test/Makefile @@ -119,6 +119,8 @@ VELERO_SERVER_DEBUG_MODE ?= false ITEM_BLOCK_WORKER_COUNT ?= 1 +WORKER_OS ?= linux + # Parameters to run migration tests along with all other E2E tests, and both of them should # be provided or left them all empty to skip migration tests with no influence to other # E2E tests. @@ -221,7 +223,8 @@ run-e2e: ginkgo --standby-cls-service-account-name=$(STANDBY_CLS_SERVICE_ACCOUNT_NAME) \ --kibishii-directory=$(KIBISHII_DIRECTORY) \ --disable-informer-cache=$(DISABLE_INFORMER_CACHE) \ - --image-registry-proxy=$(IMAGE_REGISTRY_PROXY) + --image-registry-proxy=$(IMAGE_REGISTRY_PROXY) \ + --worker-os=$(WORKER_OS) .PHONY: run-perf run-perf: ginkgo diff --git a/test/e2e/README.md b/test/e2e/README.md index 7e7dcf04d..6734c9bef 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -79,6 +79,7 @@ These configuration parameters are expected as values to the following command l 1. `--debug-velero-pod-restart`: A switch for debugging velero pod restart. 1. `--fail-fast`: A switch for for failing fast on meeting error. 1. `--has-vsphere-plugin`: A switch to indicate whether the Velero vSphere plugin is installed for vSphere environment. +1. `--worker-os`: A switch to indicate the workload should be ran on windows or linux OS. These configurations or parameters are used to generate install options for Velero for each test suite. @@ -131,6 +132,7 @@ Below is a mapping between `make` variables to E2E configuration flags. 1. `DEBUG_VELERO_POD_RESTART`: `-debug-velero-pod-restart`. Optional. 1. `FAIL_FAST`: `--fail-fast`. Optional. 1. `HAS_VSPHERE_PLUGIN`: `--has-vsphere-plugin`. Optional. +1. `WORKER_OS`: `--worker-os`. Optional. diff --git a/test/e2e/backup/backup.go b/test/e2e/backup/backup.go index 91678b5b2..b0e7c06da 100644 --- a/test/e2e/backup/backup.go +++ b/test/e2e/backup/backup.go @@ -140,7 +140,15 @@ func BackupRestoreTest(backupRestoreTestConfig BackupRestoreTestConfig) { veleroCfg.ProvideSnapshotsVolumeParam = provideSnapshotVolumesParmInBackup // Set DefaultVolumesToFsBackup to false since DefaultVolumesToFsBackup was set to true during installation - Expect(RunKibishiiTests(veleroCfg, backupName, restoreName, "", kibishiiNamespace, useVolumeSnapshots, false)).To(Succeed(), + Expect(RunKibishiiTests( + veleroCfg, + backupName, + restoreName, + "", + kibishiiNamespace, + useVolumeSnapshots, + false, + )).To(Succeed(), "Failed to successfully backup and restore Kibishii namespace") }) @@ -212,7 +220,17 @@ func BackupRestoreTest(backupRestoreTestConfig BackupRestoreTestConfig) { } veleroCfg.ProvideSnapshotsVolumeParam = !provideSnapshotVolumesParmInBackup workloadNS := kibishiiNamespace + bsl - Expect(RunKibishiiTests(veleroCfg, backupName, restoreName, bsl, workloadNS, useVolumeSnapshots, !useVolumeSnapshots)).To(Succeed(), + Expect( + RunKibishiiTests( + veleroCfg, + backupName, + restoreName, + bsl, + workloadNS, + useVolumeSnapshots, + !useVolumeSnapshots, + ), + ).To(Succeed(), "Failed to successfully backup and restore Kibishii namespace using BSL %s", bsl) } }) diff --git a/test/e2e/backups/deletion.go b/test/e2e/backups/deletion.go index 263567d33..b24325b5c 100644 --- a/test/e2e/backups/deletion.go +++ b/test/e2e/backups/deletion.go @@ -125,6 +125,7 @@ func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupLoc kibishiiDirectory, DefaultKibishiiData, veleroCfg.ImageRegistryProxy, + veleroCfg.WorkerOS, ); err != nil { return errors.Wrapf(err, "Failed to install and prepare data for kibishii %s", ns) } diff --git a/test/e2e/backups/ttl.go b/test/e2e/backups/ttl.go index 83df75eb0..e65506fa3 100644 --- a/test/e2e/backups/ttl.go +++ b/test/e2e/backups/ttl.go @@ -110,6 +110,7 @@ func TTLTest() { veleroCfg.KibishiiDirectory, DefaultKibishiiData, veleroCfg.ImageRegistryProxy, + veleroCfg.WorkerOS, )).To(Succeed()) }) diff --git a/test/e2e/basic/backup-volume-info/base.go b/test/e2e/basic/backup-volume-info/base.go index c3993a34b..04c962fe4 100644 --- a/test/e2e/basic/backup-volume-info/base.go +++ b/test/e2e/basic/backup-volume-info/base.go @@ -138,8 +138,16 @@ func (v *BackupVolumeInfo) CreateResources() error { // Hitting issue https://github.com/vmware-tanzu/velero/issues/7388 // So populate data only to some of pods, leave other pods empty to verify empty PV datamover if i%2 == 0 { - Expect(CreateFileToPod(v.Ctx, createNSName, pod.Name, DefaultContainerName, vols[i].Name, - fmt.Sprintf("file-%s", pod.Name), CreateFileContent(createNSName, pod.Name, vols[i].Name))).To(Succeed()) + Expect(CreateFileToPod( + v.Ctx, + createNSName, + pod.Name, + DefaultContainerName, + vols[i].Name, + fmt.Sprintf("file-%s", pod.Name), + CreateFileContent(createNSName, pod.Name, vols[i].Name), + WorkerOSLinux, + )).To(Succeed()) } } } diff --git a/test/e2e/basic/namespace-mapping.go b/test/e2e/basic/namespace-mapping.go index a2736b3f1..293bb1796 100644 --- a/test/e2e/basic/namespace-mapping.go +++ b/test/e2e/basic/namespace-mapping.go @@ -101,6 +101,7 @@ func (n *NamespaceMapping) CreateResources() error { n.VeleroCfg.KibishiiDirectory, n.kibishiiData, n.VeleroCfg.ImageRegistryProxy, + n.VeleroCfg.WorkerOS, )).To(Succeed()) }) } @@ -111,8 +112,14 @@ func (n *NamespaceMapping) Verify() error { for index, ns := range n.MappedNamespaceList { n.kibishiiData.Levels = len(*n.NSIncluded) + index By(fmt.Sprintf("Verify workload %s after restore ", ns), func() { - Expect(KibishiiVerifyAfterRestore(n.Client, ns, - n.Ctx, n.kibishiiData, "")).To(Succeed(), "Fail to verify workload after restore") + Expect(KibishiiVerifyAfterRestore( + n.Client, + ns, + n.Ctx, + n.kibishiiData, + "", + n.VeleroCfg.WorkerOS, + )).To(Succeed(), "Fail to verify workload after restore") }) } for _, ns := range *n.NSIncluded { diff --git a/test/e2e/basic/resources-check/namespaces.go b/test/e2e/basic/resources-check/namespaces.go index 0b21822fa..a9f59738b 100644 --- a/test/e2e/basic/resources-check/namespaces.go +++ b/test/e2e/basic/resources-check/namespaces.go @@ -31,7 +31,7 @@ import ( type MultiNSBackup struct { TestCase - IsScalTest bool + IsScaleTest bool NSExcluded *[]string TimeoutDuration time.Duration } @@ -43,7 +43,7 @@ func (m *MultiNSBackup) Init() error { m.RestoreName = "restore-" + m.CaseBaseName m.NSExcluded = &[]string{} - if m.IsScalTest { + if m.IsScaleTest { m.NamespacesTotal = 2500 m.TimeoutDuration = time.Hour * 2 m.TestMsg = &TestMSG{ diff --git a/test/e2e/basic/resources-check/resources_check.go b/test/e2e/basic/resources-check/resources_check.go index 9db5198bc..880063657 100644 --- a/test/e2e/basic/resources-check/resources_check.go +++ b/test/e2e/basic/resources-check/resources_check.go @@ -39,7 +39,7 @@ import ( func GetResourcesCheckTestCases() []VeleroBackupRestoreTest { return []VeleroBackupRestoreTest{ &NSAnnotationCase{}, - &MultiNSBackup{IsScalTest: false}, + &MultiNSBackup{IsScaleTest: false}, &RBACCase{}, } } diff --git a/test/e2e/bsl-mgmt/deletion.go b/test/e2e/bsl-mgmt/deletion.go index 832f9a6fd..9b79c6a8e 100644 --- a/test/e2e/bsl-mgmt/deletion.go +++ b/test/e2e/bsl-mgmt/deletion.go @@ -162,6 +162,7 @@ func BslDeletionTest(useVolumeSnapshots bool) { veleroCfg.KibishiiDirectory, DefaultKibishiiData, veleroCfg.ImageRegistryProxy, + veleroCfg.WorkerOS, )).To(Succeed()) }) diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 4aa76eb32..fb2b594d0 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -356,6 +356,12 @@ func init() { "", "The image registry proxy, e.g. when the DockerHub access limitation is reached, can use available proxy to replace. Default is nil.", ) + flag.StringVar( + &test.VeleroCfg.WorkerOS, + "worker-os", + "linux", + "test k8s worker node OS version, should be either linux or windows.", + ) } // Add label [SkipVanillaZfs]: @@ -621,12 +627,12 @@ var _ = Describe( var _ = Describe( "Backup resources should follow the specific order in schedule", - Label("PVBackup", "OptIn"), + Label("PVBackup", "OptIn", "FSB"), OptInPVBackupTest, ) var _ = Describe( "Backup resources should follow the specific order in schedule", - Label("PVBackup", "OptOut"), + Label("PVBackup", "OptOut", "FSB"), OptOutPVBackupTest, ) diff --git a/test/e2e/migration/migration.go b/test/e2e/migration/migration.go index 6c39f0734..32119846d 100644 --- a/test/e2e/migration/migration.go +++ b/test/e2e/migration/migration.go @@ -23,9 +23,11 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "golang.org/x/mod/semver" "github.com/vmware-tanzu/velero/test" framework "github.com/vmware-tanzu/velero/test/e2e/test" + "github.com/vmware-tanzu/velero/test/util/common" util "github.com/vmware-tanzu/velero/test/util/csi" k8sutil "github.com/vmware-tanzu/velero/test/util/k8s" "github.com/vmware-tanzu/velero/test/util/kibishii" @@ -160,6 +162,10 @@ func (m *migrationE2E) Backup() error { version, err := veleroutil.GetVeleroVersion(m.Ctx, OriginVeleroCfg.VeleroCLI, true) Expect(err).To(Succeed(), "Fail to get Velero version") OriginVeleroCfg.VeleroVersion = version + if OriginVeleroCfg.WorkerOS == common.WorkerOSWindows && + (version != "main" && semver.Compare(version, "v1.16") < 0) { + Skip(fmt.Sprintf("Velero CLI version %s doesn't support Windows migration test.", version)) + } if OriginVeleroCfg.SnapshotMoveData { OriginVeleroCfg.UseNodeAgent = true @@ -197,6 +203,7 @@ func (m *migrationE2E) Backup() error { OriginVeleroCfg.KibishiiDirectory, &m.kibishiiData, OriginVeleroCfg.ImageRegistryProxy, + OriginVeleroCfg.WorkerOS, )).To(Succeed()) }) @@ -401,6 +408,7 @@ func (m *migrationE2E) Verify() error { m.Ctx, &m.kibishiiData, "", + m.VeleroCfg.WorkerOS, )).To(Succeed(), "Fail to verify workload after restore") }) @@ -413,56 +421,66 @@ func (m *migrationE2E) Clean() error { }) By("Clean resource on standby cluster.", func() { + defer func() { + By("Switch to default KubeConfig context", func() { + k8sutil.KubectlConfigUseContext( + m.Ctx, + m.VeleroCfg.DefaultClusterContext, + ) + }) + }() + Expect(k8sutil.KubectlConfigUseContext( m.Ctx, m.VeleroCfg.StandbyClusterContext)).To(Succeed()) m.VeleroCfg.ClientToInstallVelero = m.VeleroCfg.StandbyClient m.VeleroCfg.ClusterToInstallVelero = m.VeleroCfg.StandbyClusterName By("Delete StorageClasses created by E2E") - Expect( - k8sutil.DeleteStorageClass( - m.Ctx, - *m.VeleroCfg.ClientToInstallVelero, - test.StorageClassName, - ), - ).To(Succeed()) - Expect( - k8sutil.DeleteStorageClass( - m.Ctx, - *m.VeleroCfg.ClientToInstallVelero, - test.StorageClassName2, - ), - ).To(Succeed()) + if err := k8sutil.DeleteStorageClass( + m.Ctx, + *m.VeleroCfg.ClientToInstallVelero, + test.StorageClassName, + ); err != nil { + fmt.Println("Fail to delete StorageClass1: ", err) + return + } + + if err := k8sutil.DeleteStorageClass( + m.Ctx, + *m.VeleroCfg.ClientToInstallVelero, + test.StorageClassName2, + ); err != nil { + fmt.Println("Fail to delete StorageClass2: ", err) + return + } if strings.EqualFold(m.VeleroCfg.Features, test.FeatureCSI) && m.VeleroCfg.UseVolumeSnapshots { By("Delete VolumeSnapshotClass created by E2E") - Expect( - k8sutil.KubectlDeleteByFile( - m.Ctx, - fmt.Sprintf("../testdata/volume-snapshot-class/%s.yaml", - m.VeleroCfg.StandbyClusterCloudProvider), - ), - ).To(Succeed()) + if err := k8sutil.KubectlDeleteByFile( + m.Ctx, + fmt.Sprintf("../testdata/volume-snapshot-class/%s.yaml", + m.VeleroCfg.StandbyClusterCloudProvider), + ); err != nil { + fmt.Println("Fail to delete VolumeSnapshotClass: ", err) + return + } } - Expect(veleroutil.VeleroUninstall(m.Ctx, m.VeleroCfg)).To(Succeed()) + if err := veleroutil.VeleroUninstall(m.Ctx, m.VeleroCfg); err != nil { + fmt.Println("Fail to uninstall Velero: ", err) + return + } - Expect( - k8sutil.DeleteNamespace( - m.Ctx, - *m.VeleroCfg.StandbyClient, - m.CaseBaseName, - true, - ), - ).To(Succeed()) - }) - - By("Switch to default KubeConfig context", func() { - Expect(k8sutil.KubectlConfigUseContext( + if err := k8sutil.DeleteNamespace( m.Ctx, - m.VeleroCfg.DefaultClusterContext, - )).To(Succeed()) + *m.VeleroCfg.StandbyClient, + m.CaseBaseName, + true, + ); err != nil { + fmt.Println("Fail to delete the workload namespace: ", err) + return + } }) return nil diff --git a/test/e2e/pv-backup/pv-backup-filter.go b/test/e2e/pv-backup/pv-backup-filter.go index e71af4717..6dc94b881 100644 --- a/test/e2e/pv-backup/pv-backup-filter.go +++ b/test/e2e/pv-backup/pv-backup-filter.go @@ -115,8 +115,16 @@ func (p *PVBackupFiltering) CreateResources() error { Expect(WaitForPods(p.Ctx, p.Client, ns, p.podsList[index])).To(Succeed()) for i, pod := range p.podsList[index] { for j := range p.volumesList[i] { - Expect(CreateFileToPod(p.Ctx, ns, pod, pod, p.volumesList[i][j], - FILE_NAME, CreateFileContent(ns, pod, p.volumesList[i][j]))).To(Succeed()) + Expect(CreateFileToPod( + p.Ctx, + ns, + pod, + pod, + p.volumesList[i][j], + FILE_NAME, + CreateFileContent(ns, pod, p.volumesList[i][j]), + WorkerOSLinux, + )).To(Succeed()) } } }) @@ -142,21 +150,45 @@ func (p *PVBackupFiltering) Verify() error { if j%2 == 0 { if p.annotation == OPT_IN_ANN { By(fmt.Sprintf("File should exists in PV %s of pod %s under namespace %s\n", p.volumesList[i][j], p.podsList[k][i], ns), func() { - Expect(fileExist(p.Ctx, ns, p.podsList[k][i], p.volumesList[i][j])).To(Succeed(), "File not exist as expect") + Expect(fileExist( + p.Ctx, + ns, + p.podsList[k][i], + p.volumesList[i][j], + p.VeleroCfg.WorkerOS, + )).To(Succeed(), "File not exist as expect") }) } else { By(fmt.Sprintf("File should not exist in PV %s of pod %s under namespace %s\n", p.volumesList[i][j], p.podsList[k][i], ns), func() { - Expect(fileNotExist(p.Ctx, ns, p.podsList[k][i], p.volumesList[i][j])).To(Succeed(), "File exists, not as expect") + Expect(fileNotExist( + p.Ctx, + ns, + p.podsList[k][i], + p.volumesList[i][j], + p.VeleroCfg.WorkerOS, + )).To(Succeed(), "File exists, not as expect") }) } } else { if p.annotation == OPT_OUT_ANN { By(fmt.Sprintf("File should exists in PV %s of pod %s under namespace %s\n", p.volumesList[i][j], p.podsList[k][i], ns), func() { - Expect(fileExist(p.Ctx, ns, p.podsList[k][i], p.volumesList[i][j])).To(Succeed(), "File not exist as expect") + Expect(fileExist( + p.Ctx, + ns, + p.podsList[k][i], + p.volumesList[i][j], + p.VeleroCfg.WorkerOS, + )).To(Succeed(), "File not exist as expect") }) } else { By(fmt.Sprintf("File should not exist in PV %s of pod %s under namespace %s\n", p.volumesList[i][j], p.podsList[k][i], ns), func() { - Expect(fileNotExist(p.Ctx, ns, p.podsList[k][i], p.volumesList[i][j])).To(Succeed(), "File exists, not as expect") + Expect(fileNotExist( + p.Ctx, + ns, + p.podsList[k][i], + p.volumesList[i][j], + p.VeleroCfg.WorkerOS, + )).To(Succeed(), "File exists, not as expect") }) } } @@ -168,8 +200,14 @@ func (p *PVBackupFiltering) Verify() error { return nil } -func fileExist(ctx context.Context, namespace, podName, volume string) error { - c, _, err := ReadFileFromPodVolume(ctx, namespace, podName, podName, volume, FILE_NAME) +func fileExist( + ctx context.Context, + namespace string, + podName string, + volume string, + workerOS string, +) error { + c, _, err := ReadFileFromPodVolume(ctx, namespace, podName, podName, volume, FILE_NAME, workerOS) if err != nil { return errors.Wrap(err, fmt.Sprintf("Fail to read file %s from volume %s of pod %s in %s ", FILE_NAME, volume, podName, namespace)) @@ -183,8 +221,14 @@ func fileExist(ctx context.Context, namespace, podName, volume string) error { FILE_NAME, volume, podName, namespace)) } } -func fileNotExist(ctx context.Context, namespace, podName, volume string) error { - _, _, err := ReadFileFromPodVolume(ctx, namespace, podName, podName, volume, FILE_NAME) +func fileNotExist( + ctx context.Context, + namespace string, + podName string, + volume string, + workerOS string, +) error { + _, _, err := ReadFileFromPodVolume(ctx, namespace, podName, podName, volume, FILE_NAME, workerOS) if err != nil { return nil } else { diff --git a/test/e2e/resourcepolicies/resource_policies.go b/test/e2e/resourcepolicies/resource_policies.go index 117a3ef1b..ef5e5afe8 100644 --- a/test/e2e/resourcepolicies/resource_policies.go +++ b/test/e2e/resourcepolicies/resource_policies.go @@ -28,6 +28,7 @@ import ( . "github.com/vmware-tanzu/velero/test" . "github.com/vmware-tanzu/velero/test/e2e/test" + "github.com/vmware-tanzu/velero/test/util/common" . "github.com/vmware-tanzu/velero/test/util/k8s" ) @@ -151,7 +152,15 @@ func (r *ResourcePoliciesCase) Verify() error { if vol.Name != volName { continue } - content, _, err := ReadFileFromPodVolume(r.Ctx, ns, pod.Name, "container-busybox", vol.Name, FileName) + content, _, err := ReadFileFromPodVolume( + r.Ctx, + ns, + pod.Name, + "container-busybox", + vol.Name, + FileName, + r.VeleroCfg.WorkerOS, + ) if i%2 == 0 { Expect(err).To(HaveOccurred(), "Expected file not found") // File should not exist } else { @@ -231,7 +240,16 @@ func (r *ResourcePoliciesCase) writeDataIntoPods(namespace, volName string) erro if vol.Name != volName { continue } - err := CreateFileToPod(r.Ctx, namespace, pod.Name, "container-busybox", vol.Name, FileName, fmt.Sprintf("ns-%s pod-%s volume-%s", namespace, pod.Name, vol.Name)) + err := CreateFileToPod( + r.Ctx, + namespace, + pod.Name, + "container-busybox", + vol.Name, + FileName, + fmt.Sprintf("ns-%s pod-%s volume-%s", namespace, pod.Name, vol.Name), + common.WorkerOSLinux, + ) if err != nil { return errors.Wrap(err, fmt.Sprintf("failed to create file into pod %s in namespace: %q", pod.Name, namespace)) } diff --git a/test/e2e/scale/multiple_namespaces.go b/test/e2e/scale/multiple_namespaces.go index d63880c6b..6bc4e4f9a 100644 --- a/test/e2e/scale/multiple_namespaces.go +++ b/test/e2e/scale/multiple_namespaces.go @@ -21,4 +21,4 @@ import ( . "github.com/vmware-tanzu/velero/test/e2e/test" ) -var MultiNSBackupRestore func() = TestFunc(&basic.MultiNSBackup{IsScalTest: true}) +var MultiNSBackupRestore func() = TestFunc(&basic.MultiNSBackup{IsScaleTest: true}) diff --git a/test/e2e/upgrade/upgrade.go b/test/e2e/upgrade/upgrade.go index 5ad868547..4b4a9c824 100644 --- a/test/e2e/upgrade/upgrade.go +++ b/test/e2e/upgrade/upgrade.go @@ -126,15 +126,6 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC tmpCfgForOldVeleroInstall.UpgradeFromVeleroVersion = veleroCLI2Version.VeleroVersion tmpCfgForOldVeleroInstall.VeleroCLI = veleroCLI2Version.VeleroCLI - // CLI under version v1.14.x - if veleroCLI2Version.VeleroVersion < "v1.15" { - tmpCfgForOldVeleroInstall.BackupRepoConfigMap = "" - fmt.Printf( - "CLI version %s is lower than v1.15. Set BackupRepoConfigMap to empty, because it's not supported", - veleroCLI2Version.VeleroVersion, - ) - } - tmpCfgForOldVeleroInstall, err = SetImagesToDefaultValues( tmpCfgForOldVeleroInstall, veleroCLI2Version.VeleroVersion, @@ -176,6 +167,7 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC tmpCfg.KibishiiDirectory, DefaultKibishiiData, tmpCfg.ImageRegistryProxy, + veleroCfg.WorkerOS, )).To(Succeed()) }) @@ -269,8 +261,14 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC }) By(fmt.Sprintf("Verify workload %s after restore ", upgradeNamespace), func() { - Expect(KibishiiVerifyAfterRestore(*veleroCfg.ClientToInstallVelero, upgradeNamespace, - oneHourTimeout, DefaultKibishiiData, "")).To(Succeed(), "Fail to verify workload after restore") + Expect(KibishiiVerifyAfterRestore( + *veleroCfg.ClientToInstallVelero, + upgradeNamespace, + oneHourTimeout, + DefaultKibishiiData, + "", + veleroCfg.WorkerOS, + )).To(Succeed(), "Fail to verify workload after restore") }) }) }) diff --git a/test/types.go b/test/types.go index c326d4bd7..bc54948e4 100644 --- a/test/types.go +++ b/test/types.go @@ -129,6 +129,7 @@ type VeleroConfig struct { FailFast bool HasVspherePlugin bool ImageRegistryProxy string + WorkerOS string } type VeleroCfgInPerf struct { diff --git a/test/util/common/common.go b/test/util/common/common.go index 3b130f209..4b0041e44 100644 --- a/test/util/common/common.go +++ b/test/util/common/common.go @@ -11,6 +11,11 @@ import ( "os/exec" ) +const ( + WorkerOSLinux string = "linux" + WorkerOSWindows string = "windows" +) + type OsCommandLine struct { Cmd string Args []string diff --git a/test/util/k8s/common.go b/test/util/k8s/common.go index b31525e74..c60450c33 100644 --- a/test/util/k8s/common.go +++ b/test/util/k8s/common.go @@ -322,32 +322,80 @@ func WriteRandomDataToFileInPod(ctx context.Context, namespace, podName, contain return cmd.Run() } -func CreateFileToPod(ctx context.Context, namespace, podName, containerName, volume, filename, content string) error { +func CreateFileToPod( + ctx context.Context, + namespace string, + podName string, + containerName string, + volume string, + filename string, + content string, + workerOS string, +) error { + filePath := fmt.Sprintf("/%s/%s", volume, filename) + shell := "/bin/sh" + shellParameter := "-c" + + if workerOS == common.WorkerOSWindows { + filePath = fmt.Sprintf("C:\\%s\\%s", volume, filename) + shell = "cmd" + shellParameter = "/c" + } + arg := []string{"exec", "-n", namespace, "-c", containerName, podName, - "--", "/bin/sh", "-c", fmt.Sprintf("echo ns-%s pod-%s volume-%s > /%s/%s", namespace, podName, volume, volume, filename)} + "--", shell, shellParameter, fmt.Sprintf("echo ns-%s pod-%s volume-%s > %s", namespace, podName, volume, filePath)} + cmd := exec.CommandContext(ctx, "kubectl", arg...) fmt.Printf("Kubectl exec cmd =%v\n", cmd) return cmd.Run() } -func FileExistInPV(ctx context.Context, namespace, podName, containerName, volume, filename string) (bool, error) { - stdout, stderr, err := ReadFileFromPodVolume(ctx, namespace, podName, containerName, volume, filename) +func FileExistInPV( + ctx context.Context, + namespace string, + podName string, + containerName string, + volume string, + filename string, + workerOS string, +) (bool, error) { + stdout, stderr, err := ReadFileFromPodVolume(ctx, namespace, podName, containerName, volume, filename, workerOS) output := fmt.Sprintf("%s:%s", stdout, stderr) - if strings.Contains(output, fmt.Sprintf("/%s/%s: No such file or directory", volume, filename)) { - return false, nil - } else { - if err == nil { - return true, nil - } else { - return false, errors.Wrap(err, fmt.Sprintf("Fail to read file %s from volume %s of pod %s in %s", - filename, volume, podName, namespace)) + + if workerOS == common.WorkerOSWindows { + if strings.Contains(output, "The system cannot find the file specified") { + return false, nil } } + + if strings.Contains(output, fmt.Sprintf("/%s/%s: No such file or directory", volume, filename)) { + return false, nil + } + + if err == nil { + return true, nil + } else { + return false, errors.Wrap(err, fmt.Sprintf("Fail to read file %s from volume %s of pod %s in %s", + filename, volume, podName, namespace)) + } } -func ReadFileFromPodVolume(ctx context.Context, namespace, podName, containerName, volume, filename string) (string, string, error) { +func ReadFileFromPodVolume( + ctx context.Context, + namespace string, + podName string, + containerName string, + volume string, + filename string, + workerOS string, +) (string, string, error) { arg := []string{"exec", "-n", namespace, "-c", containerName, podName, "--", "cat", fmt.Sprintf("/%s/%s", volume, filename)} + if workerOS == common.WorkerOSWindows { + arg = []string{"exec", "-n", namespace, "-c", containerName, podName, + "--", "cmd", "/c", fmt.Sprintf("type C:\\%s\\%s", volume, filename)} + } + cmd := exec.CommandContext(ctx, "kubectl", arg...) fmt.Printf("Kubectl exec cmd =%v\n", cmd) stdout, stderr, err := veleroexec.RunCommand(cmd) diff --git a/test/util/kibishii/kibishii_utils.go b/test/util/kibishii/kibishii_utils.go index 7883d9b73..1897e2b16 100644 --- a/test/util/kibishii/kibishii_utils.go +++ b/test/util/kibishii/kibishii_utils.go @@ -36,6 +36,7 @@ import ( veleroexec "github.com/vmware-tanzu/velero/pkg/util/exec" . "github.com/vmware-tanzu/velero/test" + "github.com/vmware-tanzu/velero/test/util/common" . "github.com/vmware-tanzu/velero/test/util/k8s" . "github.com/vmware-tanzu/velero/test/util/providers" . "github.com/vmware-tanzu/velero/test/util/velero" @@ -117,6 +118,7 @@ func RunKibishiiTests( kibishiiDirectory, DefaultKibishiiData, veleroCfg.ImageRegistryProxy, + veleroCfg.WorkerOS, ); err != nil { return errors.Wrapf(err, "Failed to install and prepare data for kibishii %s", kibishiiNamespace) } @@ -206,14 +208,22 @@ func RunKibishiiTests( // Modify PV data right after backup. If PV's reclaim policy is retain, PV will be restored with the origin resource config fileName := "file-" + kibishiiNamespace fileBaseContent := fileName - fmt.Printf("Re-poulate volume %s\n", time.Now().Format("2006-01-02 15:04:05")) + fmt.Printf("Re-populate volume %s\n", time.Now().Format("2006-01-02 15:04:05")) for _, pod := range KibishiiPodNameList { // To ensure Kibishii verification result is accurate ClearKibishiiData(oneHourTimeout, kibishiiNamespace, pod, "kibishii", "data") CreateFileContent := fileBaseContent + pod - err := CreateFileToPod(oneHourTimeout, kibishiiNamespace, pod, "kibishii", "data", - fileName, CreateFileContent) + err := CreateFileToPod( + oneHourTimeout, + kibishiiNamespace, + pod, + "kibishii", + "data", + fileName, + CreateFileContent, + veleroCfg.WorkerOS, + ) if err != nil { return errors.Wrapf(err, "failed to create file %s", fileName) } @@ -269,7 +279,7 @@ func RunKibishiiTests( } fmt.Printf("KibishiiVerifyAfterRestore %s\n", time.Now().Format("2006-01-02 15:04:05")) - if err := KibishiiVerifyAfterRestore(client, kibishiiNamespace, oneHourTimeout, DefaultKibishiiData, fileName); err != nil { + if err := KibishiiVerifyAfterRestore(client, kibishiiNamespace, oneHourTimeout, DefaultKibishiiData, fileName, veleroCfg.WorkerOS); err != nil { return errors.Wrapf(err, "Error verifying kibishii after restore") } @@ -279,12 +289,13 @@ func RunKibishiiTests( func installKibishii( ctx context.Context, - namespace string, + namespace, cloudPlatform, veleroFeatures, kibishiiDirectory string, workerReplicas int, imageRegistryProxy string, + workerOS string, ) error { if strings.EqualFold(cloudPlatform, Azure) && strings.EqualFold(veleroFeatures, FeatureCSI) { @@ -295,15 +306,28 @@ func installKibishii( cloudPlatform = AwsCSI } + targetKustomizeDir := path.Join(kibishiiDirectory, cloudPlatform) + if strings.EqualFold(cloudPlatform, Vsphere) { if strings.HasPrefix(kibishiiDirectory, "https://") { return errors.New("vSphere needs to download the Kibishii repository first because it needs to inject some image patch file to work.") } + // TODO: blackpiglet debug + fmt.Printf("targetKustomizeDir %s, workerOS: %s, WorkerOSWindows: %s.\n", targetKustomizeDir, workerOS, common.WorkerOSWindows) + + if workerOS == common.WorkerOSWindows { + targetKustomizeDir += "-windows" + + // TODO: blackpiglet debug + fmt.Printf("targetKustomizeDir for windows %s\n", targetKustomizeDir) + } + fmt.Printf("The installed Kibishii Kustomize package directory is %s.\n", targetKustomizeDir) + kibishiiImage := readBaseKibishiiImage(path.Join(kibishiiDirectory, "base", "kibishii.yaml")) if err := generateKibishiiImagePatch( path.Join(imageRegistryProxy, kibishiiImage), - path.Join(kibishiiDirectory, cloudPlatform, "worker-image-patch.yaml"), + path.Join(targetKustomizeDir, "worker-image-patch.yaml"), ); err != nil { return nil } @@ -311,22 +335,39 @@ func installKibishii( jumpPadImage := readBaseJumpPadImage(path.Join(kibishiiDirectory, "base", "jump-pad.yaml")) if err := generateJumpPadPatch( path.Join(imageRegistryProxy, jumpPadImage), - path.Join(kibishiiDirectory, cloudPlatform, "jump-pad-image-patch.yaml"), + path.Join(targetKustomizeDir, "jump-pad-image-patch.yaml"), ); err != nil { return nil } } // We use kustomize to generate YAML for Kibishii from the checked-in yaml directories + kibishiiInstallCmd := exec.CommandContext(ctx, "kubectl", "apply", "-n", namespace, "-k", - path.Join(kibishiiDirectory, cloudPlatform), "--timeout=90s") + targetKustomizeDir, "--timeout=90s") _, stderr, err := veleroexec.RunCommand(kibishiiInstallCmd) fmt.Printf("Install Kibishii cmd: %s\n", kibishiiInstallCmd) if err != nil { return errors.Wrapf(err, "failed to install kibishii, stderr=%s", stderr) } - labelNamespaceCmd := exec.CommandContext(ctx, "kubectl", "label", "namespace", namespace, "pod-security.kubernetes.io/enforce=baseline", "pod-security.kubernetes.io/enforce-version=latest", "--overwrite=true") + psa_enforce_policy := "baseline" + if workerOS == common.WorkerOSWindows { + // Windows container volume mount root directory's permission only allow privileged user write. + // https://github.com/kubernetes/kubernetes/issues/131341 + psa_enforce_policy = "privileged" + } + + labelNamespaceCmd := exec.CommandContext( + ctx, + "kubectl", + "label", + "namespace", + namespace, + fmt.Sprintf("pod-security.kubernetes.io/enforce=%s", psa_enforce_policy), + "pod-security.kubernetes.io/enforce-version=latest", + "--overwrite=true", + ) _, stderr, err = veleroexec.RunCommand(labelNamespaceCmd) fmt.Printf("Label namespace with PSA policy: %s\n", labelNamespaceCmd) if err != nil { @@ -558,7 +599,7 @@ func waitForKibishiiPods(ctx context.Context, client TestClient, kibishiiNamespa ) } -func KibishiiGenerateData(oneHourTimeout context.Context, kibishiiNamespace string, kibishiiData *KibishiiData) error { +func kibishiiGenerateData(oneHourTimeout context.Context, kibishiiNamespace string, kibishiiData *KibishiiData) error { fmt.Printf("generateData %s\n", time.Now().Format("2006-01-02 15:04:05")) if err := generateData(oneHourTimeout, kibishiiNamespace, kibishiiData); err != nil { return errors.Wrap(err, "Failed to generate data") @@ -577,6 +618,7 @@ func KibishiiPrepareBeforeBackup( kibishiiDirectory string, kibishiiData *KibishiiData, imageRegistryProxy string, + workerOS string, ) error { fmt.Printf("installKibishii %s\n", time.Now().Format("2006-01-02 15:04:05")) serviceAccountName := "default" @@ -599,6 +641,7 @@ func KibishiiPrepareBeforeBackup( kibishiiDirectory, kibishiiData.ExpectedNodes, imageRegistryProxy, + workerOS, ); err != nil { return errors.Wrap(err, "Failed to install Kibishii workload") } @@ -611,12 +654,18 @@ func KibishiiPrepareBeforeBackup( if kibishiiData == nil { kibishiiData = DefaultKibishiiData } - KibishiiGenerateData(oneHourTimeout, kibishiiNamespace, kibishiiData) + kibishiiGenerateData(oneHourTimeout, kibishiiNamespace, kibishiiData) return nil } -func KibishiiVerifyAfterRestore(client TestClient, kibishiiNamespace string, oneHourTimeout context.Context, - kibishiiData *KibishiiData, incrementalFileName string) error { +func KibishiiVerifyAfterRestore( + client TestClient, + kibishiiNamespace string, + oneHourTimeout context.Context, + kibishiiData *KibishiiData, + incrementalFileName string, + workerOS string, +) error { if kibishiiData == nil { kibishiiData = DefaultKibishiiData } @@ -628,7 +677,7 @@ func KibishiiVerifyAfterRestore(client TestClient, kibishiiNamespace string, one } if incrementalFileName != "" { for _, pod := range KibishiiPodNameList { - exist, err := FileExistInPV(oneHourTimeout, kibishiiNamespace, pod, "kibishii", "data", incrementalFileName) + exist, err := FileExistInPV(oneHourTimeout, kibishiiNamespace, pod, "kibishii", "data", incrementalFileName, workerOS) if err != nil { return errors.Wrapf(err, "fail to get file %s", incrementalFileName) } diff --git a/test/util/velero/install.go b/test/util/velero/install.go index e77a83db2..1cdaaa4f5 100644 --- a/test/util/velero/install.go +++ b/test/util/velero/install.go @@ -28,6 +28,7 @@ import ( "github.com/pkg/errors" "golang.org/x/exp/slices" + "golang.org/x/mod/semver" appsv1api "k8s.io/api/apps/v1" corev1api "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -40,6 +41,7 @@ import ( "github.com/vmware-tanzu/velero/pkg/cmd/cli/install" velerexec "github.com/vmware-tanzu/velero/pkg/util/exec" "github.com/vmware-tanzu/velero/test" + common "github.com/vmware-tanzu/velero/test/util/common" eksutil "github.com/vmware-tanzu/velero/test/util/eks" "github.com/vmware-tanzu/velero/test/util/k8s" ) @@ -51,6 +53,7 @@ type installOptions struct { RestoreHelperImage string VeleroServerDebugMode bool WithoutDisableInformerCacheParam bool + WorkerOS string } func VeleroInstall(ctx context.Context, veleroCfg *test.VeleroConfig, isStandbyCluster bool) error { @@ -174,13 +177,14 @@ func VeleroInstall(ctx context.Context, veleroCfg *test.VeleroConfig, isStandbyC if err := installVeleroServer( ctx, veleroCfg.VeleroCLI, - veleroCfg.CloudProvider, + veleroCfg.VeleroVersion, &installOptions{ Options: veleroInstallOptions, RegistryCredentialFile: veleroCfg.RegistryCredentialFile, RestoreHelperImage: veleroCfg.RestoreHelperImage, VeleroServerDebugMode: veleroCfg.VeleroServerDebugMode, WithoutDisableInformerCacheParam: veleroCfg.WithoutDisableInformerCacheParam, + WorkerOS: veleroCfg.WorkerOS, }, ); err != nil { time.Sleep(1 * time.Minute) @@ -282,7 +286,12 @@ func cleanVSpherePluginConfig(c clientset.Interface, ns, secretName, configMapNa return nil } -func installVeleroServer(ctx context.Context, cli, cloudProvider string, options *installOptions) error { +func installVeleroServer( + ctx context.Context, + cli string, + version string, + options *installOptions, +) error { args := []string{"install"} namespace := "velero" if len(options.Namespace) > 0 { @@ -295,6 +304,16 @@ func installVeleroServer(ctx context.Context, cli, cloudProvider string, options if options.UseNodeAgent { args = append(args, "--use-node-agent") } + + // TODO: need to consider align options.UseNodeAgentWindows usage + // with options.UseNodeAgent + // Only version after v1.16.0 support windows node agent. + if options.WorkerOS == common.WorkerOSWindows && + (semver.Compare(version, "v1.16") >= 0 || version == "main") { + fmt.Println("Install node-agent-windows. The Velero version is ", version) + args = append(args, "--use-node-agent-windows") + } + if options.DefaultVolumesToFsBackup { args = append(args, "--default-volumes-to-fs-backup") } @@ -391,7 +410,10 @@ func installVeleroServer(ctx context.Context, cli, cloudProvider string, options args = append(args, fmt.Sprintf("--item-block-worker-count=%d", options.ItemBlockWorkerCount)) } - if options.BackupRepoConfigMap != "" { + // Only version no older than v1.15 support --backup-repository-configmap. + if options.BackupRepoConfigMap != "" && + (semver.Compare(version, "v1.15") >= 0 || version == "main") { + fmt.Println("Associate backup repository ConfigMap. The Velero version is ", version) args = append(args, fmt.Sprintf("--backup-repository-configmap=%s", options.BackupRepoConfigMap)) } @@ -399,7 +421,7 @@ func installVeleroServer(ctx context.Context, cli, cloudProvider string, options return err } - return waitVeleroReady(ctx, namespace, options.UseNodeAgent) + return waitVeleroReady(ctx, namespace, options.UseNodeAgent, options.UseNodeAgentWindows) } func createVeleroResources(ctx context.Context, cli, namespace string, args []string, options *installOptions) error { @@ -617,7 +639,7 @@ func toUnstructured(res any) (unstructured.Unstructured, error) { return un, err } -func waitVeleroReady(ctx context.Context, namespace string, useNodeAgent bool) error { +func waitVeleroReady(ctx context.Context, namespace string, useNodeAgent bool, useNodeAgentWindows bool) error { fmt.Println("Waiting for Velero deployment to be ready.") // when doing upgrade by the "kubectl apply" the command "kubectl wait --for=condition=available deployment/velero -n velero --timeout=600s" returns directly // use "rollout status" instead to avoid this. For more detail information, refer to https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#complete-deployment @@ -649,6 +671,28 @@ func waitVeleroReady(ctx context.Context, namespace string, useNodeAgent bool) e } } + if useNodeAgentWindows { + fmt.Println("Waiting for node-agent-windows DaemonSet to be ready.") + err := wait.PollUntilContextTimeout(context.Background(), 5*time.Second, 1*time.Minute, true, func(ctx context.Context) (bool, error) { + stdout, stderr, err := velerexec.RunCommand(exec.CommandContext(ctx, "kubectl", "get", "DaemonSet/node-agent-windows", + "-o", "json", "-n", namespace)) + if err != nil { + return false, errors.Wrapf(err, "failed to get the node-agent-windows DaemonSet, stdout=%s, stderr=%s", stdout, stderr) + } + ds := &appsv1api.DaemonSet{} + if err = json.Unmarshal([]byte(stdout), ds); err != nil { + return false, errors.Wrapf(err, "failed to unmarshal the node-agent-windows DaemonSet") + } + if ds.Status.DesiredNumberScheduled == ds.Status.NumberAvailable { + return true, nil + } + return false, nil + }) + if err != nil { + return errors.Wrap(err, "fail to wait for the node-agent-windows ready") + } + } + fmt.Printf("Velero is installed and ready to be tested in the %s namespace! ⛵ \n", namespace) return nil } diff --git a/test/util/velero/velero_utils.go b/test/util/velero/velero_utils.go index bc7f76645..1fe362e7a 100644 --- a/test/util/velero/velero_utils.go +++ b/test/util/velero/velero_utils.go @@ -1393,7 +1393,7 @@ func VeleroUpgrade(ctx context.Context, veleroCfg VeleroConfig) error { return errors.Wrap(err, "Fail to update node agent") } } - return waitVeleroReady(ctx, veleroCfg.VeleroNamespace, veleroCfg.UseNodeAgent) + return waitVeleroReady(ctx, veleroCfg.VeleroNamespace, veleroCfg.UseNodeAgent, veleroCfg.UseNodeAgentWindows) } func ApplyCRDs(ctx context.Context, veleroCLI string) ([]string, error) {