Use labels instead of regex to filter E2E test cases.
Signed-off-by: Xun Jiang <blackpigletbruce@gmail.com>pull/8038/head
Xun Jiang
parent
442cc76417
commit
e862b976a4
|
|
@ -67,16 +67,12 @@ jobs:
|
|||
- 1.27.10
|
||||
- 1.28.6
|
||||
- 1.29.1
|
||||
focus:
|
||||
# tests to focus on, use `|` to concatenate multiple regexes to run on the same job
|
||||
# ordered according to e2e_suite_test.go order
|
||||
- Basic\]\[ClusterResource
|
||||
- ResourceFiltering
|
||||
- ResourceModifier|Backups|PrivilegesMgmt\]\[SSR
|
||||
- Schedule\]\[OrderedResources
|
||||
- NamespaceMapping\]\[Single\]\[Restic|NamespaceMapping\]\[Multiple\]\[Restic
|
||||
- Basic\]\[Nodeport
|
||||
- Basic\]\[StorageClass
|
||||
labels:
|
||||
# labels are used to filter running E2E cases
|
||||
- Basic && (ClusterResource || NodePort || StorageClass)
|
||||
- ResourceFiltering && !Restic
|
||||
- ResourceModifier || (Backups && BackupsSync) || PrivilegesMgmt || OrderedResources
|
||||
- (NamespaceMapping && Single && Restic) || (NamespaceMapping && Multiple && Restic)
|
||||
fail-fast: false
|
||||
steps:
|
||||
- name: Set up Go
|
||||
|
|
@ -128,13 +124,18 @@ jobs:
|
|||
curl -LO https://dl.k8s.io/release/v${{ matrix.k8s }}/bin/linux/amd64/kubectl
|
||||
sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
|
||||
|
||||
GOPATH=~/go CLOUD_PROVIDER=kind \
|
||||
OBJECT_STORE_PROVIDER=aws BSL_CONFIG=region=minio,s3ForcePathStyle="true",s3Url=http://$(hostname -i):9000 \
|
||||
CREDS_FILE=/tmp/credential BSL_BUCKET=bucket \
|
||||
ADDITIONAL_OBJECT_STORE_PROVIDER=aws ADDITIONAL_BSL_CONFIG=region=minio,s3ForcePathStyle="true",s3Url=http://$(hostname -i):9000 \
|
||||
ADDITIONAL_CREDS_FILE=/tmp/credential ADDITIONAL_BSL_BUCKET=additional-bucket \
|
||||
GINKGO_FOCUS='${{ matrix.focus }}' VELERO_IMAGE=velero:pr-test \
|
||||
GINKGO_SKIP='SKIP_KIND|pv-backup|Restic|Snapshot|LongTime' \
|
||||
GOPATH=~/go \
|
||||
CLOUD_PROVIDER=kind \
|
||||
OBJECT_STORE_PROVIDER=aws \
|
||||
BSL_CONFIG=region=minio,s3ForcePathStyle="true",s3Url=http://$(hostname -i):9000 \
|
||||
CREDS_FILE=/tmp/credential \
|
||||
BSL_BUCKET=bucket \
|
||||
ADDITIONAL_OBJECT_STORE_PROVIDER=aws \
|
||||
ADDITIONAL_BSL_CONFIG=region=minio,s3ForcePathStyle="true",s3Url=http://$(hostname -i):9000 \
|
||||
ADDITIONAL_CREDS_FILE=/tmp/credential \
|
||||
ADDITIONAL_BSL_BUCKET=additional-bucket \
|
||||
VELERO_IMAGE=velero:pr-test \
|
||||
GINKGO_LABELS="${{ matrix.labels }}" \
|
||||
make -C test/ run-e2e
|
||||
timeout-minutes: 30
|
||||
- name: Upload debug bundle
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ jobs:
|
|||
with:
|
||||
# ignore the config/.../crd.go file as it's generated binary data that is edited elswhere.
|
||||
skip: .git,*.png,*.jpg,*.woff,*.ttf,*.gif,*.ico,./config/crd/v1beta1/crds/crds.go,./config/crd/v1/crds/crds.go,./config/crd/v2alpha1/crds/crds.go,./go.sum,./LICENSE
|
||||
ignore_words_list: iam,aks,ist,bridget,ue,shouldnot,atleast,notin,sme
|
||||
ignore_words_list: iam,aks,ist,bridget,ue,shouldnot,atleast,notin,sme,optin
|
||||
check_filenames: true
|
||||
check_hidden: true
|
||||
|
||||
|
|
|
|||
135
test/Makefile
135
test/Makefile
|
|
@ -47,10 +47,9 @@ TOOLS_BIN_DIR := $(TOOLS_DIR)/$(BIN_DIR)
|
|||
GINKGO := $(GOBIN)/ginkgo
|
||||
KUSTOMIZE := $(TOOLS_BIN_DIR)/kustomize
|
||||
OUTPUT_DIR := _output/$(GOOS)/$(GOARCH)/bin
|
||||
GINKGO_FOCUS ?=
|
||||
GINKGO_SKIP ?=
|
||||
SKIP_STR := $(foreach var, $(subst ., ,$(GINKGO_SKIP)),-skip "$(var)")
|
||||
FOCUS_STR := $(foreach var, $(subst ., ,$(GINKGO_FOCUS)),-focus "$(var)")
|
||||
# Please reference to this document for Ginkgo label spec format.
|
||||
# https://onsi.github.io/ginkgo/#spec-labels
|
||||
GINKGO_LABELS ?=
|
||||
VELERO_CLI ?=$$(pwd)/../_output/bin/$(GOOS)/$(GOARCH)/velero
|
||||
VELERO_IMAGE ?= velero/velero:main
|
||||
PLUGINS ?=
|
||||
|
|
@ -129,26 +128,26 @@ VELERO_POD_CPU_REQUEST ?= 2
|
|||
VELERO_POD_MEM_REQUEST ?= 2Gi
|
||||
POD_VOLUME_OPERATION_TIMEOUT ?= 6h
|
||||
|
||||
COMMON_ARGS := -velerocli=$(VELERO_CLI) \
|
||||
-velero-image=$(VELERO_IMAGE) \
|
||||
-plugins=$(PLUGINS) \
|
||||
-velero-version=$(VERSION) \
|
||||
-restore-helper-image=$(RESTORE_HELPER_IMAGE) \
|
||||
-velero-namespace=$(VELERO_NAMESPACE) \
|
||||
-credentials-file=$(CREDS_FILE) \
|
||||
-bucket=$(BSL_BUCKET) \
|
||||
-prefix=$(BSL_PREFIX) \
|
||||
-bsl-config=$(BSL_CONFIG) \
|
||||
-vsl-config=$(VSL_CONFIG) \
|
||||
-cloud-provider=$(CLOUD_PROVIDER) \
|
||||
-object-store-provider="$(OBJECT_STORE_PROVIDER)" \
|
||||
-features=$(FEATURES) \
|
||||
-install-velero=$(INSTALL_VELERO) \
|
||||
-registry-credential-file=$(REGISTRY_CREDENTIAL_FILE) \
|
||||
-debug-e2e-test=$(DEBUG_E2E_TEST) \
|
||||
-velero-server-debug-mode=$(VELERO_SERVER_DEBUG_MODE) \
|
||||
-uploader-type=$(UPLOADER_TYPE) \
|
||||
-debug-velero-pod-restart=$(DEBUG_VELERO_POD_RESTART)
|
||||
COMMON_ARGS := --velerocli=$(VELERO_CLI) \
|
||||
--velero-image=$(VELERO_IMAGE) \
|
||||
--plugins=$(PLUGINS) \
|
||||
--velero-version=$(VERSION) \
|
||||
--restore-helper-image=$(RESTORE_HELPER_IMAGE) \
|
||||
--velero-namespace=$(VELERO_NAMESPACE) \
|
||||
--credentials-file=$(CREDS_FILE) \
|
||||
--bucket=$(BSL_BUCKET) \
|
||||
--prefix=$(BSL_PREFIX) \
|
||||
--bsl-config=$(BSL_CONFIG) \
|
||||
--vsl-config=$(VSL_CONFIG) \
|
||||
--cloud-provider=$(CLOUD_PROVIDER) \
|
||||
--object-store-provider="$(OBJECT_STORE_PROVIDER)" \
|
||||
--features=$(FEATURES) \
|
||||
--install-velero=$(INSTALL_VELERO) \
|
||||
--registry-credential-file=$(REGISTRY_CREDENTIAL_FILE) \
|
||||
--debug-e2e-test=$(DEBUG_E2E_TEST) \
|
||||
--velero-server-debug-mode=$(VELERO_SERVER_DEBUG_MODE) \
|
||||
--uploader-type=$(UPLOADER_TYPE) \
|
||||
--debug-velero-pod-restart=$(DEBUG_VELERO_POD_RESTART)
|
||||
|
||||
# Make sure ginkgo is in $GOBIN
|
||||
.PHONY:ginkgo
|
||||
|
|
@ -166,31 +165,36 @@ run-e2e: ginkgo
|
|||
(echo "Bucket to store the backups from E2E tests is required, please re-run with BSL_BUCKET=<BucketName>"; exit 1 )
|
||||
@[ "${CLOUD_PROVIDER}" ] && echo "Using cloud provider ${CLOUD_PROVIDER}" || \
|
||||
(echo "Cloud provider for target cloud/plugin provider is required, please rerun with CLOUD_PROVIDER=<aws,azure,kind,vsphere>"; exit 1)
|
||||
@$(GINKGO) run -v $(FOCUS_STR) $(SKIP_STR) --junit-report report.xml ./e2e -- $(COMMON_ARGS) \
|
||||
-upgrade-from-velero-cli=$(UPGRADE_FROM_VELERO_CLI) \
|
||||
-upgrade-from-velero-version=$(UPGRADE_FROM_VELERO_VERSION) \
|
||||
-migrate-from-velero-cli=$(MIGRATE_FROM_VELERO_CLI) \
|
||||
-migrate-from-velero-version=$(MIGRATE_FROM_VELERO_VERSION) \
|
||||
-additional-bsl-plugins=$(ADDITIONAL_BSL_PLUGINS) \
|
||||
-additional-bsl-object-store-provider="$(ADDITIONAL_OBJECT_STORE_PROVIDER)" \
|
||||
-additional-bsl-credentials-file=$(ADDITIONAL_CREDS_FILE) \
|
||||
-additional-bsl-bucket=$(ADDITIONAL_BSL_BUCKET) \
|
||||
-additional-bsl-prefix=$(ADDITIONAL_BSL_PREFIX) \
|
||||
-additional-bsl-config=$(ADDITIONAL_BSL_CONFIG) \
|
||||
-default-cluster-context=$(DEFAULT_CLUSTER) \
|
||||
-standby-cluster-context=$(STANDBY_CLUSTER) \
|
||||
-snapshot-move-data=$(SNAPSHOT_MOVE_DATA) \
|
||||
-data-mover-plugin=$(DATA_MOVER_PLUGIN) \
|
||||
-standby-cluster-cloud-provider=$(STANDBY_CLUSTER_CLOUD_PROVIDER) \
|
||||
-standby-cluster-plugins=$(STANDBY_CLUSTER_PLUGINS) \
|
||||
-standby-cluster-object-store-provider=$(STANDBY_CLUSTER_OBJECT_STORE_PROVIDER) \
|
||||
-default-cluster-name=$(DEFAULT_CLUSTER_NAME) \
|
||||
-standby-cluster-name=$(STANDBY_CLUSTER_NAME) \
|
||||
-eks-policy-arn=$(EKS_POLICY_ARN) \
|
||||
-default-cls-service-account-name=$(DEFAULT_CLS_SERVICE_ACCOUNT_NAME) \
|
||||
-standby-cls-service-account-name=$(STANDBY_CLS_SERVICE_ACCOUNT_NAME)
|
||||
-kibishii-directory=$(KIBISHII_DIRECTORY) \
|
||||
-disable-informer-cache=$(DISABLE_INFORMER_CACHE)
|
||||
@$(GINKGO) run \
|
||||
-v \
|
||||
--junit-report report.xml \
|
||||
--label-filter="$(GINKGO_LABELS)" \
|
||||
./e2e \
|
||||
-- $(COMMON_ARGS) \
|
||||
--upgrade-from-velero-cli=$(UPGRADE_FROM_VELERO_CLI) \
|
||||
--upgrade-from-velero-version=$(UPGRADE_FROM_VELERO_VERSION) \
|
||||
--migrate-from-velero-cli=$(MIGRATE_FROM_VELERO_CLI) \
|
||||
--migrate-from-velero-version=$(MIGRATE_FROM_VELERO_VERSION) \
|
||||
--additional-bsl-plugins=$(ADDITIONAL_BSL_PLUGINS) \
|
||||
--additional-bsl-object-store-provider="$(ADDITIONAL_OBJECT_STORE_PROVIDER)" \
|
||||
--additional-bsl-credentials-file=$(ADDITIONAL_CREDS_FILE) \
|
||||
--additional-bsl-bucket=$(ADDITIONAL_BSL_BUCKET) \
|
||||
--additional-bsl-prefix=$(ADDITIONAL_BSL_PREFIX) \
|
||||
--additional-bsl-config=$(ADDITIONAL_BSL_CONFIG) \
|
||||
--default-cluster-context=$(DEFAULT_CLUSTER) \
|
||||
--standby-cluster-context=$(STANDBY_CLUSTER) \
|
||||
--snapshot-move-data=$(SNAPSHOT_MOVE_DATA) \
|
||||
--data-mover-plugin=$(DATA_MOVER_PLUGIN) \
|
||||
--standby-cluster-cloud-provider=$(STANDBY_CLUSTER_CLOUD_PROVIDER) \
|
||||
--standby-cluster-plugins=$(STANDBY_CLUSTER_PLUGINS) \
|
||||
--standby-cluster-object-store-provider=$(STANDBY_CLUSTER_OBJECT_STORE_PROVIDER) \
|
||||
--default-cluster-name=$(DEFAULT_CLUSTER_NAME) \
|
||||
--standby-cluster-name=$(STANDBY_CLUSTER_NAME) \
|
||||
--eks-policy-arn=$(EKS_POLICY_ARN) \
|
||||
--default-cls-service-account-name=$(DEFAULT_CLS_SERVICE_ACCOUNT_NAME) \
|
||||
--standby-cls-service-account-name=$(STANDBY_CLS_SERVICE_ACCOUNT_NAME)
|
||||
--kibishii-directory=$(KIBISHII_DIRECTORY) \
|
||||
--disable-informer-cache=$(DISABLE_INFORMER_CACHE)
|
||||
|
||||
.PHONY: run-perf
|
||||
run-perf: ginkgo
|
||||
|
|
@ -200,20 +204,25 @@ run-perf: ginkgo
|
|||
(echo "Bucket to store the backups from E2E tests is required, please re-run with BSL_BUCKET=<BucketName>"; exit 1 )
|
||||
@[ "${CLOUD_PROVIDER}" ] && echo "Using cloud provider ${CLOUD_PROVIDER}" || \
|
||||
(echo "Cloud provider for target cloud/plugin provider is required, please rerun with CLOUD_PROVIDER=<aws,azure,kind,vsphere>"; exit 1)
|
||||
@$(GINKGO) run -v $(FOCUS_STR) $(SKIP_STR) --junit-report report.xml ./perf -- $(COMMON_ARGS) \
|
||||
-nfs-server-path=$(NFS_SERVER_PATH) \
|
||||
-test-case-describe=$(TEST_CASE_DESCRIBE) \
|
||||
-backup-for-restore=$(BACKUP_FOR_RESTORE) \
|
||||
-delete-cluster-resource=$(Delete_Cluster_Resource) \
|
||||
-node-agent-pod-cpu-limit=$(NODE_AGENT_POD_CPU_LIMIT) \
|
||||
-node-agent-pod-mem-limit=$(NODE_AGENT_POD_MEM_LIMIT) \
|
||||
-node-agent-pod-cpu-request=$(NODE_AGENT_POD_CPU_REQUEST) \
|
||||
-node-agent-pod-mem-request=$(NODE_AGENT_POD_MEM_REQUEST) \
|
||||
-velero-pod-cpu-limit=$(VELERO_POD_CPU_LIMIT) \
|
||||
-velero-pod-mem-limit=$(VELERO_POD_MEM_LIMIT) \
|
||||
-velero-pod-cpu-request=$(VELERO_POD_CPU_REQUEST) \
|
||||
-velero-pod-mem-request=$(VELERO_POD_MEM_REQUEST) \
|
||||
-pod-volume-operation-timeout=$(POD_VOLUME_OPERATION_TIMEOUT)
|
||||
@$(GINKGO) run \
|
||||
-v \
|
||||
--junit-report report.xml \
|
||||
--label-filter="$(GINKGO_LABELS)" \
|
||||
./perf \
|
||||
-- $(COMMON_ARGS) \
|
||||
--nfs-server-path=$(NFS_SERVER_PATH) \
|
||||
--test-case-describe=$(TEST_CASE_DESCRIBE) \
|
||||
--backup-for-restore=$(BACKUP_FOR_RESTORE) \
|
||||
--delete-cluster-resource=$(Delete_Cluster_Resource) \
|
||||
--node-agent-pod-cpu-limit=$(NODE_AGENT_POD_CPU_LIMIT) \
|
||||
--node-agent-pod-mem-limit=$(NODE_AGENT_POD_MEM_LIMIT) \
|
||||
--node-agent-pod-cpu-request=$(NODE_AGENT_POD_CPU_REQUEST) \
|
||||
--node-agent-pod-mem-request=$(NODE_AGENT_POD_MEM_REQUEST) \
|
||||
--velero-pod-cpu-limit=$(VELERO_POD_CPU_LIMIT) \
|
||||
--velero-pod-mem-limit=$(VELERO_POD_MEM_LIMIT) \
|
||||
--velero-pod-cpu-request=$(VELERO_POD_CPU_REQUEST) \
|
||||
--velero-pod-mem-request=$(VELERO_POD_MEM_REQUEST) \
|
||||
--pod-volume-operation-timeout=$(POD_VOLUME_OPERATION_TIMEOUT)
|
||||
|
||||
build: ginkgo
|
||||
mkdir -p $(OUTPUT_DIR)
|
||||
|
|
|
|||
|
|
@ -157,9 +157,9 @@ Basic examples:
|
|||
BSL_CONFIG="resourceGroup=$AZURE_BACKUP_RESOURCE_GROUP,storageAccount=$AZURE_STORAGE_ACCOUNT_ID,subscriptionId=$AZURE_BACKUP_SUBSCRIPTION_ID" BSL_BUCKET=<BUCKET_FOR_E2E_TEST_BACKUP> CREDS_FILE=/path/to/azure-creds CLOUD_PROVIDER=azure make test-e2e
|
||||
```
|
||||
Please refer to `velero-plugin-for-microsoft-azure` documentation for instruction to [set up permissions for Velero](https://github.com/vmware-tanzu/velero-plugin-for-microsoft-azure#set-permissions-for-velero) and to [set up azure storage account and blob container](https://github.com/vmware-tanzu/velero-plugin-for-microsoft-azure#setup-azure-storage-account-and-blob-container)
|
||||
1. Run Ginko-focused Restore Multi-API Groups tests using Minio as the backup storage location:
|
||||
1. Run Multi-API group and version tests using MinIO as the backup storage location:
|
||||
```bash
|
||||
BSL_CONFIG="region=minio,s3ForcePathStyle=\"true\",s3Url=<ip address>:9000" BSL_PREFIX=<prefix> BSL_BUCKET=<bucket> CREDS_FILE=<absolute path to minio credentials file> CLOUD_PROVIDER=kind OBJECT_STORE_PROVIDER=aws VELERO_NAMESPACE="velero" GINKGO_FOCUS="API group versions" make test-e2e
|
||||
BSL_CONFIG="region=minio,s3ForcePathStyle=\"true\",s3Url=<ip address>:9000" BSL_PREFIX=<prefix> BSL_BUCKET=<bucket> CREDS_FILE=<absolute path to minio credentials file> CLOUD_PROVIDER=kind OBJECT_STORE_PROVIDER=aws VELERO_NAMESPACE="velero" GINKGO_LABELS="APIGroup && APIVersion" make test-e2e
|
||||
```
|
||||
1. Run Velero tests in a kind cluster with AWS (or Minio) as the storage provider and use Microsoft Azure as the storage provider for an additional Backup Storage Location:
|
||||
```bash
|
||||
|
|
@ -208,60 +208,66 @@ Migration examples:
|
|||
```
|
||||
|
||||
|
||||
1. Datamover tests:
|
||||
1. Data mover tests:
|
||||
|
||||
The example shows all essential `make` variables for a Datamover test which is migrate from a AKS cluster to a EKS cluster.
|
||||
The example shows all essential `make` variables for a data mover test which is migrate from a AKS cluster to a EKS cluster.
|
||||
|
||||
Note: STANDBY_CLUSTER_CLOUD_PROVIDER and STANDBY_CLUSTER_OBJECT_STORE_PROVIDER is essential here, it is for identify plugins to be installed on target cluster, since DEFAULT cluster's provider is different from STANDBY cluster, plugins are different as well.
|
||||
```bash
|
||||
CLOUD_PROVIDER=azure \
|
||||
DEFAULT_CLUSTER=<AKS_CLUSTER_KUBECONFIG_CONTEXT> \
|
||||
STANDBY_CLUSTER=<EKS_CLUSTER_KUBECONFIG_CONTEXT> \
|
||||
FEATURES=EnableCSI \
|
||||
OBJECT_STORE_PROVIDER=aws \
|
||||
CREDS_FILE=<AWS_CREDENTIAL_FILE> \
|
||||
BSL_CONFIG=region=<AWS_REGION> \
|
||||
BSL_BUCKET=<S3_BUCKET> \
|
||||
BSL_PREFIX=<S3_BUCKET_PREFIC> \
|
||||
VSL_CONFIG=region=<AWS_REGION> \
|
||||
SNAPSHOT_MOVE_DATA=true \
|
||||
STANDBY_CLUSTER_CLOUD_PROVIDER=aws \
|
||||
STANDBY_CLUSTER_OBJECT_STORE_PROVIDER=aws \
|
||||
GINKGO_LABELS="Migration" \
|
||||
make test-e2e
|
||||
CLOUD_PROVIDER=azure \
|
||||
DEFAULT_CLUSTER=<AKS_CLUSTER_KUBECONFIG_CONTEXT> \
|
||||
STANDBY_CLUSTER=<EKS_CLUSTER_KUBECONFIG_CONTEXT> \
|
||||
FEATURES=EnableCSI \
|
||||
OBJECT_STORE_PROVIDER=aws \
|
||||
CREDS_FILE=<AWS_CREDENTIAL_FILE> \
|
||||
BSL_CONFIG=region=<AWS_REGION> \
|
||||
BSL_BUCKET=<S3_BUCKET> \
|
||||
BSL_PREFIX=<S3_BUCKET_PREFIC> \
|
||||
VSL_CONFIG=region=<AWS_REGION> \
|
||||
SNAPSHOT_MOVE_DATA=true \
|
||||
STANDBY_CLUSTER_CLOUD_PROVIDER=aws \
|
||||
STANDBY_CLUSTER_OBJECT_STORE_PROVIDER=aws \
|
||||
GINKGO_FOCUS=Migration
|
||||
```
|
||||
|
||||
## Filtering tests
|
||||
|
||||
Velero E2E tests uses [Ginkgo](https://onsi.github.io/ginkgo/) testing framework which allows a subset of the tests to be run using the [`-focus` and `-skip`](https://onsi.github.io/ginkgo/#focused-specs) flags to ginkgo.
|
||||
In release-1.15, Velero bumps the [Ginkgo](https://onsi.github.io/ginkgo/) version to [v2](https://onsi.github.io/ginkgo/MIGRATING_TO_V2).
|
||||
Velero E2E start to use [labels](https://onsi.github.io/ginkgo/#spec-labels) to filter cases instead of [`-focus` and `-skip`](https://onsi.github.io/ginkgo/#focused-specs) parameters.
|
||||
|
||||
For filtering tests, using `make` variables `GINKGO_FOCUS` and `GINKGO_SKIP` :
|
||||
1. `GINKGO_FOCUS`: Dot-separated list of labels to be included for Ginkgo description-based filtering. Optional. The `-focus` flag is passed to ginkgo using the `GINKGO_FOCUS` `make` variable. This can be used to focus on specific tests.
|
||||
1. `GINKGO_SKIP`: Dot-separated list of labels to be excluded for Ginkgo description-based filtering.Optional. The `-skip ` flag is passed to ginkgo using the `GINKGO_SKIP` `make` variable. This can be used to skip specific tests.
|
||||
Both `make run-e2e` and `make run-perf` CLI support using parameter `GINKGO_LABELS` to filter test cases.
|
||||
|
||||
`GINKGO_LABELS` is interpreted into `ginkgo run` CLI's parameter [`--label-filter`](https://onsi.github.io/ginkgo/#spec-labels).
|
||||
|
||||
|
||||
|
||||
`GINKGO_FOCUS`/`GINKGO_SKIP` can be interpreted into multiple `-focus`/`-skip ` describe in [Description-Based Filtering](https://onsi.github.io/ginkgo/#description-based-filtering:~:text=Description%2DBased%20Filtering) by dot-separated format for test execution management please refer to examples below.:
|
||||
|
||||
|
||||
For example, E2E tests can be run with specific cases to be included and/or excluded using the commands below:
|
||||
### Examples
|
||||
E2E tests can be run with specific cases to be included and/or excluded using the commands below:
|
||||
1. Run Velero tests with specific cases to be included:
|
||||
```bash
|
||||
GINKGO_LABELS="Basic && Restic" \
|
||||
CLOUD_PROVIDER=aws \
|
||||
BSL_BUCKET=example-bucket \
|
||||
CREDS_FILE=/path/to/aws-creds \
|
||||
make test-e2e \
|
||||
GINKGO_FOCUS =Basic\][\Restic \
|
||||
CLOUD_PROVIDER=aws BSL_BUCKET=<BUCKET_FOR_E2E_TEST_BACKUP> BSL_PREFIX=<PREFIX_UNDER_BUCKET> CREDS_FILE=/path/to/aws-creds
|
||||
```
|
||||
In this example, only case `[Basic][Restic]` is included.
|
||||
In this example, only case have both `Basic` and `Restic` labels are included.
|
||||
|
||||
1. Run Velero tests with specific cases to be excluded:
|
||||
```bash
|
||||
make test-e2e \
|
||||
GINKGO_SKIP=Scale.Schedule.TTL.Upgrade\]\[Restic.Migration\][\Restic \
|
||||
CLOUD_PROVIDER=aws BSL_BUCKET=<BUCKET_FOR_E2E_TEST_BACKUP> BSL_PREFIX=<PREFIX_UNDER_BUCKET> CREDS_FILE=/path/to/aws-creds
|
||||
GINKGO_LABELS="!Scale || !Schedule || !TTL || !(Upgrade && Restic) || !(Migration && Restic)" \
|
||||
CLOUD_PROVIDER=aws \
|
||||
BSL_BUCKET=example-bucket \
|
||||
CREDS_FILE=/path/to/aws-creds \
|
||||
make test-e2e
|
||||
```
|
||||
In this example, case `Scale`, `Schedule`, `TTL`, `[Upgrade][Restic]` and `[Migration][Restic]` will be skipped.
|
||||
|
||||
|
||||
In this example, cases are labelled as
|
||||
* `Scale`
|
||||
* `Schedule`
|
||||
* `TTL`
|
||||
* `Upgrade` and `Restic`
|
||||
* `Migration` and `Restic`
|
||||
will be skipped.
|
||||
|
||||
## Full Tests execution
|
||||
|
||||
|
|
|
|||
|
|
@ -53,7 +53,7 @@ type apiGropuVersionsTest struct {
|
|||
want map[string]map[string]string
|
||||
}
|
||||
|
||||
func APIGropuVersionsTest() {
|
||||
func APIGroupVersionsTest() {
|
||||
var (
|
||||
group string
|
||||
err error
|
||||
|
|
|
|||
|
|
@ -112,79 +112,126 @@ func init() {
|
|||
// caused by no expected snapshot found. If we use retain as reclaim policy, then this label can be ignored, all test
|
||||
// cases can be executed as expected successful result.
|
||||
|
||||
var _ = Describe("[APIGroup][APIVersion][SKIP_KIND] Velero tests with various CRD API group versions", APIGropuVersionsTest)
|
||||
var _ = Describe("[APIGroup][APIExtensions][SKIP_KIND] CRD of apiextentions v1beta1 should be B/R successfully from cluster(k8s version < 1.22) to cluster(k8s version >= 1.22)", APIExtensionsVersionsTest)
|
||||
var _ = Describe("Velero tests with various CRD API group versions",
|
||||
Label("APIGroup", "APIVersion", "SKIP_KIND"), APIGroupVersionsTest)
|
||||
var _ = Describe("CRD of apiextentions v1beta1 should be B/R successfully from cluster(k8s version < 1.22) to cluster(k8s version >= 1.22)",
|
||||
Label("APIGroup", "APIExtensions", "SKIP_KIND"), APIExtensionsVersionsTest)
|
||||
|
||||
// Test backup and restore of Kibishi using restic
|
||||
var _ = Describe("[Basic][Restic] Velero tests on cluster using the plugin provider for object storage and Restic for volume backups", BackupRestoreWithRestic)
|
||||
// Test backup and restore of Kibishii using restic
|
||||
var _ = Describe("Velero tests on cluster using the plugin provider for object storage and Restic for volume backups",
|
||||
Label("Basic", "Restic"), BackupRestoreWithRestic)
|
||||
|
||||
var _ = Describe("[Basic][Snapshot][SkipVanillaZfs] Velero tests on cluster using the plugin provider for object storage and snapshots for volume backups", BackupRestoreWithSnapshots)
|
||||
var _ = Describe("Velero tests on cluster using the plugin provider for object storage and snapshots for volume backups",
|
||||
Label("Basic", "Snapshot", "SkipVanillaZfs"), BackupRestoreWithSnapshots)
|
||||
|
||||
var _ = Describe("[Basic][Snapshot][RetainPV] Velero tests on cluster using the plugin provider for object storage and snapshots for volume backups", BackupRestoreRetainedPVWithSnapshots)
|
||||
var _ = Describe("Velero tests on cluster using the plugin provider for object storage and snapshots for volume backups",
|
||||
Label("Basic", "Snapshot", "RetainPV"), BackupRestoreRetainedPVWithSnapshots)
|
||||
|
||||
var _ = Describe("[Basic][Restic][RetainPV] Velero tests on cluster using the plugin provider for object storage and snapshots for volume backups", BackupRestoreRetainedPVWithRestic)
|
||||
var _ = Describe("Velero tests on cluster using the plugin provider for object storage and snapshots for volume backups",
|
||||
Label("Basic", "Restic", "RetainPV"), BackupRestoreRetainedPVWithRestic)
|
||||
|
||||
var _ = Describe("[Basic][ClusterResource] Backup/restore of cluster resources", ResourcesCheckTest)
|
||||
var _ = Describe("Backup/restore of cluster resources",
|
||||
Label("Basic", "ClusterResource"), ResourcesCheckTest)
|
||||
|
||||
var _ = Describe("[Scale][LongTime] Backup/restore of 2500 namespaces", MultiNSBackupRestore)
|
||||
var _ = Describe("Service NodePort reservation during restore is configurable",
|
||||
Label("Basic", "NodePort"), NodePortTest)
|
||||
|
||||
// Upgrade test by Kibishi using restic
|
||||
var _ = Describe("[Upgrade][Restic] Velero upgrade tests on cluster using the plugin provider for object storage and Restic for volume backups", BackupUpgradeRestoreWithRestic)
|
||||
var _ = Describe("[Upgrade][Snapshot][SkipVanillaZfs] Velero upgrade tests on cluster using the plugin provider for object storage and snapshots for volume backups", BackupUpgradeRestoreWithSnapshots)
|
||||
var _ = Describe("Storage class of persistent volumes and persistent volume claims can be changed during restores",
|
||||
Label("Basic", "StorageClass"), StorageClasssChangingTest)
|
||||
|
||||
var _ = Describe("Node selectors of persistent volume claims can be changed during restores",
|
||||
Label("Basic", "SelectedNode", "SKIP_KIND"), PVCSelectedNodeChangingTest)
|
||||
|
||||
var _ = Describe("Backup/restore of 2500 namespaces",
|
||||
Label("Scale", "LongTime"), MultiNSBackupRestore)
|
||||
|
||||
// Upgrade test by Kibishii using Restic
|
||||
var _ = Describe("Velero upgrade tests on cluster using the plugin provider for object storage and Restic for volume backups",
|
||||
Label("Upgrade", "Restic"), BackupUpgradeRestoreWithRestic)
|
||||
var _ = Describe("Velero upgrade tests on cluster using the plugin provider for object storage and snapshots for volume backups",
|
||||
Label("Upgrade", "Snapshot", "SkipVanillaZfs"), BackupUpgradeRestoreWithSnapshots)
|
||||
|
||||
// test filter objects by namespace, type, or labels when backup or restore.
|
||||
var _ = Describe("[ResourceFiltering][ExcludeFromBackup] Resources with the label velero.io/exclude-from-backup=true are not included in backup", ExcludeFromBackupTest)
|
||||
var _ = Describe("[ResourceFiltering][ExcludeNamespaces][Backup] Velero test on exclude namespace from the cluster backup", BackupWithExcludeNamespaces)
|
||||
var _ = Describe("[ResourceFiltering][ExcludeNamespaces][Restore] Velero test on exclude namespace from the cluster restore", RestoreWithExcludeNamespaces)
|
||||
var _ = Describe("[ResourceFiltering][ExcludeResources][Backup] Velero test on exclude resources from the cluster backup", BackupWithExcludeResources)
|
||||
var _ = Describe("[ResourceFiltering][ExcludeResources][Restore] Velero test on exclude resources from the cluster restore", RestoreWithExcludeResources)
|
||||
var _ = Describe("[ResourceFiltering][IncludeNamespaces][Backup] Velero test on include namespace from the cluster backup", BackupWithIncludeNamespaces)
|
||||
var _ = Describe("[ResourceFiltering][IncludeNamespaces][Restore] Velero test on include namespace from the cluster restore", RestoreWithIncludeNamespaces)
|
||||
var _ = Describe("[ResourceFiltering][IncludeResources][Backup] Velero test on include resources from the cluster backup", BackupWithIncludeResources)
|
||||
var _ = Describe("[ResourceFiltering][IncludeResources][Restore] Velero test on include resources from the cluster restore", RestoreWithIncludeResources)
|
||||
var _ = Describe("[ResourceFiltering][LabelSelector] Velero test on backup include resources matching the label selector", BackupWithLabelSelector)
|
||||
var _ = Describe("[ResourceFiltering][ResourcePolicies][Restic] Velero test on skip backup of volume by resource policies", ResourcePoliciesTest)
|
||||
var _ = Describe("Resources with the label velero.io/exclude-from-backup=true are not included in backup",
|
||||
Label("ResourceFiltering", "ExcludeFromBackup"), ExcludeFromBackupTest)
|
||||
var _ = Describe("Velero test on exclude namespace from the cluster backup",
|
||||
Label("ResourceFiltering", "ExcludeNamespaces", "Backup"), BackupWithExcludeNamespaces)
|
||||
var _ = Describe("Velero test on exclude namespace from the cluster restore",
|
||||
Label("ResourceFiltering", "ExcludeNamespaces", "Restore"), RestoreWithExcludeNamespaces)
|
||||
var _ = Describe("Velero test on exclude resources from the cluster backup",
|
||||
Label("ResourceFiltering", "ExcludeResources", "Backup"), BackupWithExcludeResources)
|
||||
var _ = Describe("Velero test on exclude resources from the cluster restore",
|
||||
Label("ResourceFiltering", "ExcludeResources", "Restore"), RestoreWithExcludeResources)
|
||||
var _ = Describe("Velero test on include namespace from the cluster backup",
|
||||
Label("ResourceFiltering", "IncludeNamespaces", "Backup"), BackupWithIncludeNamespaces)
|
||||
var _ = Describe("Velero test on include namespace from the cluster restore",
|
||||
Label("ResourceFiltering", "IncludeNamespaces", "Restore"), RestoreWithIncludeNamespaces)
|
||||
var _ = Describe("Velero test on include resources from the cluster backup",
|
||||
Label("ResourceFiltering", "IncludeResources", "Backup"), BackupWithIncludeResources)
|
||||
var _ = Describe("Velero test on include resources from the cluster restore",
|
||||
Label("ResourceFiltering", "IncludeResources", "Restore"), RestoreWithIncludeResources)
|
||||
var _ = Describe("Velero test on backup include resources matching the label selector",
|
||||
Label("ResourceFiltering", "LabelSelector"), BackupWithLabelSelector)
|
||||
var _ = Describe("Velero test on skip backup of volume by resource policies",
|
||||
Label("ResourceFiltering", "ResourcePolicies", "Restic"), ResourcePoliciesTest)
|
||||
|
||||
// backup VolumeInfo test
|
||||
var _ = Describe("[BackupVolumeInfo][SkippedVolume]", SkippedVolumeInfoTest)
|
||||
var _ = Describe("[BackupVolumeInfo][FilesystemUpload]", FilesystemUploadVolumeInfoTest)
|
||||
var _ = Describe("[BackupVolumeInfo][CSIDataMover]", CSIDataMoverVolumeInfoTest)
|
||||
var _ = Describe("[BackupVolumeInfo][CSISnapshot]", CSISnapshotVolumeInfoTest)
|
||||
var _ = Describe("[BackupVolumeInfo][NativeSnapshot]", NativeSnapshotVolumeInfoTest)
|
||||
var _ = Describe("", Label("BackupVolumeInfo", "SkippedVolume"), SkippedVolumeInfoTest)
|
||||
var _ = Describe("", Label("BackupVolumeInfo", "FilesystemUpload"), FilesystemUploadVolumeInfoTest)
|
||||
var _ = Describe("", Label("BackupVolumeInfo", "CSIDataMover"), CSIDataMoverVolumeInfoTest)
|
||||
var _ = Describe("", Label("BackupVolumeInfo", "CSISnapshot"), CSISnapshotVolumeInfoTest)
|
||||
var _ = Describe("", Label("BackupVolumeInfo", "NativeSnapshot"), NativeSnapshotVolumeInfoTest)
|
||||
|
||||
var _ = Describe("[ResourceModifier][Restore] Velero test on resource modifiers from the cluster restore", ResourceModifiersTest)
|
||||
var _ = Describe("Velero test on resource modifiers from the cluster restore",
|
||||
Label("ResourceModifier", "Restore"), ResourceModifiersTest)
|
||||
|
||||
var _ = Describe("[Backups][Deletion][Restic] Velero tests of Restic backup deletion", BackupDeletionWithRestic)
|
||||
var _ = Describe("[Backups][Deletion][Snapshot][SkipVanillaZfs] Velero tests of snapshot backup deletion", BackupDeletionWithSnapshots)
|
||||
var _ = Describe("[Backups][TTL][LongTime][Snapshot][SkipVanillaZfs] Local backups and restic repos will be deleted once the corresponding backup storage location is deleted", TTLTest)
|
||||
var _ = Describe("[Backups][BackupsSync] Backups in object storage are synced to a new Velero and deleted backups in object storage are synced to be deleted in Velero", BackupsSyncTest)
|
||||
var _ = Describe("Velero tests of Restic backup deletion",
|
||||
Label("Backups", "Deletion", "Restic"), BackupDeletionWithRestic)
|
||||
var _ = Describe("Velero tests of snapshot backup deletion",
|
||||
Label("Backups", "Deletion", "Snapshot", "SkipVanillaZfs"), BackupDeletionWithSnapshots)
|
||||
var _ = Describe("Local backups and Restic repos will be deleted once the corresponding backup storage location is deleted",
|
||||
Label("Backups", "TTL", "LongTime", "Snapshot", "SkipVanillaZfs"), TTLTest)
|
||||
var _ = Describe("Backups in object storage are synced to a new Velero and deleted backups in object storage are synced to be deleted in Velero",
|
||||
Label("Backups", "BackupsSync"), BackupsSyncTest)
|
||||
|
||||
var _ = Describe("[Schedule][BR][Pause][LongTime] Backup will be created periodly by schedule defined by a Cron expression", ScheduleBackupTest)
|
||||
var _ = Describe("[Schedule][OrderedResources] Backup resources should follow the specific order in schedule", ScheduleOrderedResources)
|
||||
var _ = Describe("[Schedule][BackupCreation][SKIP_KIND] Schedule controller wouldn't create a new backup when it still has pending or InProgress backup", ScheduleBackupCreationTest)
|
||||
var _ = Describe("Backup will be created periodically by schedule defined by a Cron expression",
|
||||
Label("Schedule", "BR", "Pause", "LongTime"), ScheduleBackupTest)
|
||||
var _ = Describe("Backup resources should follow the specific order in schedule",
|
||||
Label("Schedule", "OrderedResources"), ScheduleOrderedResources)
|
||||
var _ = Describe("Schedule controller wouldn't create a new backup when it still has pending or InProgress backup",
|
||||
Label("Schedule", "BackupCreation", "SKIP_KIND"), ScheduleBackupCreationTest)
|
||||
|
||||
var _ = Describe("[PrivilegesMgmt][SSR] Velero test on ssr object when controller namespace mix-ups", SSRTest)
|
||||
var _ = Describe("Velero test on ssr object when controller namespace mix-ups",
|
||||
Label("PrivilegesMgmt", "SSR"), SSRTest)
|
||||
|
||||
var _ = Describe("[BSL][Deletion][Snapshot][SkipVanillaZfs] Local backups will be deleted once the corresponding backup storage location is deleted", BslDeletionWithSnapshots)
|
||||
var _ = Describe("[BSL][Deletion][Restic] Local backups and restic repos will be deleted once the corresponding backup storage location is deleted", BslDeletionWithRestic)
|
||||
var _ = Describe("Local backups will be deleted once the corresponding backup storage location is deleted",
|
||||
Label("BSL", "Deletion", "Snapshot", "SkipVanillaZfs"), BslDeletionWithSnapshots)
|
||||
var _ = Describe("Local backups and Restic repos will be deleted once the corresponding backup storage location is deleted",
|
||||
Label("BSL", "Deletion", "Restic"), BslDeletionWithRestic)
|
||||
|
||||
var _ = Describe("[Migration][Restic] Migrate resources between clusters by Restic", MigrationWithRestic)
|
||||
var _ = Describe("[Migration][Snapshot][SkipVanillaZfs] Migrate resources between clusters by snapshot", MigrationWithSnapshots)
|
||||
var _ = Describe("Migrate resources between clusters by Restic",
|
||||
Label("Migration", "Restic"), MigrationWithRestic)
|
||||
var _ = Describe("Migrate resources between clusters by snapshot",
|
||||
Label("Migration", "Snapshot", "SkipVanillaZfs"), MigrationWithSnapshots)
|
||||
|
||||
var _ = Describe("[NamespaceMapping][Single][Restic] Backup resources should follow the specific order in schedule", OneNamespaceMappingResticTest)
|
||||
var _ = Describe("[NamespaceMapping][Multiple][Restic] Backup resources should follow the specific order in schedule", MultiNamespacesMappingResticTest)
|
||||
var _ = Describe("[NamespaceMapping][Single][Snapshot][SkipVanillaZfs] Backup resources should follow the specific order in schedule", OneNamespaceMappingSnapshotTest)
|
||||
var _ = Describe("[NamespaceMapping][Multiple][Snapshot]SkipVanillaZfs] Backup resources should follow the specific order in schedule", MultiNamespacesMappingSnapshotTest)
|
||||
var _ = Describe("Backup resources should follow the specific order in schedule",
|
||||
Label("NamespaceMapping", "Single", "Restic"), OneNamespaceMappingResticTest)
|
||||
var _ = Describe("Backup resources should follow the specific order in schedule",
|
||||
Label("NamespaceMapping", "Multiple", "Restic"), MultiNamespacesMappingResticTest)
|
||||
var _ = Describe("Backup resources should follow the specific order in schedule",
|
||||
Label("NamespaceMapping", "Single", "Snapshot", "SkipVanillaZfs"), OneNamespaceMappingSnapshotTest)
|
||||
var _ = Describe("Backup resources should follow the specific order in schedule",
|
||||
Label("NamespaceMapping", "Multiple", "Snapshot", "SkipVanillaZfs"), MultiNamespacesMappingSnapshotTest)
|
||||
|
||||
var _ = Describe("Backup resources should follow the specific order in schedule", Label("pv-backup", "Opt-In"), OptInPVBackupTest)
|
||||
var _ = Describe("Backup resources should follow the specific order in schedule", Label("pv-backup", "Opt-Out"), OptOutPVBackupTest)
|
||||
var _ = Describe("Backup resources should follow the specific order in schedule",
|
||||
Label("PVBackup", "OptIn"), OptInPVBackupTest)
|
||||
var _ = Describe("Backup resources should follow the specific order in schedule",
|
||||
Label("PVBackup", "OptOut"), OptOutPVBackupTest)
|
||||
|
||||
var _ = Describe("[Basic][Nodeport] Service nodeport reservation during restore is configurable", NodePortTest)
|
||||
var _ = Describe("[Basic][StorageClass] Storage class of persistent volumes and persistent volume claims can be changed during restores", StorageClasssChangingTest)
|
||||
var _ = Describe("[Basic][SelectedNode][SKIP_KIND] Node selectors of persistent volume claims can be changed during restores", PVCSelectedNodeChangingTest)
|
||||
|
||||
var _ = Describe("[UploaderConfig][ParallelFilesUpload] Velero test on parallel files upload", ParallelFilesUploadTest)
|
||||
var _ = Describe("[UploaderConfig][ParallelFilesDownload] Velero test on parallel files download", ParallelFilesDownloadTest)
|
||||
var _ = Describe("Velero test on parallel files upload",
|
||||
Label("UploaderConfig", "ParallelFilesUpload"), ParallelFilesUploadTest)
|
||||
var _ = Describe("Velero test on parallel files download",
|
||||
Label("UploaderConfig", "ParallelFilesDownload"), ParallelFilesDownloadTest)
|
||||
|
||||
func GetKubeconfigContext() error {
|
||||
var err error
|
||||
|
|
|
|||
|
|
@ -94,11 +94,14 @@ func initConfig() error {
|
|||
return nil
|
||||
}
|
||||
|
||||
var _ = Describe("[PerformanceTest][BackupAndRestore] Velero test on both backup and restore resources", test.TestFunc(&basic.BasicTest{}))
|
||||
var _ = Describe("Velero test on both backup and restore resources",
|
||||
Label("PerformanceTest", "BackupAndRestore"), test.TestFunc(&basic.BasicTest{}))
|
||||
|
||||
var _ = Describe("[PerformanceTest][Backup] Velero test on only backup resources", test.TestFunc(&backup.BackupTest{}))
|
||||
var _ = Describe("Velero test on only backup resources",
|
||||
Label("PerformanceTest", "Backup"), test.TestFunc(&backup.BackupTest{}))
|
||||
|
||||
var _ = Describe("[PerformanceTest][Restore] Velero test on only restore resources", test.TestFunc(&restore.RestoreTest{}))
|
||||
var _ = Describe("Velero test on only restore resources",
|
||||
Label("PerformanceTest", "Restore"), test.TestFunc(&restore.RestoreTest{}))
|
||||
|
||||
func TestE2e(t *testing.T) {
|
||||
flag.Parse()
|
||||
|
|
|
|||
Loading…
Reference in New Issue