diff --git a/.github/workflows/e2e-test-kind.yaml b/.github/workflows/e2e-test-kind.yaml index d8fa8a1ec..b5747ee3a 100644 --- a/.github/workflows/e2e-test-kind.yaml +++ b/.github/workflows/e2e-test-kind.yaml @@ -135,7 +135,7 @@ jobs: ADDITIONAL_CREDS_FILE=/tmp/credential ADDITIONAL_BSL_BUCKET=additional-bucket \ GINKGO_FOCUS='${{ matrix.focus }}' VELERO_IMAGE=velero:pr-test \ GINKGO_SKIP='SKIP_KIND|pv-backup|Restic|Snapshot|LongTime' \ - make -C test/e2e run + make -C test/ run-e2e timeout-minutes: 30 - name: Upload debug bundle if: ${{ failure() }} diff --git a/Makefile b/Makefile index ce752cb16..8950bbc1b 100644 --- a/Makefile +++ b/Makefile @@ -370,11 +370,11 @@ gen-docs: .PHONY: test-e2e test-e2e: local - $(MAKE) -e VERSION=$(VERSION) -C test/e2e run + $(MAKE) -e VERSION=$(VERSION) -C test/ run-e2e .PHONY: test-perf test-perf: local - $(MAKE) -e VERSION=$(VERSION) -C test/perf run + $(MAKE) -e VERSION=$(VERSION) -C test/ run-perf go-generate: go generate ./pkg/... \ No newline at end of file diff --git a/test/e2e/Makefile b/test/Makefile similarity index 68% rename from test/e2e/Makefile rename to test/Makefile index 97a7ffb49..01b45a7ad 100644 --- a/test/e2e/Makefile +++ b/test/Makefile @@ -51,7 +51,7 @@ GINKGO_FOCUS ?= GINKGO_SKIP ?= SKIP_STR := $(foreach var, $(subst ., ,$(GINKGO_SKIP)),-skip "$(var)") FOCUS_STR := $(foreach var, $(subst ., ,$(GINKGO_FOCUS)),-focus "$(var)") -VELERO_CLI ?=$$(pwd)/../../_output/bin/$(GOOS)/$(GOARCH)/velero +VELERO_CLI ?=$$(pwd)/../_output/bin/$(GOOS)/$(GOARCH)/velero VELERO_IMAGE ?= velero/velero:main PLUGINS ?= RESTORE_HELPER_IMAGE ?= @@ -114,6 +114,41 @@ DEFAULT_CLUSTER_NAME ?= STANDBY_CLUSTER_NAME ?= EKS_POLICY_ARN ?= +# perf test related parameters +TEST_CASE_DESCRIBE ?= 'velero performance test' +BACKUP_FOR_RESTORE ?= +Delete_Cluster_Resource ?= false +Debug_Velero_Pod_Restart ?= false +NODE_AGENT_POD_CPU_LIMIT ?= 4 +NODE_AGENT_POD_MEM_LIMIT ?= 4Gi +NODE_AGENT_POD_CPU_REQUEST ?= 2 +NODE_AGENT_POD_MEM_REQUEST ?= 2Gi +VELERO_POD_CPU_LIMIT ?= 4 +VELERO_POD_MEM_LIMIT ?= 4Gi +VELERO_POD_CPU_REQUEST ?= 2 +VELERO_POD_MEM_REQUEST ?= 2Gi +POD_VOLUME_OPERATION_TIMEOUT ?= 6h + +COMMON_ARGS := -velerocli=$(VELERO_CLI) \ + -velero-image=$(VELERO_IMAGE) \ + -plugins=$(PLUGINS) \ + -velero-version=$(VELERO_VERSION) \ + -restore-helper-image=$(RESTORE_HELPER_IMAGE) \ + -velero-namespace=$(VELERO_NAMESPACE) \ + -credentials-file=$(CREDS_FILE) \ + -bucket=$(BSL_BUCKET) \ + -prefix=$(BSL_PREFIX) \ + -bsl-config=$(BSL_CONFIG) \ + -vsl-config=$(VSL_CONFIG) \ + -cloud-provider=$(CLOUD_PROVIDER) \ + -object-store-provider="$(OBJECT_STORE_PROVIDER)" \ + -features=$(FEATURES) \ + -install-velero=$(INSTALL_VELERO) \ + -registry-credential-file=$(REGISTRY_CREDENTIAL_FILE) \ + -debug-e2e-test=$(DEBUG_E2E_TEST) \ + -velero-server-debug-mode=$(VELERO_SERVER_DEBUG_MODE) \ + -uploader-type=$(UPLOADER_TYPE) \ + -debug-velero-pod-restart=$(DEBUG_VELERO_POD_RESTART) # Make sure ginkgo is in $GOBIN .PHONY:ginkgo @@ -123,59 +158,63 @@ ginkgo: ${GOBIN}/ginkgo ${GOBIN}/ginkgo: GOBIN=${GOBIN} go install github.com/onsi/ginkgo/ginkgo@v1.16.5 -.PHONY: run -run: ginkgo +.PHONY: run-e2e +run-e2e: ginkgo @[ "${CREDS_FILE}" ] && echo "Using credentials from ${CREDS_FILE}" || \ ( echo "A credentials file is required to run E2E tests, please re-run the make target with CREDS_FILE="; exit 1 ) @[ "${BSL_BUCKET}" ] && echo "Using bucket ${BSL_BUCKET} to store backups from E2E tests" || \ (echo "Bucket to store the backups from E2E tests is required, please re-run with BSL_BUCKET="; exit 1 ) @[ "${CLOUD_PROVIDER}" ] && echo "Using cloud provider ${CLOUD_PROVIDER}" || \ (echo "Cloud provider for target cloud/plugin provider is required, please rerun with CLOUD_PROVIDER="; exit 1) - @$(GINKGO) -v $(FOCUS_STR) $(SKIP_STR) . -- -velerocli=$(VELERO_CLI) \ - -velero-image=$(VELERO_IMAGE) \ - -plugins=$(PLUGINS) \ - -velero-version=$(VERSION) \ - -restore-helper-image=$(RESTORE_HELPER_IMAGE) \ + @$(GINKGO) -v $(FOCUS_STR) $(SKIP_STR) ./e2e -- $(COMMON_ARGS) \ -upgrade-from-velero-cli=$(UPGRADE_FROM_VELERO_CLI) \ -upgrade-from-velero-version=$(UPGRADE_FROM_VELERO_VERSION) \ -migrate-from-velero-cli=$(MIGRATE_FROM_VELERO_CLI) \ -migrate-from-velero-version=$(MIGRATE_FROM_VELERO_VERSION) \ - -velero-namespace=$(VELERO_NAMESPACE) \ - -credentials-file=$(CREDS_FILE) \ - -bucket=$(BSL_BUCKET) \ - -prefix=$(BSL_PREFIX) \ - -bsl-config=$(BSL_CONFIG) \ - -vsl-config=$(VSL_CONFIG) \ - -cloud-provider=$(CLOUD_PROVIDER) \ - -object-store-provider="$(OBJECT_STORE_PROVIDER)" \ -additional-bsl-plugins=$(ADDITIONAL_BSL_PLUGINS) \ -additional-bsl-object-store-provider="$(ADDITIONAL_OBJECT_STORE_PROVIDER)" \ -additional-bsl-credentials-file=$(ADDITIONAL_CREDS_FILE) \ -additional-bsl-bucket=$(ADDITIONAL_BSL_BUCKET) \ -additional-bsl-prefix=$(ADDITIONAL_BSL_PREFIX) \ -additional-bsl-config=$(ADDITIONAL_BSL_CONFIG) \ - -features=$(FEATURES) \ - -install-velero=$(INSTALL_VELERO) \ - -registry-credential-file=$(REGISTRY_CREDENTIAL_FILE) \ - -kibishii-directory=$(KIBISHII_DIRECTORY) \ - -debug-e2e-test=$(DEBUG_E2E_TEST) \ - -velero-server-debug-mode=$(VELERO_SERVER_DEBUG_MODE) \ -default-cluster-context=$(DEFAULT_CLUSTER) \ -standby-cluster-context=$(STANDBY_CLUSTER) \ - -uploader-type=$(UPLOADER_TYPE) \ -snapshot-move-data=$(SNAPSHOT_MOVE_DATA) \ -data-mover-plugin=$(DATA_MOVER_PLUGIN) \ -standby-cluster-cloud-provider=$(STANDBY_CLUSTER_CLOUD_PROVIDER) \ -standby-cluster-plugins=$(STANDBY_CLUSTER_PLUGINS) \ -standby-cluster-object-store-provider=$(STANDBY_CLUSTER_OBJECT_STORE_PROVIDER) \ - -debug-velero-pod-restart=$(DEBUG_VELERO_POD_RESTART) \ - -disable-informer-cache=$(DISABLE_INFORMER_CACHE) \ -default-cluster-name=$(DEFAULT_CLUSTER_NAME) \ -standby-cluster-name=$(STANDBY_CLUSTER_NAME) \ -eks-policy-arn=$(EKS_POLICY_ARN) \ -default-cls-service-account-name=$(DEFAULT_CLS_SERVICE_ACCOUNT_NAME) \ -standby-cls-service-account-name=$(STANDBY_CLS_SERVICE_ACCOUNT_NAME) + -kibishii-directory=$(KIBISHII_DIRECTORY) \ + -disable-informer-cache=$(DISABLE_INFORMER_CACHE) + +.PHONY: run-perf +run-perf: ginkgo + @[ "${CREDS_FILE}" ] && echo "Using credentials from ${CREDS_FILE}" || \ + ( echo "A credentials file is required to run E2E tests, please re-run the make target with CREDS_FILE="; exit 1 ) + @[ "${BSL_BUCKET}" ] && echo "Using bucket ${BSL_BUCKET} to store backups from E2E tests" || \ + (echo "Bucket to store the backups from E2E tests is required, please re-run with BSL_BUCKET="; exit 1 ) + @[ "${CLOUD_PROVIDER}" ] && echo "Using cloud provider ${CLOUD_PROVIDER}" || \ + (echo "Cloud provider for target cloud/plugin provider is required, please rerun with CLOUD_PROVIDER="; exit 1) + @$(GINKGO) -v $(FOCUS_STR) $(SKIP_STR) ./perf -- $(COMMON_ARGS) \ + -nfs-server-path=$(NFS_SERVER_PATH) \ + -test-case-describe=$(TEST_CASE_DESCRIBE) \ + -backup-for-restore=$(BACKUP_FOR_RESTORE) \ + -delete-cluster-resource=$(Delete_Cluster_Resource) \ + -node-agent-pod-cpu-limit=$(NODE_AGENT_POD_CPU_LIMIT) \ + -node-agent-pod-mem-limit=$(NODE_AGENT_POD_MEM_LIMIT) \ + -node-agent-pod-cpu-request=$(NODE_AGENT_POD_CPU_REQUEST) \ + -node-agent-pod-mem-request=$(NODE_AGENT_POD_MEM_REQUEST) \ + -velero-pod-cpu-limit=$(VELERO_POD_CPU_LIMIT) \ + -velero-pod-mem-limit=$(VELERO_POD_MEM_LIMIT) \ + -velero-pod-cpu-request=$(VELERO_POD_CPU_REQUEST) \ + -velero-pod-mem-request=$(VELERO_POD_MEM_REQUEST) \ + -pod-volume-operation-timeout=$(POD_VOLUME_OPERATION_TIMEOUT) build: ginkgo mkdir -p $(OUTPUT_DIR) - $(GINKGO) build . + $(GINKGO) build . \ No newline at end of file diff --git a/test/perf/Makefile b/test/perf/Makefile deleted file mode 100644 index 7500d0b1b..000000000 --- a/test/perf/Makefile +++ /dev/null @@ -1,145 +0,0 @@ -# Copyright the Velero contributors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# If you update this file, please follow: -# https://suva.sh/posts/well-documented-makefiles/ - -# Use GOPROXY environment variable if set - -.DEFAULT_GOAL:=help - -ARCH ?= $(shell go env GOOS)-$(shell go env GOARCH) -platform_temp = $(subst -, ,$(ARCH)) -GOOS = $(word 1, $(platform_temp)) -GOARCH = $(word 2, $(platform_temp)) - -GOPROXY := $(shell go env GOPROXY) -ifeq ($(GOPROXY),) -GOPROXY := https://proxy.golang.org -endif -export GOPROXY - -REPO_ROOT := $(shell git rev-parse --show-toplevel) - -help: ## Display this help - @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z0-9_-]+:.*?##/ { printf " \033[36m%-25s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) - -## -------------------------------------- -## Binaries -## -------------------------------------- - -TOOLS_DIR := $(REPO_ROOT)/hack/tools -BIN_DIR := bin -# Try to not modify PATH if possible -GOBIN := $(REPO_ROOT)/.go/bin -TOOLS_BIN_DIR := $(TOOLS_DIR)/$(BIN_DIR) -GINKGO := $(GOBIN)/ginkgo -KUSTOMIZE := $(TOOLS_BIN_DIR)/kustomize -OUTPUT_DIR := _output/$(GOOS)/$(GOARCH)/bin -GINKGO_FOCUS ?= -GINKGO_SKIP ?= -SKIP_STR := $(foreach var, $(subst ., ,$(GINKGO_SKIP)),-skip "$(var)") -FOCUS_STR := $(foreach var, $(subst ., ,$(GINKGO_FOCUS)),-focus "$(var)") -VELERO_CLI ?=$$(pwd)/../../_output/bin/$(GOOS)/$(GOARCH)/velero -VELERO_IMAGE ?= velero/velero:main -VELERO_VERSION ?= $(VERSION) -PLUGINS ?= -RESTORE_HELPER_IMAGE ?= - -VELERO_NAMESPACE ?= velero -CREDS_FILE ?= -BSL_BUCKET ?= -BSL_PREFIX ?= -BSL_CONFIG ?= -VSL_CONFIG ?= -CLOUD_PROVIDER ?= -OBJECT_STORE_PROVIDER ?= -INSTALL_VELERO ?= true -REGISTRY_CREDENTIAL_FILE ?= - -FEATURES ?= -DEBUG_E2E_TEST ?= false -VELERO_SERVER_DEBUG_MODE ?= false -NFS_SERVER_PATH ?= -# Parameters to run migration tests along with all other E2E tests, and both of them should -# be provided or left them all empty to skip migration tests with no influence to other -# E2E tests. -UPLOADER_TYPE ?= -TEST_CASE_DESCRIBE ?= 'velero performance test' -BACKUP_FOR_RESTORE ?= -Delete_Cluster_Resource ?= false -Debug_Velero_Pod_Restart ?= false -NODE_AGENT_POD_CPU_LIMIT ?= 4 -NODE_AGENT_POD_MEM_LIMIT ?= 4Gi -NODE_AGENT_POD_CPU_REQUEST ?= 2 -NODE_AGENT_POD_MEM_REQUEST ?= 2Gi -VELERO_POD_CPU_LIMIT ?= 4 -VELERO_POD_MEM_LIMIT ?= 4Gi -VELERO_POD_CPU_REQUEST ?= 2 -VELERO_POD_MEM_REQUEST ?= 2Gi -POD_VOLUME_OPERATION_TIMEOUT ?= 6h - -# Make sure ginkgo is in $GOBIN -.PHONY:ginkgo -ginkgo: ${GOBIN}/ginkgo - -# This target does not run if ginkgo is already in $GOBIN -${GOBIN}/ginkgo: - GOBIN=${GOBIN} go install github.com/onsi/ginkgo/ginkgo@v1.16.5 - -.PHONY: run -run: ginkgo - @[ "${CREDS_FILE}" ] && echo "Using credentials from ${CREDS_FILE}" || \ - ( echo "A credentials file is required to run E2E tests, please re-run the make target with CREDS_FILE="; exit 1 ) - @[ "${BSL_BUCKET}" ] && echo "Using bucket ${BSL_BUCKET} to store backups from E2E tests" || \ - (echo "Bucket to store the backups from E2E tests is required, please re-run with BSL_BUCKET="; exit 1 ) - @[ "${CLOUD_PROVIDER}" ] && echo "Using cloud provider ${CLOUD_PROVIDER}" || \ - (echo "Cloud provider for target cloud/plugin provider is required, please rerun with CLOUD_PROVIDER="; exit 1) - @$(GINKGO) -v $(FOCUS_STR) $(SKIP_STR) . -- -velerocli=$(VELERO_CLI) \ - -velero-image=$(VELERO_IMAGE) \ - -plugins=$(PLUGINS) \ - -velero-version=$(VELERO_VERSION) \ - -restore-helper-image=$(RESTORE_HELPER_IMAGE) \ - -velero-namespace=$(VELERO_NAMESPACE) \ - -credentials-file=$(CREDS_FILE) \ - -bucket=$(BSL_BUCKET) \ - -prefix=$(BSL_PREFIX) \ - -bsl-config=$(BSL_CONFIG) \ - -vsl-config=$(VSL_CONFIG) \ - -cloud-provider=$(CLOUD_PROVIDER) \ - -object-store-provider="$(OBJECT_STORE_PROVIDER)" \ - -features=$(FEATURES) \ - -install-velero=$(INSTALL_VELERO) \ - -registry-credential-file=$(REGISTRY_CREDENTIAL_FILE) \ - -debug-e2e-test=$(DEBUG_E2E_TEST) \ - -velero-server-debug-mode=$(VELERO_SERVER_DEBUG_MODE) \ - -uploader-type=$(UPLOADER_TYPE) \ - -nfs-server-path=$(NFS_SERVER_PATH) \ - -test-case-describe=$(TEST_CASE_DESCRIBE) \ - -backup-for-restore=$(BACKUP_FOR_RESTORE) \ - -delete-cluster-resource=$(Delete_Cluster_Resource) \ - -debug-velero-pod-restart=$(Debug_Velero_Pod_Restart) \ - -node-agent-pod-cpu-limit=$(NODE_AGENT_POD_CPU_LIMIT) \ - -node-agent-pod-mem-limit=$(NODE_AGENT_POD_MEM_LIMIT) \ - -node-agent-pod-cpu-request=$(NODE_AGENT_POD_CPU_REQUEST) \ - -node-agent-pod-mem-request=$(NODE_AGENT_POD_MEM_REQUEST) \ - -velero-pod-cpu-limit=$(VELERO_POD_CPU_LIMIT) \ - -velero-pod-mem-limit=$(VELERO_POD_MEM_LIMIT) \ - -velero-pod-cpu-request=$(VELERO_POD_CPU_REQUEST) \ - -velero-pod-mem-request=$(VELERO_POD_MEM_REQUEST) \ - -pod-volume-operation-timeout=$(POD_VOLUME_OPERATION_TIMEOUT) - -build: ginkgo - mkdir -p $(OUTPUT_DIR) - $(GINKGO) build .