From 4048c020a87466993c4933c5f25def8f14bc5de8 Mon Sep 17 00:00:00 2001 From: Carlisia Campos Date: Wed, 24 Jun 2020 09:55:18 -0700 Subject: [PATCH] Convert manifests + BSL api client to kubebuilder (#2561) * kubebuilder init - minimalist version Signed-off-by: Carlisia * Add back main.go, apparently kb needs it Signed-off-by: Carlisia * Tweak makefile to accomodate kubebuilder expectations Signed-off-by: Carlisia * Port BSL to kubebuilder api client Signed-off-by: Carlisia * s/cache/client bc client fetches from cache And other naming improvements Signed-off-by: Carlisia * So, .GetAPIReader is how we bypass the cache In this case, the cache hasn't started yet Signed-off-by: Carlisia * Oh that's what this code was for... adding back We still need to embed the CRDs as binary data in the Velero binary to access the generated CRDs at runtime. Signed-off-by: Carlisia * Tie in CRD/code generation w/ existing scripts Signed-off-by: Carlisia * Mostly result of running update-fmt, updated file formatting Signed-off-by: Carlisia * Just a copyright fix Signed-off-by: Carlisia * All the test fixes Signed-off-by: Carlisia * Add changelog + some cleanup Signed-off-by: Carlisia * Update backup manifest Signed-off-by: Carlisia * Remove unneeded auto-generated files Signed-off-by: Carlisia * Keep everything in the same (existing) package Signed-off-by: Carlisia * Fix/clean scripts, generated code, and calls Deleting the entire `generated` directory and running `make update` works. Modifying an api and running `make verify` works as expected. Signed-off-by: Carlisia * Clean up schema and client calls + code reviews Signed-off-by: Carlisia * Move all code gen to inside builder container Signed-off-by: Carlisia * Address code review Signed-off-by: Carlisia * Fix imports/aliases Signed-off-by: Carlisia * More code reviews Signed-off-by: Carlisia * Add waitforcachesync Signed-off-by: Carlisia * Have manager register ALL controllers This will allow for proper cache management. Signed-off-by: Carlisia * Status subresource is now enabled; cleanup Signed-off-by: Carlisia * More code reviews Signed-off-by: Carlisia * Clean up Signed-off-by: Carlisia * Manager registers ALL controllers for restic too Signed-off-by: Carlisia * More code reviews Signed-off-by: Carlisia * Add deprecation warning/todo Signed-off-by: Carlisia * Add documentation Signed-off-by: Carlisia * Add helpful comments Signed-off-by: Carlisia * Address code review Signed-off-by: Carlisia * More idiomatic Runnable Signed-off-by: Carlisia * Clean up imports Signed-off-by: Carlisia --- PROJECT | 7 + changelogs/unreleased/2561-carlisia | 1 + config/crd/bases/velero.io_backups.yaml | 423 ++++++++++++++++++ .../velero.io_backupstoragelocations.yaml | 19 +- .../velero.io_deletebackuprequests.yaml | 0 .../bases}/velero.io_downloadrequests.yaml | 0 .../bases}/velero.io_podvolumebackups.yaml | 0 .../bases}/velero.io_podvolumerestores.yaml | 0 .../bases}/velero.io_resticrepositories.yaml | 0 .../crd/bases}/velero.io_restores.yaml | 0 config/crd/bases/velero.io_schedules.yaml | 379 ++++++++++++++++ .../velero.io_serverstatusrequests.yaml | 0 .../velero.io_volumesnapshotlocations.yaml | 0 config/crd/crds/crds.go | 69 +++ {pkg/generated => config/crd}/crds/doc.go | 0 .../velero_v1_backupstoragelocation.yaml | 16 + go.mod | 3 +- go.sum | 39 +- hack/crd-gen/main.go | 14 +- hack/update-generated-crd-code.sh | 11 +- hack/verify-generated-crd-code.sh | 6 +- .../managercontroller/managercontroller.go | 54 +++ ...tion.go => backupstoragelocation_types.go} | 161 +++---- pkg/apis/velero/v1/groupversion_info.go | 36 ++ pkg/apis/velero/v1/register.go | 14 - pkg/client/factory.go | 25 +- pkg/cmd/cli/backup/create.go | 14 +- pkg/cmd/cli/backuplocation/create.go | 7 +- pkg/cmd/cli/backuplocation/get.go | 21 +- pkg/cmd/cli/restic/server.go | 63 ++- pkg/cmd/server/server.go | 80 ++-- .../output/backup_storage_location_printer.go | 2 +- pkg/controller/backup_controller.go | 16 +- pkg/controller/backup_controller_test.go | 52 +-- pkg/controller/backup_deletion_controller.go | 29 +- .../backup_deletion_controller_test.go | 243 +++++----- pkg/controller/backup_sync_controller.go | 100 ++--- pkg/controller/backup_sync_controller_test.go | 15 +- pkg/controller/download_request_controller.go | 35 +- .../download_request_controller_test.go | 106 ++--- pkg/controller/gc_controller.go | 22 +- pkg/controller/gc_controller_test.go | 58 +-- .../pod_volume_backup_controller.go | 16 +- .../pod_volume_restore_controller.go | 17 +- .../restic_repository_controller.go | 64 +-- pkg/controller/restore_controller.go | 20 +- pkg/controller/restore_controller_test.go | 205 ++++----- pkg/controller/suite_test.go | 37 ++ pkg/install/resources.go | 25 +- pkg/restic/common.go | 42 +- pkg/restic/common_test.go | 30 +- pkg/restic/repository_manager.go | 67 ++- site/docs/master/code-standards.md | 4 +- site/docs/v1.3.1/code-standards.md | 3 +- site/docs/v1.3.2/code-standards.md | 4 +- site/docs/v1.4/code-standards.md | 4 +- 56 files changed, 1946 insertions(+), 732 deletions(-) create mode 100644 PROJECT create mode 100644 changelogs/unreleased/2561-carlisia create mode 100644 config/crd/bases/velero.io_backups.yaml rename {pkg/generated/crds/manifests => config/crd/bases}/velero.io_backupstoragelocations.yaml (89%) rename {pkg/generated/crds/manifests => config/crd/bases}/velero.io_deletebackuprequests.yaml (100%) rename {pkg/generated/crds/manifests => config/crd/bases}/velero.io_downloadrequests.yaml (100%) rename {pkg/generated/crds/manifests => config/crd/bases}/velero.io_podvolumebackups.yaml (100%) rename {pkg/generated/crds/manifests => config/crd/bases}/velero.io_podvolumerestores.yaml (100%) rename {pkg/generated/crds/manifests => config/crd/bases}/velero.io_resticrepositories.yaml (100%) rename {pkg/generated/crds/manifests => config/crd/bases}/velero.io_restores.yaml (100%) create mode 100644 config/crd/bases/velero.io_schedules.yaml rename {pkg/generated/crds/manifests => config/crd/bases}/velero.io_serverstatusrequests.yaml (100%) rename {pkg/generated/crds/manifests => config/crd/bases}/velero.io_volumesnapshotlocations.yaml (100%) create mode 100644 config/crd/crds/crds.go rename {pkg/generated => config/crd}/crds/doc.go (100%) create mode 100644 config/samples/velero_v1_backupstoragelocation.yaml create mode 100644 internal/util/managercontroller/managercontroller.go rename pkg/apis/velero/v1/{backup_storage_location.go => backupstoragelocation_types.go} (78%) create mode 100644 pkg/apis/velero/v1/groupversion_info.go create mode 100644 pkg/controller/suite_test.go diff --git a/PROJECT b/PROJECT new file mode 100644 index 000000000..88609cdaa --- /dev/null +++ b/PROJECT @@ -0,0 +1,7 @@ +domain: io +repo: github.com/vmware-tanzu/velero +resources: +- group: velero + kind: BackupStorageLocation + version: v1 +version: "2" diff --git a/changelogs/unreleased/2561-carlisia b/changelogs/unreleased/2561-carlisia new file mode 100644 index 000000000..bedf5568c --- /dev/null +++ b/changelogs/unreleased/2561-carlisia @@ -0,0 +1 @@ +Convert manifests + BSL api client to kubebuilder \ No newline at end of file diff --git a/config/crd/bases/velero.io_backups.yaml b/config/crd/bases/velero.io_backups.yaml new file mode 100644 index 000000000..b527f7513 --- /dev/null +++ b/config/crd/bases/velero.io_backups.yaml @@ -0,0 +1,423 @@ + +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.2.4 + creationTimestamp: null + name: backups.velero.io +spec: + group: velero.io + names: + kind: Backup + listKind: BackupList + plural: backups + singular: backup + preserveUnknownFields: false + scope: Namespaced + validation: + openAPIV3Schema: + description: Backup is a Velero resource that respresents the capture of Kubernetes + cluster state at a point in time (API objects and associated volume state). + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BackupSpec defines the specification for a Velero backup. + properties: + defaultVolumesToRestic: + description: DefaultVolumesToRestic specifies whether restic should + be used to take a backup of all pod volumes by default. + type: boolean + excludedNamespaces: + description: ExcludedNamespaces contains a list of namespaces that are + not included in the backup. + items: + type: string + nullable: true + type: array + excludedResources: + description: ExcludedResources is a slice of resource names that are + not included in the backup. + items: + type: string + nullable: true + type: array + hooks: + description: Hooks represent custom behaviors that should be executed + at different phases of the backup. + properties: + resources: + description: Resources are hooks that should be executed when backing + up individual instances of a resource. + items: + description: BackupResourceHookSpec defines one or more BackupResourceHooks + that should be executed based on the rules defined for namespaces, + resources, and label selector. + properties: + excludedNamespaces: + description: ExcludedNamespaces specifies the namespaces to + which this hook spec does not apply. + items: + type: string + nullable: true + type: array + excludedResources: + description: ExcludedResources specifies the resources to + which this hook spec does not apply. + items: + type: string + nullable: true + type: array + includedNamespaces: + description: IncludedNamespaces specifies the namespaces to + which this hook spec applies. If empty, it applies to all + namespaces. + items: + type: string + nullable: true + type: array + includedResources: + description: IncludedResources specifies the resources to + which this hook spec applies. If empty, it applies to all + resources. + items: + type: string + nullable: true + type: array + labelSelector: + description: LabelSelector, if specified, filters the resources + to which this hook spec applies. + nullable: true + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector + that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, NotIn, + Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. + If the operator is In or NotIn, the values array + must be non-empty. If the operator is Exists or + DoesNotExist, the values array must be empty. + This array is replaced during a strategic merge + patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. + A single {key,value} in the matchLabels map is equivalent + to an element of matchExpressions, whose key field is + "key", the operator is "In", and the values array contains + only "value". The requirements are ANDed. + type: object + type: object + name: + description: Name is the name of this hook. + type: string + post: + description: PostHooks is a list of BackupResourceHooks to + execute after storing the item in the backup. These are + executed after all "additional items" from item actions + are processed. + items: + description: BackupResourceHook defines a hook for a resource. + properties: + exec: + description: Exec defines an exec hook. + properties: + command: + description: Command is the command and arguments + to execute. + items: + type: string + minItems: 1 + type: array + container: + description: Container is the container in the pod + where the command should be executed. If not specified, + the pod's first container is used. + type: string + onError: + description: OnError specifies how Velero should + behave if it encounters an error executing this + hook. + enum: + - Continue + - Fail + type: string + timeout: + description: Timeout defines the maximum amount + of time Velero should wait for the hook to complete + before considering the execution a failure. + type: string + required: + - command + type: object + required: + - exec + type: object + type: array + pre: + description: PreHooks is a list of BackupResourceHooks to + execute prior to storing the item in the backup. These are + executed before any "additional items" from item actions + are processed. + items: + description: BackupResourceHook defines a hook for a resource. + properties: + exec: + description: Exec defines an exec hook. + properties: + command: + description: Command is the command and arguments + to execute. + items: + type: string + minItems: 1 + type: array + container: + description: Container is the container in the pod + where the command should be executed. If not specified, + the pod's first container is used. + type: string + onError: + description: OnError specifies how Velero should + behave if it encounters an error executing this + hook. + enum: + - Continue + - Fail + type: string + timeout: + description: Timeout defines the maximum amount + of time Velero should wait for the hook to complete + before considering the execution a failure. + type: string + required: + - command + type: object + required: + - exec + type: object + type: array + required: + - name + type: object + nullable: true + type: array + type: object + includeClusterResources: + description: IncludeClusterResources specifies whether cluster-scoped + resources should be included for consideration in the backup. + nullable: true + type: boolean + includedNamespaces: + description: IncludedNamespaces is a slice of namespace names to include + objects from. If empty, all namespaces are included. + items: + type: string + nullable: true + type: array + includedResources: + description: IncludedResources is a slice of resource names to include + in the backup. If empty, all resources are included. + items: + type: string + nullable: true + type: array + labelSelector: + description: LabelSelector is a metav1.LabelSelector to filter with + when adding individual objects to the backup. If empty or nil, all + objects are included. Optional. + nullable: true + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement is a selector that contains + values, a key, and an operator that relates the key and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: operator represents a key's relationship to a + set of values. Valid operators are In, NotIn, Exists and + DoesNotExist. + type: string + values: + description: values is an array of string values. If the operator + is In or NotIn, the values array must be non-empty. If the + operator is Exists or DoesNotExist, the values array must + be empty. This array is replaced during a strategic merge + patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels map is equivalent to an element + of matchExpressions, whose key field is "key", the operator is + "In", and the values array contains only "value". The requirements + are ANDed. + type: object + type: object + snapshotVolumes: + description: SnapshotVolumes specifies whether to take cloud snapshots + of any PV's referenced in the set of objects included in the Backup. + nullable: true + type: boolean + storageLocation: + description: StorageLocation is a string containing the name of a BackupStorageLocation + where the backup should be stored. + type: string + ttl: + description: TTL is a time.Duration-parseable string describing how + long the Backup should be retained for. + type: string + volumeSnapshotLocations: + description: VolumeSnapshotLocations is a list containing names of VolumeSnapshotLocations + associated with this backup. + items: + type: string + type: array + type: object + status: + description: BackupStatus captures the current status of a Velero backup. + properties: + completionTimestamp: + description: CompletionTimestamp records the time a backup was completed. + Completion time is recorded even on failed backups. Completion time + is recorded before uploading the backup object. The server's time + is used for CompletionTimestamps + format: date-time + nullable: true + type: string + errors: + description: Errors is a count of all error messages that were generated + during execution of the backup. The actual errors are in the backup's + log file in object storage. + type: integer + expiration: + description: Expiration is when this Backup is eligible for garbage-collection. + format: date-time + nullable: true + type: string + formatVersion: + description: FormatVersion is the backup format version, including major, + minor, and patch version. + type: string + phase: + description: Phase is the current state of the Backup. + enum: + - New + - FailedValidation + - InProgress + - Completed + - PartiallyFailed + - Failed + - Deleting + type: string + progress: + description: Progress contains information about the backup's execution + progress. Note that this information is best-effort only -- if Velero + fails to update it during a backup for any reason, it may be inaccurate/stale. + nullable: true + properties: + itemsBackedUp: + description: ItemsBackedUp is the number of items that have actually + been written to the backup tarball so far. + type: integer + totalItems: + description: TotalItems is the total number of items to be backed + up. This number may change throughout the execution of the backup + due to plugins that return additional related items to back up, + the velero.io/exclude-from-backup label, and various other filters + that happen as items are processed. + type: integer + type: object + startTimestamp: + description: StartTimestamp records the time a backup was started. Separate + from CreationTimestamp, since that value changes on restores. The + server's time is used for StartTimestamps + format: date-time + nullable: true + type: string + validationErrors: + description: ValidationErrors is a slice of all validation errors (if + applicable). + items: + type: string + nullable: true + type: array + version: + description: 'Version is the backup format major version. Deprecated: + Please see FormatVersion' + type: integer + volumeSnapshotsAttempted: + description: VolumeSnapshotsAttempted is the total number of attempted + volume snapshots for this backup. + type: integer + volumeSnapshotsCompleted: + description: VolumeSnapshotsCompleted is the total number of successfully + completed volume snapshots for this backup. + type: integer + warnings: + description: Warnings is a count of all warning messages that were generated + during execution of the backup. The actual warnings are in the backup's + log file in object storage. + type: integer + type: object + type: object + version: v1 + versions: + - name: v1 + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/pkg/generated/crds/manifests/velero.io_backupstoragelocations.yaml b/config/crd/bases/velero.io_backupstoragelocations.yaml similarity index 89% rename from pkg/generated/crds/manifests/velero.io_backupstoragelocations.yaml rename to config/crd/bases/velero.io_backupstoragelocations.yaml index fe503368e..0a4eb4fb1 100644 --- a/pkg/generated/crds/manifests/velero.io_backupstoragelocations.yaml +++ b/config/crd/bases/velero.io_backupstoragelocations.yaml @@ -8,6 +8,14 @@ metadata: creationTimestamp: null name: backupstoragelocations.velero.io spec: + additionalPrinterColumns: + - JSONPath: .status.phase + description: Backup Storage Location status such as Available/Unavailable + name: Phase + type: string + - JSONPath: .metadata.creationTimestamp + name: Age + type: date group: velero.io names: kind: BackupStorageLocation @@ -16,10 +24,12 @@ spec: singular: backupstoragelocation preserveUnknownFields: false scope: Namespaced + subresources: + status: {} validation: openAPIV3Schema: description: BackupStorageLocation is a location where Velero stores backup - objects. + objects properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation @@ -34,8 +44,8 @@ spec: metadata: type: object spec: - description: BackupStorageLocationSpec defines the specification for a Velero - BackupStorageLocation. + description: BackupStorageLocationSpec defines the desired state of a Velero + BackupStorageLocation properties: accessMode: description: AccessMode defines the permissions for the backup storage @@ -81,8 +91,7 @@ spec: - provider type: object status: - description: BackupStorageLocationStatus describes the current status of - a Velero BackupStorageLocation. + description: BackupStorageLocationStatus defines the observed state of BackupStorageLocation properties: accessMode: description: "AccessMode is an unused field. \n Deprecated: there is diff --git a/pkg/generated/crds/manifests/velero.io_deletebackuprequests.yaml b/config/crd/bases/velero.io_deletebackuprequests.yaml similarity index 100% rename from pkg/generated/crds/manifests/velero.io_deletebackuprequests.yaml rename to config/crd/bases/velero.io_deletebackuprequests.yaml diff --git a/pkg/generated/crds/manifests/velero.io_downloadrequests.yaml b/config/crd/bases/velero.io_downloadrequests.yaml similarity index 100% rename from pkg/generated/crds/manifests/velero.io_downloadrequests.yaml rename to config/crd/bases/velero.io_downloadrequests.yaml diff --git a/pkg/generated/crds/manifests/velero.io_podvolumebackups.yaml b/config/crd/bases/velero.io_podvolumebackups.yaml similarity index 100% rename from pkg/generated/crds/manifests/velero.io_podvolumebackups.yaml rename to config/crd/bases/velero.io_podvolumebackups.yaml diff --git a/pkg/generated/crds/manifests/velero.io_podvolumerestores.yaml b/config/crd/bases/velero.io_podvolumerestores.yaml similarity index 100% rename from pkg/generated/crds/manifests/velero.io_podvolumerestores.yaml rename to config/crd/bases/velero.io_podvolumerestores.yaml diff --git a/pkg/generated/crds/manifests/velero.io_resticrepositories.yaml b/config/crd/bases/velero.io_resticrepositories.yaml similarity index 100% rename from pkg/generated/crds/manifests/velero.io_resticrepositories.yaml rename to config/crd/bases/velero.io_resticrepositories.yaml diff --git a/pkg/generated/crds/manifests/velero.io_restores.yaml b/config/crd/bases/velero.io_restores.yaml similarity index 100% rename from pkg/generated/crds/manifests/velero.io_restores.yaml rename to config/crd/bases/velero.io_restores.yaml diff --git a/config/crd/bases/velero.io_schedules.yaml b/config/crd/bases/velero.io_schedules.yaml new file mode 100644 index 000000000..e612fc11d --- /dev/null +++ b/config/crd/bases/velero.io_schedules.yaml @@ -0,0 +1,379 @@ + +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.2.4 + creationTimestamp: null + name: schedules.velero.io +spec: + group: velero.io + names: + kind: Schedule + listKind: ScheduleList + plural: schedules + singular: schedule + preserveUnknownFields: false + scope: Namespaced + validation: + openAPIV3Schema: + description: Schedule is a Velero resource that represents a pre-scheduled or + periodic Backup that should be run. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: ScheduleSpec defines the specification for a Velero schedule + properties: + schedule: + description: Schedule is a Cron expression defining when to run the + Backup. + type: string + template: + description: Template is the definition of the Backup to be run on the + provided schedule + properties: + defaultVolumesToRestic: + description: DefaultVolumesToRestic specifies whether restic should + be used to take a backup of all pod volumes by default. + type: boolean + excludedNamespaces: + description: ExcludedNamespaces contains a list of namespaces that + are not included in the backup. + items: + type: string + nullable: true + type: array + excludedResources: + description: ExcludedResources is a slice of resource names that + are not included in the backup. + items: + type: string + nullable: true + type: array + hooks: + description: Hooks represent custom behaviors that should be executed + at different phases of the backup. + properties: + resources: + description: Resources are hooks that should be executed when + backing up individual instances of a resource. + items: + description: BackupResourceHookSpec defines one or more BackupResourceHooks + that should be executed based on the rules defined for namespaces, + resources, and label selector. + properties: + excludedNamespaces: + description: ExcludedNamespaces specifies the namespaces + to which this hook spec does not apply. + items: + type: string + nullable: true + type: array + excludedResources: + description: ExcludedResources specifies the resources + to which this hook spec does not apply. + items: + type: string + nullable: true + type: array + includedNamespaces: + description: IncludedNamespaces specifies the namespaces + to which this hook spec applies. If empty, it applies + to all namespaces. + items: + type: string + nullable: true + type: array + includedResources: + description: IncludedResources specifies the resources + to which this hook spec applies. If empty, it applies + to all resources. + items: + type: string + nullable: true + type: array + labelSelector: + description: LabelSelector, if specified, filters the + resources to which this hook spec applies. + nullable: true + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector + that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, + NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. + If the operator is In or NotIn, the values + array must be non-empty. If the operator is + Exists or DoesNotExist, the values array must + be empty. This array is replaced during a + strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. + A single {key,value} in the matchLabels map is equivalent + to an element of matchExpressions, whose key field + is "key", the operator is "In", and the values array + contains only "value". The requirements are ANDed. + type: object + type: object + name: + description: Name is the name of this hook. + type: string + post: + description: PostHooks is a list of BackupResourceHooks + to execute after storing the item in the backup. These + are executed after all "additional items" from item + actions are processed. + items: + description: BackupResourceHook defines a hook for a + resource. + properties: + exec: + description: Exec defines an exec hook. + properties: + command: + description: Command is the command and arguments + to execute. + items: + type: string + minItems: 1 + type: array + container: + description: Container is the container in the + pod where the command should be executed. + If not specified, the pod's first container + is used. + type: string + onError: + description: OnError specifies how Velero should + behave if it encounters an error executing + this hook. + enum: + - Continue + - Fail + type: string + timeout: + description: Timeout defines the maximum amount + of time Velero should wait for the hook to + complete before considering the execution + a failure. + type: string + required: + - command + type: object + required: + - exec + type: object + type: array + pre: + description: PreHooks is a list of BackupResourceHooks + to execute prior to storing the item in the backup. + These are executed before any "additional items" from + item actions are processed. + items: + description: BackupResourceHook defines a hook for a + resource. + properties: + exec: + description: Exec defines an exec hook. + properties: + command: + description: Command is the command and arguments + to execute. + items: + type: string + minItems: 1 + type: array + container: + description: Container is the container in the + pod where the command should be executed. + If not specified, the pod's first container + is used. + type: string + onError: + description: OnError specifies how Velero should + behave if it encounters an error executing + this hook. + enum: + - Continue + - Fail + type: string + timeout: + description: Timeout defines the maximum amount + of time Velero should wait for the hook to + complete before considering the execution + a failure. + type: string + required: + - command + type: object + required: + - exec + type: object + type: array + required: + - name + type: object + nullable: true + type: array + type: object + includeClusterResources: + description: IncludeClusterResources specifies whether cluster-scoped + resources should be included for consideration in the backup. + nullable: true + type: boolean + includedNamespaces: + description: IncludedNamespaces is a slice of namespace names to + include objects from. If empty, all namespaces are included. + items: + type: string + nullable: true + type: array + includedResources: + description: IncludedResources is a slice of resource names to include + in the backup. If empty, all resources are included. + items: + type: string + nullable: true + type: array + labelSelector: + description: LabelSelector is a metav1.LabelSelector to filter with + when adding individual objects to the backup. If empty or nil, + all objects are included. Optional. + nullable: true + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement is a selector that + contains values, a key, and an operator that relates the + key and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of string values. If the + operator is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. This array is replaced during a + strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels map is equivalent to an element + of matchExpressions, whose key field is "key", the operator + is "In", and the values array contains only "value". The requirements + are ANDed. + type: object + type: object + snapshotVolumes: + description: SnapshotVolumes specifies whether to take cloud snapshots + of any PV's referenced in the set of objects included in the Backup. + nullable: true + type: boolean + storageLocation: + description: StorageLocation is a string containing the name of + a BackupStorageLocation where the backup should be stored. + type: string + ttl: + description: TTL is a time.Duration-parseable string describing + how long the Backup should be retained for. + type: string + volumeSnapshotLocations: + description: VolumeSnapshotLocations is a list containing names + of VolumeSnapshotLocations associated with this backup. + items: + type: string + type: array + type: object + required: + - schedule + - template + type: object + status: + description: ScheduleStatus captures the current state of a Velero schedule + properties: + lastBackup: + description: LastBackup is the last time a Backup was run for this Schedule + schedule + format: date-time + nullable: true + type: string + phase: + description: Phase is the current phase of the Schedule + enum: + - New + - Enabled + - FailedValidation + type: string + validationErrors: + description: ValidationErrors is a slice of all validation errors (if + applicable) + items: + type: string + type: array + type: object + type: object + version: v1 + versions: + - name: v1 + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/pkg/generated/crds/manifests/velero.io_serverstatusrequests.yaml b/config/crd/bases/velero.io_serverstatusrequests.yaml similarity index 100% rename from pkg/generated/crds/manifests/velero.io_serverstatusrequests.yaml rename to config/crd/bases/velero.io_serverstatusrequests.yaml diff --git a/pkg/generated/crds/manifests/velero.io_volumesnapshotlocations.yaml b/config/crd/bases/velero.io_volumesnapshotlocations.yaml similarity index 100% rename from pkg/generated/crds/manifests/velero.io_volumesnapshotlocations.yaml rename to config/crd/bases/velero.io_volumesnapshotlocations.yaml diff --git a/config/crd/crds/crds.go b/config/crd/crds/crds.go new file mode 100644 index 000000000..ea51c0d1f --- /dev/null +++ b/config/crd/crds/crds.go @@ -0,0 +1,69 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by crds_generate.go; DO NOT EDIT. + +package crds + +import ( + "bytes" + "compress/gzip" + "io/ioutil" + + apiextinstall "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/install" + apiextv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" + "k8s.io/client-go/kubernetes/scheme" +) + +var rawCRDs = [][]byte{ + []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xecwà\x18C\xe7\xf2(\xf3J(\x90\xday\xa1\xb3@\x87\xa8q:\xa5\x03\xc6\xc5\xd9\xc36\xb8\xb5\x843\xf1\xbe\xe3\xe2\x8cF0\x16\n\xf2\xe6\xfd\xa9n\x10>\x8c\x92\xbb\x15\xe4kLPC[)t\xf1E9{\xceƮ\xafG\x00\xd7R\b{\xbf\x12[T\xe0Pa\xe6\x8d\x1dbôPØ\xf7Q#\xbc\x1b\xf0V\x8d\xff%\x12ێʌ\xc2\x04x=\xc8\xec\x10\xb6e\xd2\x17\x86\x02\xb9A\xc7\xf6+\xcaR\xbd\x0f\x13\aӒ\x0ec\u0084\x9b1i̧\xb0\xfaf\u074cY?\u05cc\x19\x8f\xd7\xe5e-\xfa\xdf\x0f+\x93\xe3>[17\xbd\x85\x1f\xa9\x98\xc4DI\xa1\xf5f\aX\x94\xfe\xfd\x1a\xa4OO)\x92\x10\x9c\x18\x8e\xb2\xa7~\xf7oN\x10\xe7\xea\xf4\xe6t\xdd\a\xea\xf4/\x94B\xfd\xeaߌ\x10\xd8\xd9?F_\xbfP\x00?\xb5\xd7\\\x83\xdc\xd5\x02ȯa'\x95G{\"\x89)rʹ$~)\v\xe6w*\x1a\x85\xf0\xd9\xe1\xfe\x8d\xa2#\xd7\xd4s\x16q\xe3ti\x88)ST\xdd\xddL'\xa1\x02'\x83\xd2b\x11R\xcc'\xe6`\xf3\x84#\x9f\xcf_\xbf`>\xce\x14X\xa2a=\x12>\x9f\xa0\xd9~m\f\x91\x97\x11\x10\x83\x94:\xbb\b\xe5\x82k\x10\xf0\x82\xef!\xba\x10\x1aH \x82^C\x93g!Z\xe4\x9a\x05+\xd4\v\xbe3\x90X\x86\x98Y\xbbL\xf4a\xbc\xe0\xfb\xfc\xa4\x13\xb6\x116\xd2Ų\n\xf1\x8f\x1e0\x038\x87]\xca2\xe0\"R\xf20sD\xc1R\x17\x91F\xe2\xf6\xd9\xe4\xd5bj\xea\x1eA\x90W.\b\x85\xb4\xfd \xcbE\x04\x92\xeb\x04\x87l\x13\xa9\x88\xf4,\x94\xcc\xeb\xd7\x04\xfd\xde\xe8k\xf8j\xfcF\x8f\x05\xab\xddq\xff&]\xac\xdd}1\xe8\xbe\x1a\xcfO>\x9c\x89\x01\xe5\xb3Y\x18\x96\xb1\t\xe9\xe0\x86\x89\xfev-jV\x89\xc3\u0604\f\xab\x16\x89t\xb0єC\x04^\x85jbxٔ\xb7\uf3a2r\\l\xd2F\xafx\xb3[\x0f\xbd'\xb2x\xa1\"\xb7\xa5\xd0G\xab~ex\xdd\"\x88O\xb4/\x84ա2\xaaD\x869\xe4\x153\x91+{\xc2\xe3^fP\xa0ݏo\x04\xedQ\x92\xcf^\xf2\xfaE\xbe4\x8c\xb3\xf4i\xc9֜Ft\xc6\xf9\x1c\x1a+\xb2\xcd\xd99I\xb43\x13\aKy\xe3\x13\xe7\xe8\xe0M\x92\xe3\x86\x19n\x8a<\xe7\x83\x16\xa1\x1e\x16{\xefŜ\xef\xef\xdb\x01\xa5\xb0\xc7\x15\x82\vt\xffK[\x15+\xed\xffA)\xa4\x9d\xb5\xd0\xcf|b\xa2\xb0\xb32V\x85\xda/!\xf8\xd2\x01I\xf3(\xd4iAx\x80,C^\x03U؆ͮ\x17i\\\xc3\xeb\xc1\xb8\xb0+\xee$\xaa\x1c\xe4T\xa4E\xe3\xf2\x05\xdf/\xaf{6~\xb9їa{\xeeYl\xda\xcbg\x00\x1b\xad\xde\xe1\x92W^~{\xe8\xb2H\xeb\x16L\xe2\xe3\xb3e\xc1,esi\x17\xa7e\xf5\x19\f\x85\xa2\xe3\xd8.й\xd28\xbf\x10\x89\a\xe3|\xa8\xd0u\x82ǁ\xda\xd0tN\x13kB v\xe1\xdc\xcb\xd8t\xc2A\x8e\xec\xa4TIRr8X\xe0\xecA\xcc#H\xa1\x14\\66\x1a\xfc\xe3e8\xf6\xe0W\x88\x8cÂ\t\x88\xa4\n\xa55\x19:7\xa5\x0e\xb3\x9ew\xa6\xe0V\x17\xdbDH*\xc2!\xc2Tq/\x8d\xa5a#\xb1\xe6\xac0\xfb\xfe\xadU\x03$Ӧ\xff\xa7\xd5\xec<\x8c\x80Ϡ\x8bB\xe8\xd9͢\x87\xdc]X\x97L!\x82\t!\xbb\xddWl\xc6K#\xbd\xa84\xbf\xee\x06[H\xbda\xe0\xf0\xe9C\xb7cH.\x11\xcf\x0f\xa9\xef\xd2ʆ\xcd\xf5\x83`\x9b\xa5\xe9\x97܇\xc6\xeb\x01-v$կ\fs8\xa7\x8do\xa5\xe7\xcb\x18\x1d\xf0\xb8r\xb0\x93\xd6\xf96\x92\x8e\x0f\xb6>>G\xd1\xf7\xd6~C\x8a\xf2簮U\x00:\x98\xd7tR8r874\xf8\x18\x04A\xee@z@\x9d\x99Js\x11\x83\x8c\x94_\x10X\x1a\x9c\xe9\xec&\x1b\xc6\x12æ\x81\xba*\x96\x10\xbeb\xed\x91z\xa2\xd6ў\xfc\xa3\x90S\x95\xaa4\xce\x12\x93\x97\x05\x9ajbSkFGLOa]爷\x10o\xb2\xa8\n\x10\x051{\x11Gig\x96\x05v\xe5\v\xafBz\xf6\xee\x04\x95]\xbd7d\x14\xa5B\xbf,\x1b\xd8\xe2\xceX\xb6E's\xac\xb7\xcc(s\xa3A\xc0NHU\xd9E\x1e\xed\f\x8e.\x8f죑\u007fLо\xe4\xb5+&\u007f\xb6L\xb9(T\x9b\xf2\xaa\xa5]\x1a\xa8=X\xfc\xc8\x10\xa9\xb4\x92t\xc6|l\x94\x14UI\xe8\xf7\xefaR\x8b7\xdfä\xde\xf8\x1e&u\xc6\xf70\xe9{\x9849\xbe\x87I\xdfä\xdfk\x984\x8dɊ\xebV\x83?ͼ}\xf6\bu\x1c\xb1Q\xc8\xf1T\xff.\xf4^/\xeb\xcb\xdb\f\xaf\x19軌-\xdd+\xee8\xef˹9\xfao\xdc|ݨGʟ\x9474\x96N\xb6\xee-h\xc4\x1b\xea͜o/\x99k*\xe9\xf6$֍\x1d\xa9)ѤW\xf4\xa8O\x9d\xec\x14f\xb6;\x18\x84R\xed\xde\x14a\x1b\xa6\xfcJ\xfd\x8a\xb3\xad\x1f3\r\x1f\xd3m\x9b\xe3\x1c:\t\xed\xbb,\xb2\x9d\x16\xc3_\x99C\x93}\x19\xe3\xdd\x18\xf1$\x03\xbd8~Zw\u007f\xf1&\xf6f\xc0\xab\xf4\x87\x1e\x01\xdc4I)\x8b\u07b7\x9b#\x93N\xc5\xeb\x03\xa7\x9c\x03cAKu=\xd8\x17S߬h\xb3\x13\xfe\\\x86\xa4\xe8,{\x9b\n\xed\x97\xf4n|s\xc7F\xb7'c\xd0ɞwر\xb4\x85tyOF\xb7\xe7bd\x93YЉqv\xa7\xc5|\xbe5\xd9U\xf1\r\xbd\x14\xa9ObjÝ\xe8\xa0X\x10s\xccwK|S\x8f\x04\x1f\xe6M`}VgD\xab\xeba\x02\xe4\xb2~\x88\x05,\x99\xeb}8\xbb\xe3\xe1\xb4\xcb`\x82\x88\xb9>\x87\xf1\x1e\x86\t\xa0\x83\xdd\rK:\x17&`\xd6=\r\x1fد0ӥ\xf01\x9d\x84\xbf4\xf6\x1c\xeb9\x98\xe94\x98\x89L\xa7\xb0\x9a\xe9%X\xdeA0ßo\xec\x16\xa8\xfb\x01\x06\xdfyn\x8f@\xb7\v`\x10\xe4\xc2\u0380\x91\xb3\xffA\x90\v\xfa\x01fN\xfc\a\xc1Nn\x8c\x13\x1a1\xfa\x93Ӣt\a\x93.lM\xc6I\x8fݹ\x03\xc9E\xba\xae\x95)S\xe55\xec>)|\x05\xf3\x1d\x1e\x9e\xd9\xc9\xf3U\x98\xac\xb9\b\x14]y\n~N\xef\t\xfd\xf0\x91Ɇ\xf3Ɗ=\xfed\xb2\xd6M\xdb1\xfa\xbbs;\xd7\"\xa3PSJ\x9f\xfa D\xba\x9f\xd7]:\x14;\xc6*[\xbc\xe7\xd6d_\x84a_ޣ\x96罚$\xe2\xe9駀\xb8\x97\x05\xae\xbfT!\x91[\x95\xc2:$\xfe%\x82¢-\xfdy0\xaf=\x84\x95\x89\x94\xfep\x8a\xafE\xae\xe1q\xb6\xb8\x18\xebp\x97/)XbӴ:>\x0f\xafiŢ-\xa1\x84\xc4\xc6\xec\xc6V\xf5\bl]d\xa6h?t\xb4|\xd4\x15\xb5a\xe7<|\xf9\xd3\v_\xb9\xb9\xeb\x9f<)]\xe6\x8e\x15\xdf\xca\xf2\r\xb3\x00 (\xe3\xd97@cy\xabs\xc3~J&w\xfd\xf9|\x95\xda\xe6\x01).\xab\u05579_\x85\xab\vh\x03\x1e\xad\x01\x16\xd6q0@\xb00\a<\xa2\x06\xa3\xb9^\xc67\xb8\xc2=\xff\xd35\xfd\xfc\xb5\x05#\x96\xe3\xaaR\x19\x91'\xcbM\xf7L\xe3\xf5\xf0'\xf6G\xf6\x88\xf6ʍB䫪;c\x87\xc8?լ\x9d\xb1\x85\xf0\xb7\x90\v\x8f\xab\x01\x80\v\xfc\u0600Jq\xf1x\xe6\xea&O\t\xd6\xc1u\xe7t\x976\x14\x9e\vtN\xecӝ\xcdWrG{Դ\xc9\rT\x89b(\xd6\x14.\xbb\xf7\x17CF'2O\xf9o@-\xa6\xb0\xadYW}\x9bSfO\x196O\x8c7ƣ\u007f\x1ev$R{\xdcc7<·R\xday_~_O#\x8ep\xea\xce\x16\xde|@\x01\x95\xdcKr\x88$ؽ\xb0[\xb1\xc7Uf\x14\xe5Q\xd2\xe8S\x8c\xfe6r\rP\a\xbe\x8e\xd0#\xe8\xc7\xf6̔\bFe\x0eP\xd2\xc7\x12\xae\xe3\x8eJ\x12,\xc4_\x8d\xed\x1f\xd6\x14R\x1b\x1b\xc2\x17\x0e\xa1\xd3\xd2\xc5\xfe\x9c\xaf\xb5N\xe2\xfb@3\xeaө\x96\xaf¤L\xc3\xfb\xfc\xd0)\xc6\n\xbe\xe2\xe9\x16\x15\x0e&0\u007f\xae\xbf\xa2ћ\xb0\xd1\x0f\xd6\xec)\xe2\xeb\xfdt\x97\xbcR\xef\x97\aa\xbd\x14J\xbd\a\xf0#o\xed=\xfe\x82\xe4\x17F6\x82!\x06F̦y\x18'5!\xa5\xd4A\xd6|\x98\xb05\x95\xef\x18\\c\xb0=\x89\xa7\xf7\xad)O\xc4T7\x90]\x88\xb4\x03\xa2\xf3+\xdc\xed\x8c\xf5!~]\xad@\xee\xe2\xc6҃Jޙ+_\xe1k\f }\x93\xc55\xbaɱ\xa0E\xe1X7=\u007f\x11\x82\xcb\xce\"\xcb(>\xc1\x1b\xe7\x85\xea\xf9\x80o.w\xf1~Mڅ\xf9_z\xdbY\x8fɛ\xf6\xec\xbaϵ*\xb6hIS\x19X\xe0\x17\x1f\xed\x05\xaf\xa7\x86S\xba-\xa2\x86W+\xbd'\u007f\xd3.\b\x82'\x0f\xa3\x148\x03;1xgx\xdc\xe7\xf1\xaf\xc6\v\xb5\x19Kh\xbb!`=5\x91Ë\xfbD\x19\x12ÖI\x1f$'4wH\x97V\x92ಃ\xd0{R k\xaa\xfd!i\xe0\xc8N1\\ī\b!(U\xb5'\x95\x8e\x855_Y\xdd\xcaLc\xa9-o\xa1*\xb2\x17\xa8\xca\xe1\x93\xe7\xf0\xad\x98\xf8\xa9\x9f\x9bx\x17x\xb5\xb3\xa6XE\xfes\xcd\xec:V\xec\xac4\x142qN\x13\xaf㍀e\xb1\x97%j\x10.\xe22\xdbw2%\xc8\xf1D\xcd\v\xeb\x97\x05a\x8f\x9d\xa93\xf1\x17\xc3\xc5|\r\x8fX\n2\xb6\xbe\x11[S\xc0\xdd釖\xae)GO_\x15\n\x1f\x85\t\xa2\xa7\xbc\x96\xbf\xeea,\x86;x=\x88\x9d\x80\xaa\x13@uQ\xff\xfb\xc4N\xcdw\x96\xee磨\xe7\x93\xc9'\a)d\xc1\r\xbc\x14\xfb\xfcA\xee\xfa\xf9EY*\x99\x11\xb6\u007f\xfc\x95\x0eH\x8e\v\xa2\x8a\xabɀ\x82\xa3\x87:6\x80/XZ\xcc\xc8*\xfb\xc8?(\xa4\xfd\xde!v#\x95\xabŁ]7Et\x9f\xbdǢ\x1cx\xd7D\x8e\xd8,\x1as|\"M\xe8\x11\x90\xbef\x95@\xc5N\x80Ѥp1!u\xa8q\x0e!\xf5\xa21B\\\x95\x91\x03\xdaUC[Q\x9ds} U\xaf\xc2R\xa2=m=\xff\x11'\rd!q\xfd\xc7\xe6!\xad4$\xe1\xf7wJD\x06\xfc\xf8ɣ\xe6cv\x9f\x9a\xff\x98}\xab\xf8\xf1\xbac\xe8'co\x99\xb7L;\xa2\x12\x9f4\x05\x02\x91eH\xba\xfb\xf5\xf4;v\x97\x97\xfcO\xfaT\x1d\xff\x9b\x19\x1d\xf6Rw\v\xff\xf9_\x17\x10\xebL\xcf\t\x0fz\xf8\xff\x01\x00\x00\xff\xff\xff\x8cC\xfd\xf8O\x00\x00"), + []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xbcXM\x8fۼ\x11\xbe\xfbW\f\xb6\x87\xbd\xd4r\x92\xf6P\xe8\xe68-\x90v\x935\xd6\xc9\xf6\xd0\x16\bM\x8elv)R\xe5\x90\u07b8E\xff\xfb\x8b!%ْ\xb5\x1f\x01\u07bc<\x99\xe4\x98|\xe6\x99Oj6\x9f\xcfg\xa2\xd1\xf7\xe8I;[\x82h4~\x0fhyF\xc5ß\xa8\xd0nqx\xbb\xc5 \xde\xce\x1e\xb4U%\xac\"\x05W\xdf!\xb9\xe8%~\xc0J[\x1d\xb4\xb3\xb3\x1a\x83P\"\x88r\x06 \xacuA\xf02\xf1\x14@:\x1b\xbc3\x06\xfd|\x87\xb6x\x88[\xdcFm\x14\xfatCw\xff\xe1M\xf1\xae\xf8\xe3\f@zL\u007f\xff\xa2k\xa4 \xea\xa6\x04\x1b\x8d\x99\x01XQc\t[!\x1fbC\xc1y\xb1C\xe3d\xbe\xab8\xa0A\xef\n\xedfԠLH\x94J\xf0\x84Y{m\x03\xfa\x953\xb1ΰ\xe6\xf0\xd7\xcd\xed\xe7\xb5\b\xfb\x12\n\n\"D*\x9a\xbd L\x90\x15\x92\xf4\xba\t\t\xd8\xfbt\x1fl\xf2\x85p\xd3\xde\b\xf9_@Q\xeeA\x10,\x0fB\x1b\xb15\xb8\xf8jE\xf7;\x9d\x96a\xaf\xfb\xd3ñ\xc1\x12(xmwc(\x1d\x93\xc5\x05\vgG-w\xe7\a)\x11x\xba\xf3.6%\x9ch\xc8ҭ\x11\xb2\x01\xb3*\xad&\x9d\"i\xdfh\n\u007f{Z\xe6FSHr\x8d\x89^\x98\xa7\x8c\x90DH\xdb]4\xc2?!4\x03h<\x12\xfa\x03~\xb5\x0f\xd6=ڿh4\x8aJ\xa8\x84I\f\x91t\xac\xd7gF\xdf\b\x89\x8a\xd7\xe2ַ~\xd7j\x94\xe9/\xe1\u007f\xff\x9f\x01\x1c\x84\xd1*\x1d\x9f7]\x83v\xb9\xfex\xff\x87\x8d\xdcc-\xf2\xe2\xa4]G\x8a\x82&\x10\xd0a\x85\xc7=z\x84\xfb\xc4)\xb0\"H\xadV\xed\x89\x00n\xfbo\x94\x81\xday\xe3]\x83>\xe8\x0e%\x8f\xb3(\xeb\xd7FX\xae\x19l\x96\x01\xc5q\x85\x04a\x8f\xd0F\a*\xa0\xa4\b\xb8\n\xc2^\x13xL$\xdap2a\x0f\xa8\x02a[X\x05l\x98hO@{\x17\x8d\xe2`<\xa0\x0f\xe0Q\xba\x9d\xd5\xff\xedO&\b.]iD\xc0\xd6\xd8\xddH\xc1c\x85a\x9a#\xfe\x1e\x84UP\x8b#x\xe4; ڳӒ\b\x15\xf0\xc9y\x04m+W\xc2>\x84\x86\xca\xc5b\xa7C\x97W\xa4\xab\xebhu8.Rv\xd0\xdb\x18\x9c\xa7\x85\xc2\x03\x9a\x05\xe9\xdd\\x\xb9\xd7\x01e\x88\x1e\x17\xa2\xd1\xf3\x04\xdc\xe6P\xaf\xd5\xefzg\xb8>C:\n\xac<\x92\xe7?\xc9;\xfb|\xb6y\xfe[\xc6\u007f\xa2\x97\x97\x98\x95\xbb?o\xbe@wi2\xc1\x90\xf3\xc4\xf6\xe9ot\"\x9e\x89ҶB\x9f\rWyW\xa7\x13Ѫ\xc6i\x1b\xd2D\x1a\x8dvH:\xc5m\xad\x03[\xfa?\x11)\xb0}\nX\xa5\xec\n[\x84\xd8pܫ\x02>ZX\x89\x1a\xcdJ\x10\xfetڙa\x9a3\xa5/\x13\u007f^\x14\x86\x82\x99\xad~\xb9\xcbד\x16\x9a\x8c\xd2M\x83r\x10'\nI{\xf6\xe5 \x02\xa6\bh\x83v@\xe9\xd3\xe9\xef\xe9\xe0M\x01,%\x12}r\n\x87\xeb#\xa8\xcb^l\x80\xadA_kJe\x15*\xe7\xd3Z\xce!Ц\xc6ѡ\xd0\xe7\x9fb\xb4\x836\xd6c\bs\xb8C\xa1n\xad9Nn\xfc\xdd\xeb0\xbe`\xd2\\<2\xac\xcd\xd1\xca5z\xedԳ\xea\xbe\x1f\t\xf7J\xef\xdd#T\xc9mm0G\xce+t\xb4r\x9c7\xbb\xb1\\\u007f\xecrh\x0e\x8e6\x96Zn\nX\xb61\xe9*x\x03J\x13\xd7UJG\x8e\xe9\xe16\x81wK\b>\xbeZi\xe9l\xa5wcUϛ\x87i\xafx\xf6\xd0\x11W\xabt\a'\x1a\xf6\x80ƻ\x83V\xe8\xe7\xec\xf9\xbaҲ\xc5\x10}\xae:U*\x88c\xed&c\a\xfa\xe4Ӻ\xf5\xb3&\xbb=\x97<52\x19E뮄\x81\xb3\x1e\x81Evg\xe1\xc7~\x05lQ\xe9\xace+\x05\a\xa2\xd7\xe7\x9a\xc6\xc6\x1b\xfd\xf5\xa9\x00㱍\xf2\x01\xc3\xe5\xfa\xd8\xeb\x92\x183\x99\xe2(ς\x83H\x98\xb8}\x1e\xc0\v6\x03\x90b\x85\xfee\x14\xab%\x8b\xf5\x1e/`\xb5\x84m\xb4\xca`\x87\xe5q\x8f\x96˷\xae\x8e\\C\xbe\xdcl&΄\x8eǔ\x1c\xda\x02ܱ9\x85\xbdr\xbe\x16\xa1\x84\xed\xf1\"\xa8_T\xad\xf1X\xe9\xef/\xaa\xb6Nb\x1d\xc1\x8d\b{Ж\xb4B\x10\x13tOd\xd9n\xf4\x01|\xdb\xe4@\xfaAcp\x06\xe1\xa4~\x99\xf02\x8c׆G\xc7糑\xb1n\x85z\xbd\xbbyj\xb8\xc6\t{:4/\xb4\x98\xd2`>\f\xd7\xc1Nw\xe9\x8b\x053\xb7\xbe?V2\xf3k\xe5\xbc0\xb9m\xea\xc2Ϫ\xe6O.\x90Wg\x15\x92{.\v\xd1FB\x95\x13^\x01\xff\xb4\xf0\x81;(ɝM\xc9\x18\xb9\x99\xa1\v\xb7\xb1\xee\x91\xff|vZ:\x00\x9cMz\xa5\xee\x80{\xd4\xdcp\xa5\xadGm\f\xb7M\x1ekwH/\x8a\xe1\xe0\x1eǣ9\xf2K\xceUpxW\xbc)\xae~\xe3\xeak\x04\x05.\xa7\xa8\xee\xf0\xa0\xc7\xef\x85K6o.\xe4;\xef\xed\v&O\xbeu\x8d\xd8·b\xdf.ԯ\xb4\xe1\xb6q\xc2\xd5Oo\xa1\xfc8\xa0\x00Aטf\xef77ה\xde\xf7\xdc\xf2^\x1c\xfa\xc8\xe6\xa3\x04\x90\x9f\x10\xae\xedt#\x05\xf4\x13\xc6\xeem\xa5\t\xac\x03\xe3\xecn\x10\ny\xb4}/8\x0f\xd9u\x9c\a\x85ܲr\x9a\x95{awxz˴\xd8\xcfP\xb2c\\\"\x1dz\xc7\xc9\x1b\xb4\x9dv\x85Wؐ\x1f\xee\xcf\xda\xeff ڙn\xc8p\x8f\xba\xb5\xa5\xb9\x8c\xc9Wp=\x92\xee\x8a\b\x139\xe7\xab~\x956*}=y>DzD\xa7\xa7\x8cޣ\r\xa7ܓ\x1cj*\xff\xbc\xae\t^\x0e>\xb9\x9c\xef\x8c?Ǽ\xa8\xcbD\xce\x1d-\x9d>Z\xbd=\xcd\xda\xefJ\xf9\x13M\xda\x00\xc8\t\xf6\x8c\xc86\xaaڕS\"\xe7\f\xda\x04T\x9fǟl\xae\xae\x06\xdf]\xd2T:\x9b\x9bS*\xe1\x1f\xff\x9a\xe5SQ\xddw8x\xf1\x97\x00\x00\x00\xff\xff\xbd\xf1$\xc1\xe0\x13\x00\x00"), + []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xb4\x96\xcdr$5\f\x80\xef\xfd\x14\xaa\xe5\xb0\x17\xa6\x87\x00\a\xaao\x90]\xaaR@*\x95,\xb9P\x1c<\xb6fZ\xc4m\x1bI\x9e\x10\x9e\x9e\xb2\xbb;\xf3\x93Ne9l\xdf,\xcb\xfa\xf9$\xb9ݬV\xab\xc6$\xbaG\x16\x8a\xa1\x03\x93\b\xffQ\fe%\xed\xc3\x0f\xd2R\\\xef/6\xa8\xe6\xa2y\xa0\xe0:\xb8̢q\xb8E\x89\x99-~\xc0-\x05R\x8a\xa1\x19P\x8d3j\xba\x06\xc0\x84\x10\xd5\x14\xb1\x94%\x80\x8dA9z\x8f\xbc\xdaah\x1f\xf2\x067\x99\xbcC\xae\x1ef\xff\xfbo\xdao\xdb\xef\x1b\x00\xcbX\x8f\u007f\xa2\x01E͐:\b\xd9\xfb\x06 \x98\x01;p\xe8Qqc\xecCN\x8c\u007fg\x14\x95v\x8f\x1e9\xb6\x14\x1bIh\x8b\xe3\x1dǜ:8l\x8c秠Ƅ>TS?US\xb7\xa3\xa9\xba\xebI\xf4\x97\xd74~\xa5I+\xf9\xcc\xc6/\aT\x15\x84\xc2.{Ë*\r@b\x14\xe4=\xfe\x1e\x1eB|\f?\x13z'\x1dl\x8d\x17l\x00\xc4Ƅ\x1d\\\x97\xa8\x93\xb1\xe8\x1a\x80\xbd\xf1\xe4*\x9e1\x8f\x980\xfcxsu\xffݝ\xedq0\xa3\x10\xc0\xa1X\xa6T\xf5\x96r\x00\x1200E\x02\x1a\xa7\x00!\x06\x84\xc80DF\x18\xa3\x95v2\x998&d\xa5\x99`\xf9\x8e\xfa\xe7Yv\xe6\xfc}\x89n\xd4\x01W:\x06\x05\xb4G\x98\xea\x8e\x0e\xa4F\x0eq\vړ\x00c\xc5\x12\xc6\x1e:2\vE\xc5\x04\x88\x9b\xbf\xd0j\vw\x05\x1d\vH\x1f\xb3w\xa5\xcd\xf6\xc8\n\x8c6\xee\x02\xfd\xfblYJ~ť7:\x17x\xfe((r0\xbep\xcd\xf85\x98\xe0`0O\xc0X|@\x0eG֪\x8a\xb4\xf0[\x81Ca\x1b;\xe8U\x93t\xeb\xf5\x8et\x9e\x18\x1b\x87!\aҧu\xed{\xdad\x8d,k\x87{\xf4k\xa1\xddʰ\xedI\xd1jf\\\x9bD\xab\x1ax\xa8\x03\xd3\x0e\xee+\x9e\xc6K\xde\x1fE\xaaO\xa5\x13D\x99\xc2\xeeY\\{\xf8U\xee\xa5\u007f\xc72\x8f\xc7\xc6\xf8\x0fx\x8b\xa8P\xb9\xfdx\xf7\tf\xa7\xb5\x04\xa7\xcc+\xed\xc319\x80/\xa0(l\x91\xc7\xc2m9\x0e\xd5\"\x06\x97\"\x05\xad\v\xeb\t\xc3)tɛ\x81T\xe6\xf6+\xf5i\xe1\xb2\xde\x1b\xb0A\xc8\xc9\x19E\xd7\xc2U\x80K3\xa0\xbf4\x82_\x1c{!,\xab\x82\xf4m\xf0\xc7\xd7ݩ\xe2H\xebY<\xdfE\x8b\x15Z\x18˻\x84\xb6Ԭ\x80+giK\xb6\x8e\x01l#\xc3cO\xb6\x9f\xc7\xf2\x84\xe8\xf3\x00\xb7G⥁-\xdfh\xa0\xdc*\xa7\xf2W\x92\x85Z'b<\xe9\xb5Ց\x997)\xa8\xd1,\xff\x8bC=1\x93\xb0\x99\x19\x83Nv\xea-\xb0t\xe8srG\xe6\xc8r\x9e\xf7I8\x1f\xabJ\xfdk\x19\n\x02&t\xb1\xb1\x16\x93\xa2\xbb>\u007fN\xbc{w\xf2.\xa8K\x1b\x83\xa3\xf15\x04\u007f\xfcٌV\xd1\xdd\xcfq\x14\xe1\u007f\x01\x00\x00\xff\xff\xcb0\x9b\f\x8c\t\x00\x00"), + []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xb4WOoܶ\x13\xbd\xebS\f\xf2;\xe4W \xd26m\x0f\x85n\xad\x93\x02A\xd3 \xb0\x13_\x8a\x1e\xb8\xe4\xacĚ\"Y\xcep\x1d\xf7\xd3\x17CI\xfbG+\xdb\xe9\xa1{\xd3p8|||oȭ꺮T\xb4\xb7\x98\xc8\x06߂\x8a\x16\xbf0z\xf9\xa2\xe6\xeeGjl\xd8\xec_o\x91\xd5\xeb\xea\xcez\xd3\xc2U&\x0e\xc35R\xc8I\xe3\x1b\xdcYo\xd9\x06_\r\xc8\xca(Vm\x05\xa0\xbc\x0f\xac$L\xf2\t\xa0\x83\xe7\x14\x9c\xc3Tw蛻\xbc\xc5m\xb6\xce`*+\xcc\xeb\xef\xbfm\xbek~\xa8\x00t\xc22\xfd\x93\x1d\x90X\r\xb1\x05\x9f\x9d\xab\x00\xbc\x1a\xb0\x05\x13\xee\xbd\v\xca$\xfc+#15{t\x98BcCE\x11\xb5,ڥ\x90c\vǁq\xee\x04h\xdc̛\xa9\xcc\xf5X\xa6\x8c8K\xfc\xeb\xda\xe8{;eD\x97\x93r\x97 \xca Y\xdfe\xa7\xd2\xc5p\x05\x10\x13\x12\xa6=~\xf6w>\xdc\xfb_,:C-\xec\x94#\xac\x00H\x87\x88-|\x10\x94Qi4\x15\xc0^9k\n\x15#\xee\x10\xd1\xff\xf4\xf1\xdd\xed\xf77\xba\xc7A\x8dA\x00\x83\xa4\x93\x8d%o\x89\x1b,\x81\x82\t\x05p8\x00\x03\xe5A%\xb6;\xa5\x19v)\f\xb0U\xfa.ǩ&@\xd8\xfe\x89\x9a\x818$\xd5\xe1+\xa0\xac{PRmL\x04\x17:\xd8Y\x87\xcd4%\xa6\x101\xb1\x9dY\x96߉\xbe\x0e\xb1\x05\xe0\x97\xb2\xa31\a\x8c(\n\t\xb8G\x98t\x81\x06\xa8\xec\x16\xc2\x0e\xb8\xb7\x04\t\v\x95~\xd4\xd8IY\x90\x14\xe5'\xe4\r\xdc\b݉\x80\xfa\x90\x9d\x11\x19\xee11$ԡ\xf3\xf6\xefCe\x12^dI\xa7x\x16\xc2\xfc\xb3\x9e1y\xe5\xe4,2\xbe\x02\xe5\r\f\xea\x01\x12\x16v\xb2?\xa9VR\xa8\x81\xdfBB\xb0~\x17Z\xe8\x99#\xb5\x9bMgyv\x94\x0eÐ\xbd\xe5\x87M\xf1\x85\xddf\x0e\x896\x06\xf7\xe86d\xbbZ%\xdd[F\xcd9\xe1FE[\x17\xe0\xbe\x18\xaa\x19\xcc\xff\xd2d?zy\x82\x94\x1fD=\xc4\xc9\xfa\xee\x10.:\u007f\x94w\xd1\xf9(\x8fqڈ\xffH\xaf\x84\x84\x95\xeb\xb77\x9f`^\xb4\x1c\xc19\xe7\xa3N\x0e\xd3\xe8H\xbc\x10e\xfd\x0e\xd3xpEeR\x11\xbd\x89\xc1z.\x1f\xdaY\xf4\xe7\xa4S\xde\x0e\x96i\x96\xad\x9cO\x03W\xa5\xaf\xc0\x16!G\xa3\x18M\x03\xef<\\\xa9\x01ݕ\"\xfc\xcfi\x17\x86\xa9\x16J\x9f'\xfe\xb4\x1d\x9e'\x8el\x1d\xc2s\xbfZ=\xa1\x85\x95o\"j9/!M\xe6ٝ\xd5\xc5\x02\xb0\v\t\xd4\xd1\xd9\x13m\xcdI\xdd5o\x16P*u\xc8\xe7\xb1\x05\x8aO%E\x16\xbe\xef\xd5y\v\xf9?6]#}\x80&\bcg\xf8\xa6Y\xd4{l\xf55\x8d\xaeb\x98\xa5*[\x17\x1e\xc5\xe8\xd2zN\xd1,\x17\x95\x1f\xfa<\xac\x15\xaf\xe1\xe7\x82\xf4}\xe8\x9e\x18\xbd\n\x9eE\xd0O\xa4\xdc\x06\x97\a\xbc\xf1*R\x1f\x9e̜/\xcd\xc3E\xb2L\xbbFi\xb5\xf8\x18\xa4i\xf8\x1a)\xbbՅV\x858\xff\xca\xc5\xf9\x1c\xcbr\xf7\xcc,˄\xb1\xe3\"ȅ\x9d<2ұ\r\xdc[\xeeᾷ\xba_\xa9\neZ9 \xe9/DA\xdb\xe2\xd8\u007f\a[tl\x13^ȣ.\xa2\xb9\b\n\xe4j\xad\xf8\xc2s\xeb\x85\xeb\xc9\v\xcf:\x96\x15g\xfajϖ\xec\x99T\x9dSB\xcfS\x8dr[-'|\x8dig\xc5\u007f\xbe~\xff\xa4s\xdf\x1c\xf3\xca\x1bLY?\xe2\x88\tk\xb2\x9dܭ2&\xde-\xceZ\x120\xfeN\xef\xf8gO\r\xbfD\x9bN\x9e,\x8f@{{H\x1b\x1b\v\xfa\xf1\x8aX\xbe^J9\xa4r\xedj\xe5/\xb0m\x11\f:d4\xb0}\x18;\xe3\x031\x0eK\xbc\xbb\x90\x06\xc5-\xc8\xc5Q\xb3\xbd\x10\x8a\xbc/\xd5\xd6a\v\x9c\xf2\xba\x8aV6\x1b{E\x17\xb6:\xdb\xe7G\xc9X;\xfe\x83\xb9\x9e8\u007fx\xa4\x83\xd5\xf0\x01\xef/b\x1fS\xd0H\x84Kc<\x82~E܋\xd0\xf1a\xfe\xfa\xf8U\xa4XO\x0f\xf12\x00P\x9e\xb5愺\xe9\xcd8E\x8e\x8eQZcd4\x1f\x96O\xf1\x17/\xce\xde\xd6\xe5S\ao\xec\xf8/\x02~\xff\xa3\x1a\xab\xa2\xb9\x9dqH\xf0\x9f\x00\x00\x00\xff\xff\xbbظ3\xc4\f\x00\x00"), + []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xc4Y_s۸\x11\u007fק\xd8\xc9=\xb87\x13RMz\xd3\xe9\xe8\xedb7\x1d\xb7w\x8e'\xf2\xe5%\x93\x87\x15\xb1\x12Q\x83\x00\x8a\x05\xa5\xa8\x9d~\xf7\xce\x02\xa4$J\xb4,_{)_l\x82\x8b\xc5\xfe\xdf\xdfB\x93\xa2(&\xe8\xf5'\n\xac\x9d\x9d\x01zM_#Yy\xe3\xf2\xf1O\\j7]\xbfYP\xc47\x93Gm\xd5\f\xae[\x8e\xae\xf9H\xec\xdaP\xd1\r-\xb5\xd5Q;;i(\xa2\u0088\xb3\t\x00Z\xeb\"\xca2\xcb+@\xe5l\f\xce\x18\nŊl\xf9\xd8.h\xd1j\xa3(\xa4\x13\xfa\xf3\u05ff/ߖ?L\x00\xaa@i\xfb\x83n\x88#6~\x06\xb65f\x02`\xb1\xa1\x19x\xa7\xd6δ\r-\xb0zl=\x97k2\x14\\\xa9݄=Ur\xe8*\xb8\xd6\xcf`\xff!\xef\xed\x04\xca\xca\xdc;\xf5)\xb1y\x97ؤ/Fs\xfc\xdb\xd8ן4\xc7D\xe1M\x1bМ\n\x91>\xb2\xb6\xab\xd6`8\xf9<\x01\xf0\x81\x98\u009a~\xb1\x8f\xd6m\xec{MF\xf1\f\x96h\x98&\x00\\9O3\xb8\x13)=V\xa4&\x00k4Z%Sd\xb9\x9d'\xfb\xe3\xfd\xed\xa7?̫\x9a\x1a̋\xc2\xd9y\nQ\xf7\xea\xc9s\xe0\xd8\xdd\x1a\x80\"\xae\x82\xf6\x89#\\\t\xabL\x03J\\I\f\xb1&\xe8\x1cB\n8\x1d\x03n\t\xb1\xd6\f\x81\x92\x0e6;\xf7\x80-\b\tZp\x8b\xbfS\x15K\x98\x8b\x9e\x81\x81k\xd7\x1a%\xfe_S\x88\x10\xa8r+\xab\xff\xb9\xe3\xcc\x10]:\xd2`\xa4ξ\xfd\xa3m\xa4`ш\x11Zz\rh\x154\xb8\x85@r\x06\xb4\xf6\x80[\"\xe1\x12~v\x81@ۥ\x9bA\x1d\xa3\xe7\xd9t\xbaұ\x0f\xe5\xca5Mku\xdcNS@\xeaE\x1b]\u0a625\x99)\xebU\x81\xa1\xaau\xa4*\xb6\x81\xa6\xe8u\x91\x04\xb7)\x92\xcbF}\x17\xba\xb8\xe7\xab\x03I\xe3V\xdc\xc61h\xbb\xda-\xa7\x00{\xd2\xee\x12`\xa0\x19\xb0ۖ\xe5ߛW\x96\xc4*\x1f\xff<\u007f\x80\xfe\xd0䂡͓\xb5\xf7\xdbxox1\x94\xb6K\n\xd9q\xcb\xe0\x9ađ\xac\xf2Nۘ^*\xa3\xc9\x0e\x8d\xce\xed\xa2\xd1Q<\xfd\x8f\x968\x8a\u007fJ\xb8N\t\r\v\x82\xd6+\x8c\xa4J\xb8\xb5p\x8d\r\x99kd\xfa\xcd\xcd.\x16\xe6BL\xfa\xbc\xe1\x0f\xebА0[k\xb7\xdc\x17\x8aQ\x0f\x1d\xe5\xfe\xdcS%\xfe\x12\xa3\xc9>\xbd\xd4UJ\x01X\xba\x00xL^\x1e\xb0\x1dKMyrU\x98G\x17pE?\xb9\xea ɟ\x90\xe9\xdd؎^*\xa9m9M\xa9c\r\x9c)\x8fX\x02\x98~릦@iG \x8e\xba\x92@r\xac\xa3\v[a+\xfbI\x95G\xfbG\x8d.\x8fu\x8a\xce\xca\u007f\xe7\x14\x8d\x89+\x1b!֘c\xf2ޥ\xcc\b\xad\xb5\x92\x05\xce^,\x80w\xea\xec\xf9\x1dg\x84@K\nd%\xa3r\xf1\xf1.\x95\xa8\x88\xda\xf6\x99\x97K7Dwb\xbeE60)\x18:\xfa\x9c\xb3\xe1\xc9z<*\xe9\x8f\xf7\xb7}\r\xee\x8d\xd4\xc9\x1c\x8fO|\xb8\xf90\xcbRI\b\xadR%\x95.\xb7Ԃ9\x04l\xe4\xce)1\x99\xcc\xd1\xe6\xe0\x88\x0e\xaa\x1a\xedHa\x85\x04Z\x92u\x97\xad\xf4\xb2\xf2\xea\xa5\xd9z\f\x1b\xfag\x04>\x1c\x17\x86\xffS\x13\xbeH\xad\x84ڟU\xeb\xee \x9eϪ%\xf3C\xb0\x14)i\xa6\\ŢTE>\xf2ԭ)\xac5m\xa6\x1b\x17\x1e\xb5]\x15\x12\x88E\x8e\x04\x9e\xa6\x11`\xfa]\xfa\xf3\xab\xb4H\xc8\xfc2U\x12\xe9\xb7\xd0G\xce\xe1\xe9\x8b\xd5\xe9q\xe5\xa5]\xe9j\xde!\x9f㝒\x12\x9bZWu?$\xec\xab\xe7h\x8e4\xa8r\xc9E\xbb\xfd\xcd\xc3V\f\xd9\x06\x91g[tch\x81V\xc9\xff\xac9\xca\xfa\x8b-\xd7\xea\v\x92\xf4\x97ۛo\x13̭~qF\x8e\x02\xe2\x1c\x13\xde\xdd*1\xdfRS8\v\xa7>\x0eH{`7\x82$w4\x17#\xb9\x88\xab\x13\x00\x85J\xa5\x8b\x064\xf7g@\xd6\x19\x9d\a\xc2?\xe0\x8a\x01\x03\x01B\x83^\xfc\xf4H\xdb\"7i\x8fZz\xac\xb4\xd1\x0e\xaf,\b\xd0{\xa3G\xdai\u05ca;\xb8\xd8!o\x19kq\xc5\xe3\xfa\x8eX=\xef>k\xed<^\x8c\xc1\xe7\xee\xe8\x8cKv\x10:\xba=P=\x8d\xdf\x13\xe0\xfa\x84\xddd\n\x14tu(Z1>\xba\f(\x04\xd2\x0f\x16\xbcS\x83\xf7a\x9c\r>e}\x9e\x9d\xde\"Ɩ/\x9e\xdf\x12uo\xbd\\\x0fb\xc7#a\x85_3\xc1UN\xb0\xe3\xf0\x9a\xea\x9c\v\xafO\xe9ӅHPY\xac\xa8\x1b\x89\xc7.\x866\xc8\xfd\t\xa7C\x18\x1c0\xcb\xfbR\xdd\x15^\xa4\x12\xb4\x13ԹDmHA\u007fGv\xbc\xe7\x84\xe7!\x8f\x05-\xa5T\xb5\xde8T\xfdPԉ\xd6_\xf2<\xc84\x9c\xee\x1b\xae\xf8I\x8e-\x93JS\xf2\x88\xfa\xc7\xeda\xe9B\x83q\x06\n#\x15#\fmk\f.\f\xcd \x86\xf6\xf8㓩\xdf\x103\xaeΧ\xd7ϙ&χ\xdd\x06\xc0\x85k\xe3n@\x1c\xa4\xf8\x15w\xd1s\xf9t:2\x82\rC\x16\x050s\a\x1f\x8dI;\x0e\xd3z\u007f\x89\x9a\xe4Y\x90\xb8\xe5\xbf\xcdp\x00_#\x9f7νP\x8c%Ϯ\x06\x9d\xc9\x1eH\x13a\xdb\x1c\x9fP\xc0\x1dmN\xd6n\xed}p\xab@|\x1c\x1aE\x1f?'\xca\x16\xf0>\xc5\xf9\xc5\xfav\a\x9cW\xb9#\x82ڙ>=]D\x03\xb6m\x16\x14D\xef\xc56\x12\x0f\x8b\xf0\xe9̟\xa6\x88\xbd\xd1\x0ev\xf7W\b\x99O7\x14Uh\xd3-\x9b\xe4Lt\xa04{\x83\xa7SQ\xafBB\x12\x922\x92\xd2\xfbh\xed\xd3\xd4SH\x9f^rK\x91\xa4\xb9qv\x14\xe3\xf6\xf9\xa9m\xfc\xe3\x0fO\"\x0em#\xad\x06E\xbd\xfb*\x06|'\xfc\xff\u05fc\x9fl\xacl\xd1s\xed\xe2\xed\xcdYo\xcfwd}\x94\xefAK\xaa]\xe9ޯ#\xea]>li\xf9\xc9apq\xeaq\xc4\x10/k\x1e\xf3\x01\xe93}#\xf1%U\u009c<\x06\x8c\xa7\x81\x99\ue0ef\x8f\u007fey\r\xacӵ\x98`\x9f\f\x86\xf2\xa8\xcb\xd2N\x04ڹ\x90c\xf5\x94\xe3\xa0\x11\f\n\xffP\xf4oQ\xf3G\xe2\xe1hi\xff\x93ӛ\xfd[\x8aˢ\xfb\x89)}\xe8\xd4R\a\x87w\xb7\xaa\xdd\xca\x1e\x86`%\x90\x9d\xd4\xdd\xf1\x8fL\xaf\xf2UI\xff\xabQz\xad\x9c\xcdh\x96g\xf0\xf9\xcb\x04\xba\xbb\xd6O\xbd\x1c\xb2\xf8\x9f\x00\x00\x00\xff\xff\x80\xb6\xf7)\x9e\x1b\x00\x00"), + []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xc4Y_s۸\x11\u007fק\xd8\xf1=\xb87\x13RMz\xd3\xe9\xe8\xed\xcen:n\xef\x1cO\xe4\xcbK&\x0f\x10\xb1\x14Q\x93\x00\x8b]HV;\xfd\xee\x9d\x05HI\x94(Y\xce\xf4rzI\b,\x16\xbf\xfd\xed\x1f,\xe0I\x96e\x13՚O\xe8\xc98;\x03\xd5\x1a|f\xb4\xf2E\xf9\xd3_(7n\xbaz\xbb@Vo'O\xc6\xea\x19\xdc\x04b\xd7|Dr\xc1\x17x\x8b\xa5\xb1\x86\x8d\xb3\x93\x06Yi\xc5j6\x01P\xd6:V2L\xf2\tP8\xcb\xde\xd55\xfal\x896\u007f\n\v\\\x04Sk\xf4q\x87~\xff\xd5\x1f\xf3w\xf9\x0f\x13\x80\xc2c\\\xfeh\x1a$VM;\x03\x1b\xeaz\x02`U\x833h\x9d^\xb9:4\xe8\x91\xd8y\xa4|\x855z\x97\x1b7\xa1\x16\v\xd9u\xe9]hg\xb0\x9bH\x8b;Dɚ\a\xa7?E=\x1f\x93\x9e8U\x1b\xe2\u007f\x8cN\xffl\x88\xa3H[\a\xaf\xea\x11\x1cq\x96\x8c]\x86Z\xf9\xe3\xf9\t@\xeb\x91Я\xf0W\xfbd\xddھ7Xk\x9aA\xa9j\x92i*\\\x8b3\xb8\x17\xa4\xad*PO\x00V\xaa6:\U00091c3b\x16\xed\x8f\x0fw\x9f\xfe4/*lT\x1a\x14ͮEϦ7Q~{\xdeݎ\x01h\xa4\u009b6j\x84kQ\x95d@\x8b?\x91\x80+\x84\xce+\xa8\x81\xe26\xe0J\xe0\xca\x10x\x8c6\xd8\xe4\xe1=\xb5 \"ʂ[\xfc\x13\v\xcea.vz\x02\xaa\\\xa8\xb5\x04\xc1\n=\x83\xc7\xc2-\xad\xf9\xf7V3\x01\xbb\xb8e\xad\x18;\x86\xfb\x9f\xb1\x8cުZH\b\xf8\x06\x94\xd5Ш\rx\x94= \xd8=mQ\x84r\xf8\xc5y\x04cK7\x83\x8a\xb9\xa5\xd9t\xba4\xdc\xc7s\xe1\x9a&XÛi\x8cJ\xb3\b\xec\xdc\xf1,!\xf2+\xa5\xca?(\xae^\xdc\xf5\xfa\xaeL\xdbĊ\xc0\x0e\x14\xb4\x06\v\x1c\x94V0\x96\x18\x95N\x83#*\x01$q\xbf\xfb2\xc6\x19\xc0{\xe7\x01\x9fU\xd3\xd6\xf8\x06Lby[\xd0\xfa\xf80\x94\x88\xd8ꃵ\xe1ʌ\x1b\xae$\x8e:\x83\xd7\xd1PVO\b\xae34 \xd4\xe6\tgp%\x19\xbc\a\xf1?\x92:\xff\xbd\x1a\xd5\xf9\x87\x94\"W\"r\x95\x80mϬ\xfd\x8c\xdb\x01\xe4J1\xb07\xcb%z\x1cg3\x16b)p߃\xf3b\xbbu{\n\xa2Z\xf1Y\xaa3\xa8\x8f\x00\u007f~\xf7\xe5\x04\xda!O`\xac\xc6gx\a\xc6&VZ\xa7\xbf\xcf\xe11F\xc4Ʋz\x96}\x8a\xca\x11Zp\xb6ތ\xa3uP\xa9\x15\x02\xb9\x06a\x8du\x9d\xa5^A\xc3Zm\xc4\xfe\xde]\x12a\nZ\xe5y\xd8\r\x8cj}\xfcp\xfba\x96PI\b-c\x1d\x93S\xa64r\xe6\xcba\x9fN.\x89\xc9HGH\xc1\xc1\x0e\x8aJّ\xb2\x06\xb1i\x88\xec\x96AΒ\xfc\xfa\xb5\xd9zxl\xf7\xbf\x91\xe3\xfb\xb00\xfcN\x87\xe0Ef\xc5\xd6\xf9E\xb3\xee\xf7\xe2\xf9\xacY\xd2\xc4{\x8b\x8c\xd12\xed\n\x12\xa3\nl\x99\xa6n\x85~ep=];\xffd\xec2\x93@\xccR$\xd04\xb6\xe1\xd3\xef\xe2?_eE\xec\x8c/3%\x8a~\v{d\x1f\x9a\xbeڜ\xbe\xaf\xbb\xf4T\xba\x9ew\x8d\xc7\xe1JI\x89ue\x8a\xaao\xd2w\xd5s4G\x1a\xa5S\xc9Uv\U000db1ed\x10\x19\xbc\xe0\xd9d\xdd]0SV\xcb\xff\xc9\x10\xcb\xf8\xab\x99\v\xe6\x82$\xfd\xf5\xee\xf6\xdb\x04s0\xaf\xce\xc8ц4\xc5D\xeb\xee\xb4\xd0W\x1a\xf4g\xbb\xa9\x8f\x03Ѿ\v\x1c\xe9\xe3\xb62\x177rdUK\x95\xe3\xbb۳\b\xe6[\xb1~\xf7\x1d\xe5]\xfb\xd6k\x92\x10=ӷ\x9dD\x92ԜE\x91\xfa\xee\xb1.\xb8Ð:\x868\"\x1d\xe8W!\x91됴9\xfbH\xb2\xf1\x0e~ \xd1:=\xf8\x1e\xfaw0\xb5#}0\x9c\x8cx\xf12Ê\x03]~\x9d\x89\xe2=g)?\xb9S\x12\xcf\uebfa\xd0\x14N\x9a\xb9\xe1\xe3\xcd9\xcf\xdd\x1c\xcb\xc7\x17\x02\xaf\x13.6\r\xc6\xdbBD\x00kE\xfd\x16\xc7~\x83=mia\xac\x84\xa2\ful\xb6\xa4\x0f,\x95\xa9Q\xc3\xf6\xe9\b\x1e\xe5>\x17\xaf\xcc\xd7ǵ\xb2W\x13\bu\xbc\xe7\x8d\x00>\\U:\xdf(\x9e\x81\\\x933Qp0oC]\xabE\x8d3`\x1f\x0e'O\xa6A\x83Djy>\x0f~I2\xe9\x86\xd5-\x00\xb5p\x81\xb7W\xac.!:\xf3\xaf\xa9\xf3\xf8\xe5\x17\xbcJ\xd1y\x10\x0f\"1\x16Wۤ<\x17X\x10o/\xa19\xdc\"\x83{\\\x1f\x8d\xdd\xd9\a\xef\x96\x1e\xe9\xd0\aY﨣\xf6;\x83\xf71\x02.6\xb8\xdb\xe0\xbc͝\x10T\xae\xee#ױ\xaa\xc1\x86f\x81^\f_l\x18\xa9g\xa0O\xf4\xe3\x1bj\xecyw\xbc\xed\xd6\xf7\xd5*)\xea:\xf8B\xd9\xf8$#\xd1\xc9\x0e\xb4\xa1\xb6V\xc7-|oC<\xf6$8%Cvq\xd1g\x97\xa4t\x9c{͝:¹uv\xb4#\xebS\xc1X\xfe\xf3\x0f'\xcfGc\x19\x97\x83R\xd8\xcd\n\x85?\x89\xfe\xff\xb7\ue4c7/\xb1\xf2|Y\xe9\x9a\x0fD_\xaaZQ\xf1X\xcd\xda/?\xc7\xe5f\xb8ɷ\xa84#\xd4\x1c\f\xed\x1e\xec\xdf\uefa2\x8b\xb2\xee\x81>N@2K\xefm\xde=Fu#\xbb\x03K\x15\xd2k\xa1\xbe?|\xa1\xbf\xba\x1a<\xb8\xc7\xcf\xc2Ym\xd2_\x17\xe0\xf3\x97\ttOT\x9fz\x1c2\xf8\xbf\x00\x00\x00\xff\xff6\x10(\x86\xdc\x18\x00\x00"), + []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xb4W\xcdn\xe36\x10\xbe\xfb)\x06\xdb\xc3^*\xb9i{(tk\xd3.\x104\t\x16\xce6\x97\xa2\a\x8a\x1a\xd9\xd3P$\xcb\x19:u\x9f\xbe %ٲ-{\x83\x05V7\x0e\x873\xdf|\xf3CjQ\x14\xc5Byz\xc6\xc0\xe4l\x05\xca\x13\xfe+hӊ˗\x9f\xb8$\xb7\xdc\xde\xd4(\xeaf\xf1B\xb6\xa9\xe06\xb2\xb8n\x85\xecb\xd0\xf8+\xb6dI\xc8\xd9E\x87\xa2\x1a%\xaaZ\x00(k\x9d\xa8$\xe6\xb4\x04\xd0\xceJp\xc6`(\xd6h˗Xc\x1d\xc94\x18\xb2\x87\xd1\xff\xf6\xbb\xf2\xfb\xf2\xc7\x05\x80\x0e\x98\x8f\u007f\xa2\x0eYT\xe7+\xb0ј\x05\x80U\x1dV\x10\x90\x85t@\xef\x98\xc4\x05B.\xb7h0\xb8\x92܂=\xea\xe4v\x1d\\\xf4\x15\x1c6\xfa\xd3\x03\xa4>\x9cU6\xb4\x1a\r\xed\xf2\x96!\x96\xdfg\xb7\xef\x89%\xabx\x13\x832s@\xf26\x93]G\xa3\u0099Br\xe0\x032\x86-\xfea_\xac{\xb5\x1f\bM\xc3\x15\xb4\xca0.\x00X;\x8f\x15<&\xa8^il\x16\x00[e\xa8Ɍ\xf4\xe0\x9dG\xfb\xf3ǻ\xe7\x1f\x9e\xf4\x06;\xd5\v\x93e\xe71\b\x8d1\xa6o\x92߽\f\xa0Aց|\xb6\b\uf4e9^\a\x9a\x94Qd\x90\r\u0090\x17l\x80\xb3\x1bp-Ȇ\x18\x02\xe6\x18l\x9f\xe3\x89YH*ʂ\xab\xffF-%<\xa58\x03\x03o\\4M*\x83-\x06\x81\x80ڭ-\xfd\xb7\xb7\xcc .\xbb4Jp\xa0x\xfc\xc8\n\x06\xabL\"!ⷠl\x03\x9d\xdaA\xc0\xe4\x03\xa2\x9dX\xcb*\\\u0083\v\bd[W\xc1F\xc4s\xb5\\\xaeIƊ֮\xeb\xa2%\xd9-s]R\x1d\xc5\x05^6\xb8E\xb3dZ\x17*\xe8\r\tj\x89\x01\x97\xcaS\x91\x81\xdb\\\xd0e\xd7|\x13\x86\xf2\xe7\xf7\x13\xa4\xb2Kic\td\xd7{q\xae\xb2\x8b\xbc\xa7\"\x03bPñ\x1e\xff\x81\xde$J\xac\xac~{\xfa\x04\xa3Ӝ\x82c\xce3ۇc| >\x11E\xb6\xc5\xd0'\xae\r\xae\xcb\x16\xd16ޑ\x95\xbcІ\xd0\x1e\x93α\xeeHR\xa6\xff\x89Ȓ\xf2S\xc2m\xeek\xa8\x11\xa2o\x94`S\u009d\x85[ա\xb9U\x8c_\x9d\xf6\xc40\x17\x89\xd2\xcf\x13?\x1dGNJ=[{\xf18-f3t\xda\xffO\x1euJXb-\x1d\xa4\x96t\xee\x01h]\x00u\xa6_N\f\xcf5g\xfaj\xa5_\xa2\u007f\x12\x17\xd4\x1a\uf75e\xb4\xf9\x05T\xbf̝\x18a\xa5\x11\xd77*\xce+\x9eX\x06\x90\x8d\x92I\x87\x8a\"\xbbo\xf3\x998.R\x9eiW\xa9]\xad\xb2\x1a?\xe4ڱzw5\x96\x87\x99\x03)\x94\x8d{\x05\xd7\nک\xc9\x11e\x8dgA\x84h\xdf\f\xb2\x9f\xc9wM*\xad\x960\\\x05\xb8:Q\x1eyn\xa31\x83\xa5B\xbb\xce+\xa1\xda\xe0\xd8ȭ\vg\x10\xa9\xb7\xb1\xeb\xbb\xfa\xcb\xf8\xdd:\x13;\xdc\xdf\rW\x91?\x1f\xebN\v\xa4\x17\f R\b\x10\x8e\xaf\xc0\xe97\xd4\x04\x83w\xcd\x00`(ZNq\xbe\x11{J.\x05<\x9a\x86\xc5|\xf1\x1fi\xccUԑ\xc2i6\x8f6O\xf8\xfa\xec0\x10%\x91\xdf>\x0e\xb2\xfaH\xac\x8e!\xa0\x95\xc1H\xbe\t\xbfh \x18\xc52i\x8b\xf4\x06\xba\x9a\xe7\xfbs\xfd\x11R2\x05\x92\x04\xd3.zU<\xd7/\xad\v\x9d\x92\n\xd2h/ҡ\x93\xfd\xf4\x02S\xb5\xc1\n$\xc4\xd3\xcd\xcb\x13\x01\x99\xd5\xfaz\x04\x0f\xbdN\u007f\x15\x0e\a@\xd5.\xca\x05b\xf3\xa5x\x85ګ\x88\xfcF\xf1u<\x1f\x93\xc6\\Z\xf1\xad\xce\xd1\xc6\xee\xd4E\x01\x8f\xf8z&[\xa1jN{\xae\x80G's\x1b\x17b\x9a\xa9\xe5\x13\xd1\xe1\x89}sX\xe5\xba+\x86'u\xde\x00\xc8/\xd3f\x92b\xee{s\x90\x1c\x1aDi\x8d^\xb0y<}R\xbf{w\xf4B\xceK\xedlC\xfd\xff\x00\xfc\xf9ע\xb7\x8a\xcd\xf3\x88#\t\xff\x0f\x00\x00\xff\xfft\x8f\x1aC\x8e\f\x00\x00"), + []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xd4YMo\xf3\xb8\xf1\xbf\xfbS\f\xf2?<\xbb@\xac\xfc\x9f\xb6\x87·4O\x16\bv7\r\x92\xa7\xe9\xa1聖\xc66\x1b\x8a\xd4\xf2ʼn[\xf4\xbb\x173$\xf5bɲ\x17\xedb[]\fS\xc3\xe1̏\xf3\xae\xc5r\xb9\\\x88F\xbe\xa2u\xd2\xe8\x15\x88F\xe2\x87GM\xff\\\xf1\xf6{WHs\xb3\xff\xbcF/>/ޤ\xaeVp\x17\x9c7\xf53:\x13l\x89_p#\xb5\xf4\xd2\xe8E\x8d^T\u008b\xd5\x02@hm\xbc\xa0eG\u007f\x01J\xa3\xbd5J\xa1]nQ\x17oa\x8d\xeb U\x85\x96O\xc8\xe7\xef\xff\xbf\xf8M\xf1\xbb\x05@i\x91\xb7\u007f\x955:/\xeaf\x05:(\xb5\x00Т\xc6\x15Xt\xdeXt\xc5\x1e\x15ZSH\xb3p\r\x96t\xd8֚Ь\xa0{\x11\xf7$A\xa2\x12\xcfq;\xaf(\xe9\xfc\xf7\xfd\xd5\x1f\xa4\xf3\xfc\xa6Q\xc1\n\xd5\x1dƋN\xeamP¶\xcb\v\x80ƢC\xbb\xc7?\xe97m\xde\xf5w\x12U\xe5V\xb0\x11\xca\xd1kW\x9a\x06W\xf0HR4\xa2\xc4j\x01\xb0\x17JV\xacb\x94\xcb4\xa8o\x9f\x1e^\u007f\xfbR\xee\xb0\x16q\x11\xa0BWZ\xd90]\x96\x0f\xa4\x03\x01\xaf\xac\x1f\t\xc1\x17\x01~')\x0eX7\xfep\rB\xa9\xbe\x03\n\xdb\x01\xf8\xeb\x1aԅ\x8e\xf5pL}ƱN#46\x8e>F\x9d\xa5%\xba\x14Y\xff\v\x00Sb\x8d\xea\x05\x15\x96\xde\xd8Y\xb0~\xe8SF\xa0(/\xee?\x17\xc37\xde\xc0F*\x8f\x16ޥߍ\x14xߡN8QE\"u%\xf7\xb2\nB\r\xac\xac\x87R\a&\x18\vZ\xaa\xeb\x11O\xc28\xef\x1e`\n\u007fd\xe1\x85\xfaY>x*\xc5\xd2S\v_\xee\xee?\xa88r]\xf73\x03\xdb\xf1\x86\x88\\N_\f?\xb8\x8c\x1dUH\xd2b\xcdu\xd7\x04g\x80\xafle\x1d\x15\xeb{\xfb\xf8el@pڈFB\xde\xce\b\x92|\xa2\xbd^\xca.9\x11Or\x86T9_\x83\x807<\xc4\"\x9b\xea\xf8\x86Bifa\x91\xcbs\xbe\xe87<0Q\xaa\xb8'\xb9\xce]J|\xde\xf0p\xeaՑ\xbat^\xaa~\xa2\u07b4\xc0Rq=\x96U\xe5\xee\nO)I\x8f7\xd3\xc2¼\xa7\xe6'#r\xa1\xd8-\x80\xbd.\x90!\xfe\xe4\"\x9cd_;\x19\x1b\xbc\x19\xa9\x1d\xb2\xed\xe5\xfe\xe6\x95:Ֆy\xb4\xa8\a}\r\x8f\xc6\xd3\xcf\xfd\x87\xa4\x9a]\xe8q\x06\xed\x9e/\x06ݣ\xf1L\xfboA\x12\x85\xba\x10\x90H\xcc\x06\xaacl#\xbd\xfa\xed\x8f\xe3\xe8A\xb7\x9a\xf5\x9bQB:jA\x8c͚s\xdb\x1a\x8f\x88\xcc\xeb\xe0\xb8c\xd1F/9\"e\xee3L\xdbK\x93.Ci\xec\x00\xaf\x13\a\xcd\xf0\\#\xa4\xe3\xbfR#\x16\xf7\xc4VZ\x89\x12+\xa8\x02C\xc0\xad\xa0\xf0\xb8\x95%\xd4h\xb7sr6\x14\xa7N_\xddL$\x89\xcf\x05w{:\v\xe5'\x85\x9dj\xfa\xa0%\xd9\xfa\x897\xb3\xd7;٫]&\x15\x87oNp\x93ڋ\xaa\x921\xc3<\x9d\x89Og\xf0\x19\xe7\x8cxhJ\xb4\xa2!\xcb\xfe\a\x85S6\x94\u007fB#\xa4u\x05\xdc\xf2\xbcJM\xdfl\x9f>U\x1e}\xd6\xc4U: \xcc\xf7BQ\xa8\xa7\xc0\xa1\x01\x15\a\xfeI\x96f3\xcah\xd7\xf0\xbe3.F\xf1\x8dD\xc5S\x88\xab7<\\]\x0f<\x0f\xe4t(\xbdz\xd0W1I\x8c\xfc\xa0m\xf8\x8cV\a\xb8\xe2wW\xc5(\tN\xb2\x9dM\x8c3\x16q\xf2U[\xe9\xfe(\x9aF\xea\xed\xf1=_f\v3v0\xb0\x81ǣ\xd3\x06\x86\xd0/K\a%\xfc\xf8\xb88\xe6\x9b(\xf6m\x1e6jo\n\xb8Շ\x11WG\x1d\xe3D\xa9;\xec I\xa4w\xa9\x14E\xa5ijb\xa6}Fi\xb0\xe0D\x1d\xf9O\x0f\r&@O\x1c\x9f^\xe7+\xf9\xe7\x96l\xa2\x0f\xec)K\x95b\xab\xc0\xd3\xeb\xd8r\xb8\xf8tZ4ng<|\xb3\x97\"M\xb5L\xa8\x1ak\xf6\xd4\x0f~\xfb\x1f\xea\xe8\\\xb9\xc3*(<;\xb4y\xe9\x11\x9e\x1f\xdbd\xb6c[\xe8ph;\xb9\x8cV\x15=p8\x1eJ-L\xe2K\x97<\xd5D\xb7\f\xa39\x18\xc73Y\n'.\x94%:\xb7\t*w<<\xf6\xa7\xf6:\x92K\xd7J{\xe1\x18i*C,{ï\xb3\x13:/|pggtL\x05\xa5h|\xb0\xa9H-\x83\xb5\xacT|g6\xa31\xdd\x05S:\xb4\xd6\xd83\xd3\x1e&\x89\xee^\x9a\xa0\xb9V#\xbb\xe5\xbdP\xa3sb\x9b\xc7<\xefh\x11\xb6\xa8)\xc2N\x8c8R\x1d\x80\x1fX\x86\xf4\x91`اR$\x15\xa5\xa7\xf6+\x8a\xc6a\xb3u\xe2S9\x93\b\xc4\xf6ĝI\xedq\x8b\xc3L\xbc\x11R\x05\x8b\xcf(\xdc\xf0\xdb\xc1H\xfd\xef\xfa\x94\xa9\xb4\x8b\x9a\xc7\xceC\x04\x87U\x1a;{i\xf1\xe4l\x92\xfaP!G\x9d\xdf\xc9\xf8\xdb섛w\xc3'\xa2\xc8\xfe\xd77\x87\xd6\x03\x9f'eA\x1d\xeac\xc6Kx\xc4\xf7\xd1\x1a)\x8f\xd5k\xfb\xe9hD𠟬\xd9R\xe6\x1d\xbd\xba3u\xa3pl\x05Kx\x12\xd6K\xa1\xd4!\xb2?q\xea\xa58u\x1f\xb6\xee\xcf\x1b\xf3\xeb\x11\xf1р\x85̺\xe3\x97M\xf0\x1b9\x1e\xad\xa5/]k\x85\xdf\xfe:\x83\x92wa\xb5\xd4\xdbyu\xff\x9c\x88&\xbc7\xed\xff\xe5\xfc7\v8\xf4\xe0\x13\xb3\xbc\x9f\xeb\xc1\x13\xb1\xf4h\xa9\xfb\xc4\xfb\xb9\xfb\xc7h-\xd3']~Aݧ\xddc\xd5\xc3>\x89\x92V\xba\x00-\xca\x12\x1b\x9f&\x98\xfd\x8f\xbbWW\xfc'\u007f\xbd忥ѱ\xfar+\xf8\xcb_\x17\x90\x10x\xcdr\xd0\xe2\xbf\x02\x00\x00\xff\xffF\x9c\x18\xb7\x0e\x1f\x00\x00"), + []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xec\x1cMo,9\xf1\x9e_Q\n\x87\aRf\xc2\x03\x0ehno\xf3\xb2\"\xe2\xf1\xf6i\x13rA\x1c<\xdd5\x19\x93n\xbb\xb1ݓ\f\x88\xff\x8e\xaa\xec\xfe\x9c\xfepg\x13\xb4\xac\xe2S\xe2\xb1\xcb\xe5\xfav\xb9\xdcg\xab\xd5\xeaL\x14\xf2\x1e\x8d\x95Zm@\x14\x12\x9f\x1d*\xfaϮ\x1f\xffh\xd7R_\x1e>nщ\x8fg\x8fR\xa5\x1b\xb8*\xad\xd3\xf9\x8fhui\x12\xfc\x8c;\xa9\xa4\x93Z\x9d\xe5\xe8D*\x9c\u061c\x01\b\xa5\xb4\x13\xd4m\xe9_\x80D+gt\x96\xa1Y=\xa0Z?\x96[ܖ2K\xd1\xf0\n\xd5\xfa\x87߮\u007f\xb7\xfe\xc3\x19@b\x90\xa7\xdf\xc9\x1c\xad\x13y\xb1\x01Uf\xd9\x19\x80\x129n\xc0&{L\xcb\f\xed\xfa\x80\x19\x1a\xbd\x96\xfa\xcc\x16\x98\xd0j\x0fF\x97\xc5\x06\x9a\x1f\xfc\xa4\x80\x89\xdf\xc5m\x98\xcf]\x99\xb4\xeeϝ\xee/\xd2:\xfe\xa9\xc8J#\xb2\xd6z\xdck\xa5z(3a\x9a\xfe3\x80\u00a0Es\xc0\xbf\xaaG\xa5\x9f\xd4\xf7\x12\xb3\xd4n`'2K?\xdbD\x17\xb8\x81\xaf\x84I!\x12L\xcf\x00\x0e\"\x93)\xef\xd3\xe3\xa6\vT\x9f\xbe\xdd\xdc\xff\x9e\xf0ȅ\xef\x04H\xd1&F\x16<\xaeF\x11\xa4\x05\x01\xf7\xbcI0\x81\x1d\xe0\xf6\u0081A\xc6E9\x1aQ\x18\\UX\xa6\xa0M\x80\tP\xa0\x91:\x95\t|'\x92Dz\xf0S\xed^\x97Y\n[\x04S\xaau\x18[\x18]\xa0q\xb2\"!\xb5\x96\xd4\xd4}=L?\xd0V\xfc\x18HINЂ\xdb#\x04nc\xca\xd4\xcb\x05\xe8\x1d\xb8\xbd\xb4\r\xdeL\x92\x16X\xa0!B\x81\xde\xfe\x03\x13\xb7\x86[\xa2\xb3\xb1\x15\xb6\x89V\a4\xb4\xefD?(\xf9\xaf\x1a\xb2\x05\xa7y\xc9L8\f\x1c\xad\x9aT\x0e\x8d\x12\x191\xa1\xc4\v\x10*\x85\\\x1c\xc1 \xad\x01\xa5jA\xe3!v\r\u007f\xd1\x06A\xaa\x9d\xde\xc0\u07b9\xc2n./\x1f\xa4\xab\xf4$\xd1y^*鎗,\xedr[:m\xece\x8a\a\xcc.\xad|X\t\x93\xec\xa5\xc3ĕ\x06/E!W\x8c\xb8b5Y\xe7\xe9\xaf*.\xda\x0f-Lݑ\xc4\xc6:#\xd5C\xdd\xcdBh\xcb\xc4\x163\xb0\x98a\xe2\xb4\x19#\xcb<\xd3}\x8b\xb3\x85#\xf4\x1c\xb0\x8a\x8dݧ-7\x1b\x9c\x04\nd\xf2\x9f\xf62\xd9\xfb\xf8\x8ad\x8a!A\xaaѲ-\x10E\x91\x1d\xc77\v\xf3\x92\x10\x16\x9a6\aM\x9b5\f}\x98\xc3&\xa2iQ\xf6\xb4i3\x96\xb5K\xe7ZD\xde\xc9\\9\x8d\x17\t\xf4\xcd\xc9\xe4\xd7\x16h\"\xb0\xa4\xb3\xd5\xcd\x0e0/\xdc\xf1\x02\xa4\xabz\xe7aR\x18\xd4\xe0\xf0\x8b`\xd4K\xf4\xe1\xa6?\xf7\x95\xf5\xe1\x15\xb8T\xa3\xf0\u007f\xcd$v6\xb7\xc1\xd7,`З\xf6\xbc\v\x90\xbb\x9aA\xe9\x05\xecd\xe6\x90\xe3\x9ei\x14[\xaeo\x96S\xafE\x968\xafI-\x17.\xd9_\xd7G\xc8\xd9\xf1=\n\xf5\xa7\xfbX\xb9:It\x9d\xfc,d\xe0\f\x834\x98\xfb\xbc\xc5\x1d\xeb@\xd3Ñڧ\xaf\x9f1\x9d&\x14\xc4J\xe4\xc9v>\xf5Pn/\x1f\x8e\x01\xf1\x9b\t\x01U}\xc2\xf2\xf9\xa8\v\x10\xf0\x88G\x1f\x05\t\x05\xc4(AK\x8d\x1e$N\x89ĉ16\x11\x8fxd@!\xd7\x151?^4|{\xc4c\xdc\xc0\x1e)\t\xb3\x90\t\xf04\xa5\x0e&\b\xa7L\x96\x90\x118s\xc9\x1a\x02N\xc7l\x12\x96\x98\x9b\xaaU\x9cx\xd1vk6vҸ\x8fx\xfc`=\xc3H;\xf6\xb2\x88\xde0\x19`\xb0\xc8zTe2\xefE&\xd3z)\xaf\x0f7j*\xea\uedaf\xdaݨ\v\xb8~\x96\x96\xd0S)|\xd6h\xbfj\xc7=oFX\x8f\xfe\x8b\xc8꧲\xea)o\xe6\x89\x1e\xed\x04i\x94\xd0\xfbv\xe3O\x985\xab\xa4\x85\x1bEg\xa5@\x17Ns3\xccx\xb1d\x94\xf2\xd2r&Ti\xb5bG\xbb\x1eX+\x1af`\x8f6\x1d\xee\xb4\xd1k-\x1b\r\x95\x0et\x1e\xb5;\xf2=\x1e\x82O\xdfg\"\xc1\x14Ғ\x89*\xa2!Zg\x84\xc3\a\x99@\x8e\xe6\x01\xa1 _\x10ˍh\xfb\xec\xdbb\x99\x8b\r\r\xaa\x16\f}\x1a\x83Ҋ\xf4:j\\\xc5\xfe\x88\xc1\x83\xf9\xe8\xe9\xc11{c\a\xcdqL\x04\xb5E\x9ar\xc2Vd\xdf\x16y\x89E\xdc9\x8d\x1f#\xda\tx\x97\xe5\xdb\xc0\xcbUȳ\x81\xd894`\x9d6\xd5U\x1f\x19\xc9^ژ\xb8h\xe7\x0e\x1c\xc4\xd8:{\xe7\xc1\xd2a\xee\xbc\xd1oo\u007f\xcf\xfd\x1d \xfd=\a1\xe1\x00\x86!\x17F'h\xed\x9c\xd8DY\xf8\x99\xc4f\x9d\xd4\x14\xfe\xb0\xc4wk\xb3\xc2:\x97l\xadڒP\x98ȹ\xf88q\xfd\xdc\xca˒\xf9\xa0\xff\xe7Ev9v\xc0Z\x9f\xe7BE9\xb0\x13D\xaf\xfc\xdcJ\xc5\x02(\u007fD1\x0f%\x9b\x8b%\x91k\x10\xbe\x9fO0\x90KuË\xc0\xc77\t\x1fj\xa3\x8b/;>\\U\xb3\x1b\x16\xd4\x1d×\xa4c\xad\xd0|_a\xb0\xc3\xc9Ӭ\xfe\xa2\xb0Yi\xd7N}\x10\xe4B\xa7\x1f,줱\xaeA6\x1a\xa6\xb4|I\xfavg9um\xcc\v\x8fr?\xf8\xb9\xadd\xdc^?\xd5\x17\xfa\xe3\x17\xbfC\x8d\xaf\xc7\x10\xe4\x0e\xa4\x03T\x89.\x15'\x8d\xc8\x18\xf0\"\x9e\x1d\xf1\x82\f\xb1~\xafi\xa8\xca<\x96\x10+\x96D\xa9f\xf2K\xed\t\xdf\v\x99\xbd\x15\x1b\x9d\xccQ\x973\x8e\xb9i\xddz\x04?\xb7S\xa9\x91\x8bg\x99\x979\x88\x9c\x18\x11Mr\x8a6d\x8e]\x19\x80'!\x1d{$\x82\xcc\xee\xc9\xe9h\x90\x89\u038b\f\x1d\xc2\x16wڰ\xbe[\x99b\xed\xfa\x83\\\xe8\xf1{\xc7~\x13\xb0\x132+M\xb4\xd5]ȍe'\xa4`x^\xf7\xe0\x13\x8b\u008a\xc9\x17\x95\x8e\x8e\x0ei\xe7G\u007f\x8e\xc0&\xaa\x84`\x1a\xd9\xc9UB5\xccUVZ\x87&\xbe\xe6\xf6fx\xde@\xfdu⇬\xf8aΰl4\xe5\x16\x8d˪\x8bpI\xd9*E\xf1\xf5\xfa\xb3\xd1qdq\xedX\x9dv\\)\xd7\\\x01W\xb7\x06\xb9.\x9e\xaa\x8a\x90\x87\xadFX:p˿\xf8hW\x03u\xeb\xb082\xaf\xb0\xfd\x19\xd5'G\x95X\xcd\x14VM\x97p\xebj\x8d\x11*v\x12\xf0]\xf2\x99N\x19\xf1ϐz\xb3\xb5O\xe3\x15O\xe1F\x0e\x9d8|\\w\u007fq:\xd4?\xc1\x93t\xfb\xc1M\xf1\x1b\x14:.\xaa\x87vat%\x8b\xe1\x8dW\x9f\xaa\xa0\r(\x99\r\xd74\x10\xc1\xab\xf9\x1dr\xc3\x0f\x85?\x94\xbeH\u007f\xe7\x8eI\xb15R/\xae\x8c\xea\xd6=\x8d\x1a\xf8\xe5\x17xK\xca\xca\xe3k\x9f\xe6J\x95\x96T<\xb5\xab\x99&@\xc6\xd69ŝxgk\x9a^P\xc9\x14]W\xf9*וq\xb5JoQ\xa1\xb4\xa0.\xa9[o4\x03wY5R$\x99b*\x8f\x16\xd7\x1b\x85ڞ\x99\xfdDT\x19\x8dV\x0f̀\x1e\xa8c\x9a\xaf\x19\x9a#\u007f\a\x95W\xa9\x14zA}\xd0\xeb\xd6\x11\xbfV\xd4=U\xed\x13Q\xe3\x13\x11\x97\xcfa\x1aQų\xacv'\x82\x86/\xacө\xabpF\xd7^Z\x9dӭ\xbd\x19\x05\x1bS\x933Rq3\ns\xb2\x12'\xb6\xcef\x14\xfa\xac\xfb\x9e\x91\x9cɟ\xad\x12\x85\xdd\xeb\xeaI\xebl\xe4w\xdb\x1d?p\xf4\xaa\x1e\xb4&\x99.\xd3\x1a\xfe\xf0\xf6\xf8\xe5\xff\x11\xbeݳs\xe1\x87~I\xf3\x042\xb8\x8f*\x94뿐\x1c~\x9d\f?\xfd(f\x9d6\xe2\x01\xbf\xe8\xa4\xf5\xf9\x86)\x9at\xc7w^\xe7\a\xe6WɖP\x954\x1c\xb6\x86\x1d\xf5\xc159\xd6\xf0B\xb89\xaf\x12\xa6\xc3r1\xa9\xb9\xcee\xb3\x9b\xba\xbb\xfb\xe27\xe2d\x8e\xebϥ?\x06\xaf\na,\x12m\xab\r\xfaI\xdb1\x03\xb1\xd7O\x90\xe9\xb0\xfb\xef\xfa\xf8\x1b\xe4l.\x9f\xb7\x17\xef¿\x8e\xae\x04\xb2\"\u05fc\b\xdf\x0f\xcfkE\xde-\xa6\xf1\xd1oLv\xc7 \tku\"\x05\xbf\t\x95.<\xe3x\x8b\a\xbd\xe3\x0eaD\xe9\x87\xfc\xd8j\xe8\x01\xfd\xaa~\xcd\u007f6\x03\xd4:\xe1J;\xff9\x04\x1e\x06\x89(\\iB\x924)\r?\xed%\x10\xe8_\xc0.\xff B&\xac\xf3\x825\xf9݁/\xf5\xb0&J\xb7\xceg[+̓'a\xf9\x8b\x03>\xd7*m\xf7S2\xed6\xf2́\x9d6\xb9p\x1bH\x85\xc3\x15\xc1\xee\xfd>i\x99F\x99\xcdO\x9f'w\xf7\x8dF\xd47:\x81\xac<\xadz0=\xb2\x93\xa1\x94\xfd\n\xbe\xe2\xd3Iߵ\"\xc4\xfb\xb94\x9f\x95\xc7\xf4\xbe\xfe\xdaM즚\xef\xe3\xf0%ȉ\x02tu\xb67\xb8\x97\xa9\xa1\x13\u007f\x03\xcf_xX\xf8\xb5<5\xb4|\xfcJh'\xbf\xe9\xfd6\xa2\x85\x13\x1a8\xac}\x03J\xd2\xebj\xbe\x88\xf4\xb1\xf9\x8f\x97^\x85/ \x1d\xfc\xdd!\u007fr(m\xc9J\xf0L\xa1\xa7\xd1<\x91$X\xb8\x90\tl\u007f\n\xe9\xfc\x9c\xff\xa9\xbet\xc4\xff&Z\xf9\x18\xd0n\xe0o\u007f?\x83\xe0E\xee+<\xa8\xf3\xbf\x01\x00\x00\xff\xff|l\xf6\x1b=J\x00\x00"), + []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xb4VM\x93\xe34\x10\xbd\xfbWt-\x87\xbd`\x87\x01\x0e\x94o\xd4\x00US\xc0\xd4\xd4d\x99\v\xc5A\x91\xdaI3\xb2dԭ\f\xe1\xd7S\x92\xecI\xe28;\xbb\x87\xf5\xcd\xed\xfexz\xfd\xba\xad\xaa\xae\xebJ\r\xf4\x84\x81ɻ\x16\xd4@\xf8\xaf\xa0Ko\xdc<\xff\xc0\r\xf9\xd5\xfef\x83\xa2n\xaagr\xa6\x85\xdb\xc8\xe2\xfbGd\x1f\x83Ɵ\xb0#GB\xdeU=\x8a2JT[\x01(缨d\xe6\xf4\n\xa0\xbd\x93\xe0\xad\xc5Po\xd15\xcfq\x83\x9bH\xd6`\xc8\x15\xa6\xfa\xfbo\x9ao\x9b\xef+\x00\x1d0\x87\u007f\xa0\x1eYT?\xb4\u08b5\x15\x80S=\xb6\xc0\x18R\x90(\x89\x1c\xf0\x9f\x88,\xdc\xec\xd1b\xf0\r\xf9\x8a\aԩ\xf06\xf88\xb4p\xfcP\xe2GP\xe5@\xeb\x9cj\x9dS=\x96T\xf9\xab%\x96_\xafy\xfcF\xa3\xd7`cPv\x19Pv`r\xdbhUXt\xa9\x00\x86\x80\xf9\xc3\x1f\xee\xd9\xf9\x17\xf7\v\xa15\xdcB\xa7,c\x05\xc0\xda\x0f\xd8\xc2}B=(\x8d\xa6\x02\xd8+K&\xd3S\xce\xe1\at?>\xdc=}\xb7\xd6;\xecU1\x02\x18d\x1dh\xc8~Kg\x00bP0\"\x01\xf1\xa0\xb4Ff\xd01\x04t\x02\x05)\x90\xeb|\xe8s\xb911\x80\xda\xf8( ;\x84\xa7L\xedx\xb6ft\x18\x82\x1f0\bMD琣\xcc^m3\x8c\xef\xd3!\x8a\x0f\x98$,\xe4\\c\x94\a\x1a\xe0|@\xf0\x1dȎ\x18\x02f\xf6\x9c\x9c\xa3˜t\xa0\x1c\xf8\xcdߨ\xa5\x19O\xcf\xc0;\x1f\xadIj\xdcc\x10\b\xa8\xfd\xd6\xd1\u007f\xaf\x999ѐJZ%\x93\x0e\xa6\x87\x9c`p\xca&\xfa#~\r\xca\x19\xe8\xd5\x01\x02\xa6\x1a\x10\xddI\xb6\xec\xc2\r\xfc\xee\x03f\x02[؉\fܮV[\x92i\xb0\xb4\xef\xfb\xe8H\x0e\xab<\x1e\xb4\x89\xe2\x03\xaf\f\xeeѮ\x98\xb6\xb5\nzG\x82Zb\xc0\x95\x1a\xa8\xce\xc0]\x9e\xab\xa67_\x85q\n\xf9\xfd\tR9$\xc1\xb0\x04r\xdbWs\x96\xfaUޓ̋\x1aJX\xc1\u007f\xa47\x99\x12+\x8f?\xaf?\xc0T4\xb7\xe0\x9c\xf3\xcc\xf61\x8c\x8f\xc4'\xa2\xc8u\x18J\xe3\xba\xe0\xfb\x9c\x11\x9d\x19<\xb9\xa2%m\t\xdd9\xe9\x1c7=\tO*M\xfdi\xe06\xaf\x17\xd8 \xc4\xc1(A\xd3\xc0\x9d\x83[գ\xbdU\x8c_\x9c\xf6\xc40\u05c9ҷ\x89?݊玅\xadW\xf3\xb4\xb2\x16;\xb40\xbd\xeb\x01u\xeaY\".\xc5RG:\x8f\x01t>\x80Z\ni\xdeĐ\xbd?\vŸ#\n\x8e\xd9\xe6H3\xf8\x16\x8e\xa5U\x91\xed;\xc5xn\x9a\xa1yH\x1e\xf3ʖ:\xd4\am\xb1$(\x9b\x02\xdf\x02\x91\x1et\xb1\x9f\u05eb\xe1\x1e_.l\x0f\xc1\xa7=\x99W\xf1\xe9\xb3\xd8\u007f(\xff\x88-9\xfe\xf8i\x8aO\xfe뜮ܓU;\xa6\x81\x10\x9dK\x13\xe9]2ϒ\xc2\xf9F\x9e}%\xc1\xfe\x02\xc7\"\x92;\xd7\xf9\xfc\xd7V\xa9\xa4\x922'86u\xacQ\x10]\xa4\xbb\xd6\xd3\xf2\xccW\xd1'\x10X\x9e\xfc\xe7\xff\xfc\xc0\xb4:(\xe0B\xcd:cY0\xa7J\x17\xe6ʼn\x19\x91Ek\xd5\xc6b\v\x12\xe2<\xb2ĩ\x10\xd4\xe1\\\x15\x93\x8c\x8ew\x9c\x8f\n\xe4\xc2=i\xffe\x87\xee\x9a\xc2\xe1E\xf1RoJ\x1a\xd8\x1c\xae\x05\u07be^\xd6\xe6CRd\xd9Bں\xb5\xd0\x05K\x9f@\xc4B\x97\x8aT\x17n\a\x17$\xacO=\xa7\xd9?\x13\xfctY\x98#\xbfR|\xa1\xa93\xd3\xf1nzs|\xcb®ǻh\xfe0\x9e\u009c\x9c\x9c\xc5\a\xb5\x9d\xb88\xee\xd6t\xcd\x1a\x04\xcd\xfd\xfc&\xfa\xee\xddٕ2\xbfj\xef\f\x95\x8b4\xfc\xf9WU\xb2\xa2y\x9ap$\xe3\xff\x01\x00\x00\xff\xff\x8e\xadi\xa0\xc7\v\x00\x00"), + []byte("\x1f\x8b\b\x00\x00\x00\x00\x00\x00\xff\xb4V\xc1\x8e\xdc6\f\xbd\xfb+\x88\xf4\x90K\xed\xe9\xb6=\x14\xbe\x05\xdb\x16\b\x9a\x06\x8bl2\x97\xa2\a\x8dDϰ+K\xaaH9\xdd~}!Y\xde\x19\xcf\xce$-\x8a\xf8&\x9a\"\x1f\x1f\x1f\t5m\xdb6*\xd0\x16#\x93w=\xa8@\xf8\x97\xa0\xcb'\xee\x1e~\xe0\x8e\xfcf\xba١\xa8\x9b恜\xe9\xe16\xb1\xf8\xf1\x1d\xb2OQ\xe3\x8f8\x90#!\xef\x9a\x11E\x19%\xaao\x00\x94s^T6s>\x02h\xef$zk1\xb6{t\xddC\xda\xe1.\x915\x18K\x86%\xff\xf4M\xf7m\xf7}\x03\xa0#\x96\xeb\xefiD\x165\x86\x1e\\\xb2\xb6\x01pj\xc4\x1e&oӈ\xecT\xe0\x83\x17\xeb\xf5\x9c\xac\x9b\xd0b\xf4\x1d\xf9\x86\x03\xea\x9c{\x1f}\n=\x1c\u007f\xcc!*\xae\xb9\xa6m\x89v_\xa3\xbd\xa9ъ\x83%\x96_>\xe1\xf4\x86X\x8ac\xb0)*{\x15Y\xf1ar\xfbdU\xbc\xe6\xd5\x00\x84\x88\x8cq\xc2\x0f\xee\xc1\xf9\x8f\xeegBk\xb8\x87AY\xc6\x06\x80\xb5\x0f\xd8\xc3\xdb\\AP\x1aM\x030)K\xa6ܟk\xf2\x01ݫ\xbb\xd7\xdb\xef\xee\xf5\x01G5\x1b\x01\f\xb2\x8e\x14\x8aߕb\x80\x18\x14,h\xe0\xe3\x01#¶0\a,>\"W\xe05$\xc0R\x01w\xd5\x14\xa2\x0f\x18\x85\x16\x82\xf3w\xa2\xb0'\xdb\x19\x9e\x97\x19\xf0\xec\x03&k\n\x19\xe4\x80P\x95\x81\x06\xb8\x14\x03~\x009\x10C\xc4\u0094\x93c\xab\x96\xcf\x0f\xa0\x1c\xf8\xdd\x1f\xa8\xa5\x83\xfb\xccfd\xe0\x83O\xd6d!N\x18\x05\"j\xbfw\xf4\xf7Sd\x06\xf1%\xa5U\x82\xb5\xa7\xcbGN0:e3\xd5\t\xbf\x06\xe5\f\x8c\xea\x11\"\xe6\x1c\x90\xdcI\xb4\xe2\xc2\x1d\xfc\xea#\x02\xb9\xc1\xf7p\x10\t\xdco6{\x92e\xa6\xb4\x1f\xc7\xe4H\x1e7e2h\x97\xc4G\xde\x18\x9c\xd0n\x98\xf6\xad\x8a\xfa@\x82ZRč\n\xd4\x16\xe0nV\xf9h\xbe\x8au\x00\xf9\xe5\tRy\xcc\xe2`\x89\xe4\xf6O\xe6\"\xf1\xab\xbcgm\xcfm\x9f\xaf\xcd\xf8\x8f\xf4fSf\xe5\xddO\xf7\xefaIZZ\xb0漰}\xbc\xc6G\xe23Q\xe4\x06\x8cs\xe3\x86\xe8\xc7\x12\x11\x9d\t\x9e\x9c\x94\x83\xb6\x84nM:\xa7\xddH\x92;\xfdgB\x96ܟ\x0en\xcbf\x81\x1dB\nF\t\x9a\x0e^;\xb8U#\xda[\xc5\xf8\xc5i\xcf\fs\x9b)\xfd<\xf1\xa7\vq\xed8\xb3u\x1c\xa2\xba\xaa.v\xe8\xf2\xa4\xde\aԫA\xc91h\xa0:\xb9\x83\x8f\xa0Vl\xd6)\xbe\x1c\xad;q\xbd4\xc00o\xf0\x81\xf6k\x1b\x802\xa6l\u007fe\xef\xaeܻJυZoK\x8e,\xc7\\@\x88~\"\x83\xb1]j\xab\x18R\xacE\x96\xdd\xd85\x97r\x9d1\\\v+\xe1\xce\xe1\xad\x10\xdcU\xa7\x8c!Ӻ\\\x9a\xf7\x0e\xd6\xf5W\x96\xa1\xda\xe3\xe5\xdc\xcf\xea\xcc\n\xa6\x88\xab)l\x9fB\u007fV\x1d\xa2$\xf1\u007f\xd5G\xb9T=wU#:ňNjD\xf0\xc3\n\xbe\xfa\xff\x1a\t\a\xc5\xf8I~/Ǿ\xcb\xf7\x16\xca-\r\xa8\x1f-\xce\xd1\xca6\u007f&\xa8\u007f\x8d4\u007f\xe8\xd2x\x0e\xaa\x85W\x93\"\xabv\x16\x9f\xfd\xf9\xe0ԕ\u007fW\xfa{\xa1mg\xa6\xe3\x03\xe7\xe6x*\xe4\xb5˃\xe6f~!\xe4\xa5iz\x90\x98\xe6\xe4Ui\xd5rԂ\xd2\x1a\x83\xa0y{\xfe\x96y\xf1b\xf5\x1c)G\xed\xdd<\xa6\xdc\xc3o\xbf7sT4\xdb\x05G6\xfe\x13\x00\x00\xff\xff\x9f\xfe\xa5\x85\f\n\x00\x00"), +} + +var CRDs = crds() + +func crds() []*apiextv1beta1.CustomResourceDefinition { + apiextinstall.Install(scheme.Scheme) + decode := scheme.Codecs.UniversalDeserializer().Decode + var objs []*apiextv1beta1.CustomResourceDefinition + for _, crd := range rawCRDs { + gzr, err := gzip.NewReader(bytes.NewReader(crd)) + if err != nil { + panic(err) + } + bytes, err := ioutil.ReadAll(gzr) + if err != nil { + panic(err) + } + gzr.Close() + + obj, _, err := decode(bytes, nil, nil) + if err != nil { + panic(err) + } + objs = append(objs, obj.(*apiextv1beta1.CustomResourceDefinition)) + } + return objs +} diff --git a/pkg/generated/crds/doc.go b/config/crd/crds/doc.go similarity index 100% rename from pkg/generated/crds/doc.go rename to config/crd/crds/doc.go diff --git a/config/samples/velero_v1_backupstoragelocation.yaml b/config/samples/velero_v1_backupstoragelocation.yaml new file mode 100644 index 000000000..7336238cb --- /dev/null +++ b/config/samples/velero_v1_backupstoragelocation.yaml @@ -0,0 +1,16 @@ +apiVersion: velero.io/v1 +kind: BackupStorageLocation +metadata: + creationTimestamp: null + labels: + component: velero + name: default + namespace: velero +spec: + config: + region: minio + s3ForcePathStyle: "true" + s3Url: http://minio.velero.svc:9000 + objectStorage: + bucket: velero + provider: aws \ No newline at end of file diff --git a/go.mod b/go.mod index 57799c6b4..45c6dbd2c 100644 --- a/go.mod +++ b/go.mod @@ -30,7 +30,7 @@ require ( github.com/spf13/cobra v0.0.5 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.4.0 - golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553 + golang.org/x/net v0.0.0-20200602114024-627f9648deb9 google.golang.org/grpc v1.26.0 k8s.io/api v0.17.4 k8s.io/apiextensions-apiserver v0.17.4 @@ -39,4 +39,5 @@ require ( k8s.io/client-go v0.17.4 k8s.io/klog v1.0.0 k8s.io/utils v0.0.0-20191218082557-f07c713de883 // indirect + sigs.k8s.io/controller-runtime v0.5.2 ) diff --git a/go.sum b/go.sum index 5256daaee..4794e33fc 100644 --- a/go.sum +++ b/go.sum @@ -58,7 +58,9 @@ github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdko github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= @@ -103,6 +105,7 @@ github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDD github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/docker/spdystream v0.0.0-20170912183627-bc6354cbbc29 h1:llBx5m8Gk0lrAaiLud2wktkX/e8haX7Ru0oVfQqtZQ4= github.com/docker/spdystream v0.0.0-20170912183627-bc6354cbbc29/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= +github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e h1:p1yVGRW3nmb85p1Sh1ZJSDm4A4iKLS5QNbvUHMgGu/M= @@ -125,7 +128,10 @@ github.com/go-ini/ini v1.28.2 h1:drmmYv7psRpoGZkPtPKKTB+ZFSnvmwCMfNj5o1nLh2Y= github.com/go-ini/ini v1.28.2/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logr/logr v0.1.0 h1:M1Tv3VzNlEHg6uyACnRdtrploV2P7wZqH8BoQMtz0cg= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= +github.com/go-logr/zapr v0.1.0 h1:h+WVe9j6HAA01niTJPA/kKH0i7e0rLZBCwauQFcRE54= +github.com/go-logr/zapr v0.1.0/go.mod h1:tabnROwaDl0UNxkVeFRbY8bwB37GwRv0P8lg6aAiEnk= github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI= github.com/go-openapi/analysis v0.17.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= github.com/go-openapi/analysis v0.18.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= @@ -212,6 +218,8 @@ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5m github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/googleapis/gnostic v0.2.0 h1:l6N3VoaVzTncYYW+9yOz2LJJammFZGBO13sqgEhpy9g= github.com/googleapis/gnostic v0.2.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= +github.com/googleapis/gnostic v0.3.1 h1:WeAefnSUHlBb0iJKwxFDZdbfGwkd7xRNuV+IpXMJhYk= +github.com/googleapis/gnostic v0.3.1/go.mod h1:on+2t9HRStVgn95RSsFWFz+6Q0Snyqv1awfrALZdbtU= github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= @@ -234,6 +242,7 @@ github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKe github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.7 h1:Y+UAYTZ7gDEuOfhxKWy+dvb5dRQ6rJjFSdX2HZY1/gI= github.com/imdario/mergo v0.3.7/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= @@ -306,9 +315,13 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.2 h1:uqH7bpe+ERSiDa34FDOF7RikN6RzXgduUF8yarlZp94= github.com/onsi/ginkgo v1.10.2/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.11.0 h1:JAKSXpt1YjtLA7YpPiqO9ss6sNXEsPfSGdwN0UHqzrw= +github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.8.1 h1:C5Dqfs/LeauYDX0jJXIe2SWmwCbGzx9yF8C8xy3Lh34= +github.com/onsi/gomega v1.8.1/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= @@ -383,8 +396,11 @@ go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qL go.mongodb.org/mongo-driver v1.1.2/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.uber.org/atomic v1.3.2 h1:2Oa65PReHzfn29GpvgsYwloV9AVFHPDk8tYxt2c2tr4= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -396,6 +412,7 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975 h1:/Tl7pH94bvbAAHBdZJT947M/+gp0+CqQXDtMRC0fseo= golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -439,6 +456,8 @@ golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553 h1:efeOvDhwQ29Dj3SdAV/MJf8oukgn+8D8WgaCaRMchF8= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200602114024-627f9648deb9 h1:pNX+40auqi2JqRfOP1akLGtYcn15TUbkhwuCO3foqqM= +golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0= @@ -468,8 +487,11 @@ golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191220220014-0732a990476f h1:72l8qCJ1nGxMGH26QVBVIxKd/D34cfGt0OvrPtpemyY= golang.org/x/sys v0.0.0-20191220220014-0732a990476f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -502,7 +524,10 @@ golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 h1:9zdDQZ7Thm29KFXgAX/+yaf3eVbP7djjWp/dXAppNCc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gomodules.xyz/jsonpatch/v2 v2.0.1 h1:xyiBuvkD2g5n7cYzx6u2sxQvsAy4QJsZFCzGVdzOXZ0= +gomodules.xyz/jsonpatch/v2 v2.0.1/go.mod h1:IhYNNY4jnS53ZnfE4PAmpKtDpTCj1JFXc+3mwe7XcUU= gonum.org/v1/gonum v0.0.0-20190331200053-3d26580ed485/go.mod h1:2ltnJ7xHfj0zHS40VVPYEAAMTa3ZGguvHGBSJeRWqE0= gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= gonum.org/v1/netlib v0.0.0-20190331212654-76723241ea4e/go.mod h1:kS+toOQn6AQKjmKJ7gzohV1XkqsFehRA2FbsbkopSuQ= @@ -533,10 +558,12 @@ google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyac google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.26.0 h1:2dTRdpdFEEhJYQD8EMLB61nnrzSCTbG38PhqdhvOltg= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= @@ -552,6 +579,7 @@ gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bl gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= @@ -561,23 +589,30 @@ honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= k8s.io/api v0.17.0/go.mod h1:npsyOePkeP0CPwyGfXDHxvypiYMJxBWAMpQxCaJ4ZxI= +k8s.io/api v0.17.2/go.mod h1:BS9fjjLc4CMuqfSO8vgbHPKMt5+SF0ET6u/RVDihTo4= k8s.io/api v0.17.4 h1:HbwOhDapkguO8lTAE8OX3hdF2qp8GtpC9CW/MQATXXo= k8s.io/api v0.17.4/go.mod h1:5qxx6vjmwUVG2nHQTKGlLts8Tbok8PzHl4vHtVFuZCA= +k8s.io/apiextensions-apiserver v0.17.2/go.mod h1:4KdMpjkEjjDI2pPfBA15OscyNldHWdBCfsWMDWAmSTs= k8s.io/apiextensions-apiserver v0.17.4 h1:ZKFnw3cJrGZ/9s6y+DerTF4FL+dmK0a04A++7JkmMho= k8s.io/apiextensions-apiserver v0.17.4/go.mod h1:rCbbbaFS/s3Qau3/1HbPlHblrWpFivoaLYccCffvQGI= k8s.io/apimachinery v0.17.0/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg= k8s.io/apimachinery v0.17.1-beta.0/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg= +k8s.io/apimachinery v0.17.2/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg= k8s.io/apimachinery v0.17.4 h1:UzM+38cPUJnzqSQ+E1PY4YxMHIzQyCg29LOoGfo79Zw= k8s.io/apimachinery v0.17.4/go.mod h1:gxLnyZcGNdZTCLnq3fgzyg2A5BVCHTNDFrw8AmuJ+0g= +k8s.io/apiserver v0.17.2/go.mod h1:lBmw/TtQdtxvrTk0e2cgtOxHizXI+d0mmGQURIHQZlo= k8s.io/apiserver v0.17.4/go.mod h1:5ZDQ6Xr5MNBxyi3iUZXS84QOhZl+W7Oq2us/29c0j9I= k8s.io/cli-runtime v0.17.4 h1:ZIJdxpBEszZqUhydrCoiI5rLXS2J/1AF5xFok2QJ9bc= k8s.io/cli-runtime v0.17.4/go.mod h1:IVW4zrKKx/8gBgNNkhiUIc7nZbVVNhc1+HcQh+PiNHc= k8s.io/client-go v0.17.0/go.mod h1:TYgR6EUHs6k45hb6KWjVD6jFZvJV4gHDikv/It0xz+k= +k8s.io/client-go v0.17.2/go.mod h1:QAzRgsa0C2xl4/eVpeVAZMvikCn8Nm81yqVx3Kk9XYI= k8s.io/client-go v0.17.4 h1:VVdVbpTY70jiNHS1eiFkUt7ZIJX3txd29nDxxXH4en8= k8s.io/client-go v0.17.4/go.mod h1:ouF6o5pz3is8qU0/qYL2RnoxOPqgfuidYLowytyLJmc= k8s.io/code-generator v0.0.0-20191121015212-c4c8f8345c7e/go.mod h1:DVmfPQgxQENqDIzVR2ddLXMH34qeszkKSdH/N+s+38s= +k8s.io/code-generator v0.17.2/go.mod h1:DVmfPQgxQENqDIzVR2ddLXMH34qeszkKSdH/N+s+38s= k8s.io/code-generator v0.17.4/go.mod h1:l8BLVwASXQZTo2xamW5mQNFCe1XPiAesVq7Y1t7PiQQ= k8s.io/component-base v0.17.0/go.mod h1:rKuRAokNMY2nn2A6LP/MiwpoaMRHpfRnrPaUJJj1Yoc= +k8s.io/component-base v0.17.2/go.mod h1:zMPW3g5aH7cHJpKYQ/ZsGMcgbsA/VyhEugF3QT1awLs= k8s.io/component-base v0.17.4/go.mod h1:5BRqHMbbQPm2kKu35v3G+CpVq4K0RJKC7TRioF0I9lE= k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20190822140433-26a664648505/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= @@ -597,6 +632,8 @@ modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03 modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs= modernc.org/xc v1.0.0/go.mod h1:mRNCo0bvLjGhHO9WsyuKVU4q0ceiDDDoEeWDJHrNx8I= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +sigs.k8s.io/controller-runtime v0.5.2 h1:pyXbUfoTo+HA3jeIfr0vgi+1WtmNh0CwlcnQGLXwsSw= +sigs.k8s.io/controller-runtime v0.5.2/go.mod h1:JZUwSMVbxDupo0lTJSSFP5pimEyxGynROImSsqIOx1A= sigs.k8s.io/kustomize v2.0.3+incompatible/go.mod h1:MkjgH3RdOWrievjo6c9T245dYlB5QeXV4WCbnt/PEpU= sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI= sigs.k8s.io/structured-merge-diff v1.0.1-0.20191108220359-b1b620dd3f06/go.mod h1:/ULNhyfzRopfcjskuui0cTITekDduZ7ycKN3oUT9R18= diff --git a/hack/crd-gen/main.go b/hack/crd-gen/main.go index edd759aef..4b44990aa 100644 --- a/hack/crd-gen/main.go +++ b/hack/crd-gen/main.go @@ -14,8 +14,8 @@ See the License for the specific language governing permissions and limitations under the License. */ -// This code embeds the CRD manifests in pkg/generated/crds/manifests in -// pkg/generated/crds/crds.go. +// This code embeds the CRD manifests in config/crd/bases in +// config/crd/crds/crds.go. package main @@ -30,7 +30,7 @@ import ( "text/template" ) -// This is relative to pkg/generated/crds +// This is relative to config/crd/crds const goHeaderFile = "../../../hack/boilerplate.go.txt" const tpl = `{{.GoHeader}} @@ -96,14 +96,14 @@ func main() { GoHeader: string(headerBytes), } - // This is relative to pkg/generated/crds - manifests, err := ioutil.ReadDir("manifests") + // This is relative to config/crd/crds + manifests, err := ioutil.ReadDir("../bases") if err != nil { log.Fatalln(err) } for _, crd := range manifests { - file, err := os.Open("manifests/" + crd.Name()) + file, err := os.Open("../bases/" + crd.Name()) if err != nil { log.Fatalln(err) } @@ -120,7 +120,7 @@ func main() { data.RawCRDs = append(data.RawCRDs, fmt.Sprintf("%q", buf.Bytes())) } - t, err := template.New("crds").Parse(tpl) + t, err := template.New("crd").Parse(tpl) if err != nil { log.Fatalln(err) } diff --git a/hack/update-generated-crd-code.sh b/hack/update-generated-crd-code.sh index 0873ca447..890835939 100755 --- a/hack/update-generated-crd-code.sh +++ b/hack/update-generated-crd-code.sh @@ -44,9 +44,12 @@ ${GOPATH}/src/k8s.io/code-generator/generate-groups.sh \ --output-base ../../.. \ $@ +# Generate manifests e.g. CRD, RBAC etc. controller-gen \ - crd:crdVersions=v1beta1,preserveUnknownFields=false \ - output:dir=./pkg/generated/crds/manifests \ - paths=./pkg/apis/velero/v1/... + crd:crdVersions=v1beta1,preserveUnknownFields=false,trivialVersions=true \ + rbac:roleName=manager-role \ + paths=./pkg/apis/velero/v1/... \ + paths=./pkg/controller/... \ + output:crd:artifacts:config=config/crd/bases -go generate ./pkg/generated/crds +go generate ./config/crd/crds diff --git a/hack/verify-generated-crd-code.sh b/hack/verify-generated-crd-code.sh index 698f8d11b..3d77a2304 100755 --- a/hack/verify-generated-crd-code.sh +++ b/hack/verify-generated-crd-code.sh @@ -19,11 +19,11 @@ HACK_DIR=$(dirname "${BASH_SOURCE}") ${HACK_DIR}/update-generated-crd-code.sh --verify-only # ensure no changes to generated CRDs -if ! git diff --exit-code pkg/generated/crds/crds.go >/dev/null; then +if ! git diff --exit-code config/crd/crds/crds.go >/dev/null; then # revert changes to state before running CRD generation to stay consistent # with code-generator `--verify-only` option which discards generated changes - git checkout pkg/generated/crds + git checkout config/crd/bases echo "CRD verification - failed! Generated CRDs are out-of-date, please run 'make update'." exit 1 -fi +fi \ No newline at end of file diff --git a/internal/util/managercontroller/managercontroller.go b/internal/util/managercontroller/managercontroller.go new file mode 100644 index 000000000..5a009d7fa --- /dev/null +++ b/internal/util/managercontroller/managercontroller.go @@ -0,0 +1,54 @@ +/* +Copyright 2020 the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// TODO(2.0) After converting all controllers to runttime-controller, +// the functions in this file will no longer be needed and should be removed. +package managercontroller + +import ( + "context" + + "sigs.k8s.io/controller-runtime/pkg/manager" + + "github.com/vmware-tanzu/velero/pkg/controller" +) + +// Runnable will turn a "regular" runnable component (such as a controller) +// into a controller-runtime Runnable +func Runnable(p controller.Interface, numWorkers int) manager.Runnable { + f := func(stop <-chan struct{}) error { + + // Create a cancel context for handling the stop signal. + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // If a signal is received on the stop channel, cancel the + // context. This will propagate the cancel into the p.Run + // function below. + go func() { + select { + case <-stop: + cancel() + case <-ctx.Done(): + } + }() + + // This is a blocking call that either completes + // or is cancellable on receiving a stop signal. + return p.Run(ctx, numWorkers) + } + return manager.RunnableFunc(f) +} diff --git a/pkg/apis/velero/v1/backup_storage_location.go b/pkg/apis/velero/v1/backupstoragelocation_types.go similarity index 78% rename from pkg/apis/velero/v1/backup_storage_location.go rename to pkg/apis/velero/v1/backupstoragelocation_types.go index 3d74b647e..675c1852d 100644 --- a/pkg/apis/velero/v1/backup_storage_location.go +++ b/pkg/apis/velero/v1/backupstoragelocation_types.go @@ -1,5 +1,5 @@ /* -Copyright 2018 the Velero contributors. +Copyright 2017, 2020 the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -21,56 +21,7 @@ import ( "k8s.io/apimachinery/pkg/types" ) -// +genclient -// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object - -// BackupStorageLocation is a location where Velero stores backup objects. -type BackupStorageLocation struct { - metav1.TypeMeta `json:",inline"` - - // +optional - metav1.ObjectMeta `json:"metadata,omitempty"` - - // +optional - Spec BackupStorageLocationSpec `json:"spec,omitempty"` - - // +optional - Status BackupStorageLocationStatus `json:"status,omitempty"` -} - -// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object - -// BackupStorageLocationList is a list of BackupStorageLocations. -type BackupStorageLocationList struct { - metav1.TypeMeta `json:",inline"` - - // +optional - metav1.ListMeta `json:"metadata,omitempty"` - - Items []BackupStorageLocation `json:"items"` -} - -// StorageType represents the type of storage that a backup location uses. -// ObjectStorage must be non-nil, since it is currently the only supported StorageType. -type StorageType struct { - ObjectStorage *ObjectStorageLocation `json:"objectStorage"` -} - -// ObjectStorageLocation specifies the settings necessary to connect to a provider's object storage. -type ObjectStorageLocation struct { - // Bucket is the bucket to use for object storage. - Bucket string `json:"bucket"` - - // Prefix is the path inside a bucket to use for Velero storage. Optional. - // +optional - Prefix string `json:"prefix,omitempty"` - - // CACert defines a CA bundle to use when verifying TLS connections to the provider. - // +optional - CACert []byte `json:"caCert,omitempty"` -} - -// BackupStorageLocationSpec defines the specification for a Velero BackupStorageLocation. +// BackupStorageLocationSpec defines the desired state of a Velero BackupStorageLocation type BackupStorageLocationSpec struct { // Provider is the provider of the backup storage. Provider string `json:"provider"` @@ -91,34 +42,7 @@ type BackupStorageLocationSpec struct { BackupSyncPeriod *metav1.Duration `json:"backupSyncPeriod,omitempty"` } -// BackupStorageLocationPhase is the lifecyle phase of a Velero BackupStorageLocation. -// +kubebuilder:validation:Enum=Available;Unavailable -type BackupStorageLocationPhase string - -const ( - // BackupStorageLocationPhaseAvailable means the location is available to read and write from. - BackupStorageLocationPhaseAvailable BackupStorageLocationPhase = "Available" - - // BackupStorageLocationPhaseUnavailable means the location is unavailable to read and write from. - BackupStorageLocationPhaseUnavailable BackupStorageLocationPhase = "Unavailable" -) - -// BackupStorageLocationAccessMode represents the permissions for a BackupStorageLocation. -// +kubebuilder:validation:Enum=ReadOnly;ReadWrite -type BackupStorageLocationAccessMode string - -const ( - // BackupStorageLocationAccessModeReadOnly represents read-only access to a BackupStorageLocation. - BackupStorageLocationAccessModeReadOnly BackupStorageLocationAccessMode = "ReadOnly" - - // BackupStorageLocationAccessModeReadWrite represents read and write access to a BackupStorageLocation. - BackupStorageLocationAccessModeReadWrite BackupStorageLocationAccessMode = "ReadWrite" -) - -// TODO(2.0): remove the AccessMode field from BackupStorageLocationStatus. -// TODO(2.0): remove the LastSyncedRevision field from BackupStorageLocationStatus. - -// BackupStorageLocationStatus describes the current status of a Velero BackupStorageLocation. +// BackupStorageLocationStatus defines the observed state of BackupStorageLocation type BackupStorageLocationStatus struct { // Phase is the current state of the BackupStorageLocation. // +optional @@ -145,3 +69,82 @@ type BackupStorageLocationStatus struct { // +optional AccessMode BackupStorageLocationAccessMode `json:"accessMode,omitempty"` } + +// TODO(2.0) After converting all resources to use the runttime-controller client, +// the genclient and k8s:deepcopy markers will no longer be needed and should be removed. +// +genclient +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +kubebuilder:object:root=true +// +kubebuilder:object:generate=true +// +kubebuilder:storageversion +// +kubebuilder:subresource:status +// +kubebuilder:printcolumn:name="Phase",type="string",JSONPath=".status.phase",description="Backup Storage Location status such as Available/Unavailable" +// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" + +// BackupStorageLocation is a location where Velero stores backup objects +type BackupStorageLocation struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec BackupStorageLocationSpec `json:"spec,omitempty"` + Status BackupStorageLocationStatus `json:"status,omitempty"` +} + +// TODO(2.0) After converting all resources to use the runttime-controller client, +// the k8s:deepcopy marker will no longer be needed and should be removed. +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object +// +kubebuilder:object:root=true + +// BackupStorageLocationList contains a list of BackupStorageLocation +type BackupStorageLocationList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []BackupStorageLocation `json:"items"` +} + +// StorageType represents the type of storage that a backup location uses. +// ObjectStorage must be non-nil, since it is currently the only supported StorageType. +type StorageType struct { + ObjectStorage *ObjectStorageLocation `json:"objectStorage"` +} + +// ObjectStorageLocation specifies the settings necessary to connect to a provider's object storage. +type ObjectStorageLocation struct { + // Bucket is the bucket to use for object storage. + Bucket string `json:"bucket"` + + // Prefix is the path inside a bucket to use for Velero storage. Optional. + // +optional + Prefix string `json:"prefix,omitempty"` + + // CACert defines a CA bundle to use when verifying TLS connections to the provider. + // +optional + CACert []byte `json:"caCert,omitempty"` +} + +// BackupStorageLocationPhase is the lifecycle phase of a Velero BackupStorageLocation. +// +kubebuilder:validation:Enum=Available;Unavailable +type BackupStorageLocationPhase string + +const ( + // BackupStorageLocationPhaseAvailable means the location is available to read and write from. + BackupStorageLocationPhaseAvailable BackupStorageLocationPhase = "Available" + + // BackupStorageLocationPhaseUnavailable means the location is unavailable to read and write from. + BackupStorageLocationPhaseUnavailable BackupStorageLocationPhase = "Unavailable" +) + +// BackupStorageLocationAccessMode represents the permissions for a BackupStorageLocation. +// +kubebuilder:validation:Enum=ReadOnly;ReadWrite +type BackupStorageLocationAccessMode string + +const ( + // BackupStorageLocationAccessModeReadOnly represents read-only access to a BackupStorageLocation. + BackupStorageLocationAccessModeReadOnly BackupStorageLocationAccessMode = "ReadOnly" + + // BackupStorageLocationAccessModeReadWrite represents read and write access to a BackupStorageLocation. + BackupStorageLocationAccessModeReadWrite BackupStorageLocationAccessMode = "ReadWrite" +) + +// TODO(2.0): remove the AccessMode field from BackupStorageLocationStatus. +// TODO(2.0): remove the LastSyncedRevision field from BackupStorageLocationStatus. diff --git a/pkg/apis/velero/v1/groupversion_info.go b/pkg/apis/velero/v1/groupversion_info.go new file mode 100644 index 000000000..ab5b20433 --- /dev/null +++ b/pkg/apis/velero/v1/groupversion_info.go @@ -0,0 +1,36 @@ +/* +Copyright 2020 the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package v1 contains API Schema definitions for the velero v1 API group +// +kubebuilder:object:generate=true +// +groupName=velero.io +package v1 + +import ( + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +var ( + // SchemeGroupVersion is group version used to register these objects + SchemeGroupVersion = schema.GroupVersion{Group: "velero.io", Version: "v1"} + + // SchemeBuilder is used to add go types to the GroupVersionKind scheme + SchemeBuilder = runtime.NewSchemeBuilder(addKnownTypes) + + // AddToScheme adds the types in this group-version to the given scheme. + AddToScheme = SchemeBuilder.AddToScheme +) diff --git a/pkg/apis/velero/v1/register.go b/pkg/apis/velero/v1/register.go index 6eb7051e8..ea7df3b5d 100644 --- a/pkg/apis/velero/v1/register.go +++ b/pkg/apis/velero/v1/register.go @@ -22,20 +22,6 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" ) -var ( - // SchemeBuilder collects the scheme builder functions for the Velero API - SchemeBuilder = runtime.NewSchemeBuilder(addKnownTypes) - - // AddToScheme applies the SchemeBuilder functions to a specified scheme - AddToScheme = SchemeBuilder.AddToScheme -) - -// GroupName is the group name for the Velero API -const GroupName = "velero.io" - -// SchemeGroupVersion is the GroupVersion for the Velero API -var SchemeGroupVersion = schema.GroupVersion{Group: GroupName, Version: "v1"} - // Resource gets a Velero GroupResource for a specified resource func Resource(resource string) schema.GroupResource { return SchemeGroupVersion.WithResource(resource).GroupResource() diff --git a/pkg/client/factory.go b/pkg/client/factory.go index 03cad73d7..e46b81dc1 100644 --- a/pkg/client/factory.go +++ b/pkg/client/factory.go @@ -19,13 +19,16 @@ package client import ( "os" + kbclient "sigs.k8s.io/controller-runtime/pkg/client" + "github.com/pkg/errors" "github.com/spf13/pflag" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" - v1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" clientset "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned" ) @@ -42,6 +45,9 @@ type Factory interface { // DynamicClient returns a Kubernetes dynamic client. It uses the following priority to specify the cluster // configuration: --kubeconfig flag, KUBECONFIG environment variable, in-cluster configuration. DynamicClient() (dynamic.Interface, error) + // KubebuilderClient returns a Kubernetes dynamic client. It uses the following priority to specify the cluster + // configuration: --kubeconfig flag, KUBECONFIG environment variable, in-cluster configuration. + KubebuilderClient() (kbclient.Client, error) // SetBasename changes the basename for an already-constructed client. // This is useful for generating clients that require a different user-agent string below the root `velero` // command, such as the server subcommand. @@ -81,7 +87,7 @@ func NewFactory(baseName string, config VeleroConfig) Factory { // We didn't get the namespace via env var or config file, so use the default. // Command line flags will override when BindFlags is called. if f.namespace == "" { - f.namespace = v1.DefaultNamespace + f.namespace = velerov1api.DefaultNamespace } f.flags.StringVar(&f.kubeconfig, "kubeconfig", "", "Path to the kubeconfig file to use to talk to the Kubernetes apiserver. If unset, try the environment variable KUBECONFIG, as well as in-cluster configuration") @@ -137,6 +143,21 @@ func (f *factory) DynamicClient() (dynamic.Interface, error) { return dynamicClient, nil } +func (f *factory) KubebuilderClient() (kbclient.Client, error) { + clientConfig, err := f.ClientConfig() + if err != nil { + return nil, err + } + + scheme := runtime.NewScheme() + velerov1api.AddToScheme(scheme) + kubebuilderClient, err := kbclient.New(clientConfig, kbclient.Options{ + Scheme: scheme, + }) + + return kubebuilderClient, nil +} + func (f *factory) SetBasename(name string) { f.baseName = name } diff --git a/pkg/cmd/cli/backup/create.go b/pkg/cmd/cli/backup/create.go index d73928c0d..fe310f962 100644 --- a/pkg/cmd/cli/backup/create.go +++ b/pkg/cmd/cli/backup/create.go @@ -17,9 +17,12 @@ limitations under the License. package backup import ( + "context" "fmt" "time" + kbclient "sigs.k8s.io/controller-runtime/pkg/client" + "github.com/spf13/cobra" "github.com/spf13/pflag" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -146,13 +149,22 @@ func (o *CreateOptions) Validate(c *cobra.Command, args []string, f client.Facto return err } + client, err := f.KubebuilderClient() + if err != nil { + return err + } + // Ensure that unless FromSchedule is set, args contains a backup name if o.FromSchedule == "" && len(args) != 1 { return fmt.Errorf("a backup name is required, unless you are creating based on a schedule") } if o.StorageLocation != "" { - if _, err := o.client.VeleroV1().BackupStorageLocations(f.Namespace()).Get(o.StorageLocation, metav1.GetOptions{}); err != nil { + location := &velerov1api.BackupStorageLocation{} + if err := client.Get(context.Background(), kbclient.ObjectKey{ + Namespace: f.Namespace(), + Name: o.StorageLocation, + }, location); err != nil { return err } } diff --git a/pkg/cmd/cli/backuplocation/create.go b/pkg/cmd/cli/backuplocation/create.go index a4dfd5ad4..633fd5593 100644 --- a/pkg/cmd/cli/backuplocation/create.go +++ b/pkg/cmd/cli/backuplocation/create.go @@ -17,6 +17,7 @@ limitations under the License. package backuplocation import ( + "context" "fmt" "strings" "time" @@ -26,6 +27,8 @@ import ( "github.com/spf13/pflag" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kbclient "sigs.k8s.io/controller-runtime/pkg/client" + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/cmd" @@ -146,12 +149,12 @@ func (o *CreateOptions) Run(c *cobra.Command, f client.Factory) error { return err } - client, err := f.Client() + client, err := f.KubebuilderClient() if err != nil { return err } - if _, err := client.VeleroV1().BackupStorageLocations(backupStorageLocation.Namespace).Create(backupStorageLocation); err != nil { + if err := client.Create(context.Background(), backupStorageLocation, &kbclient.CreateOptions{}); err != nil { return errors.WithStack(err) } diff --git a/pkg/cmd/cli/backuplocation/get.go b/pkg/cmd/cli/backuplocation/get.go index 32102c169..c0dbe4148 100644 --- a/pkg/cmd/cli/backuplocation/get.go +++ b/pkg/cmd/cli/backuplocation/get.go @@ -17,10 +17,14 @@ limitations under the License. package backuplocation import ( + "context" + + kbclient "sigs.k8s.io/controller-runtime/pkg/client" + "github.com/spf13/cobra" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/cmd" "github.com/vmware-tanzu/velero/pkg/cmd/util/output" @@ -36,19 +40,24 @@ func NewGetCommand(f client.Factory, use string) *cobra.Command { err := output.ValidateFlags(c) cmd.CheckError(err) - veleroClient, err := f.Client() + client, err := f.KubebuilderClient() cmd.CheckError(err) - var locations *api.BackupStorageLocationList + locations := new(velerov1api.BackupStorageLocationList) if len(args) > 0 { - locations = new(api.BackupStorageLocationList) + location := &velerov1api.BackupStorageLocation{} for _, name := range args { - location, err := veleroClient.VeleroV1().BackupStorageLocations(f.Namespace()).Get(name, metav1.GetOptions{}) + err = client.Get(context.Background(), kbclient.ObjectKey{ + Namespace: f.Namespace(), + Name: name, + }, location) cmd.CheckError(err) locations.Items = append(locations.Items, *location) } } else { - locations, err = veleroClient.VeleroV1().BackupStorageLocations(f.Namespace()).List(listOptions) + err := client.List(context.Background(), locations, &kbclient.ListOptions{ + Namespace: f.Namespace(), + }) cmd.CheckError(err) } diff --git a/pkg/cmd/cli/restic/server.go b/pkg/cmd/cli/restic/server.go index 2d27386eb..d72fa242e 100644 --- a/pkg/cmd/cli/restic/server.go +++ b/pkg/cmd/cli/restic/server.go @@ -20,19 +20,24 @@ import ( "fmt" "os" "strings" - "sync" + + "github.com/vmware-tanzu/velero/internal/util/managercontroller" + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/spf13/cobra" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/sets" kubeinformers "k8s.io/client-go/informers" corev1informers "k8s.io/client-go/informers/core/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/cache" + ctrl "sigs.k8s.io/controller-runtime" + "github.com/vmware-tanzu/velero/pkg/buildinfo" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/cmd" @@ -43,6 +48,13 @@ import ( "github.com/vmware-tanzu/velero/pkg/restic" "github.com/vmware-tanzu/velero/pkg/util/filesystem" "github.com/vmware-tanzu/velero/pkg/util/logging" + + "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/controller-runtime/pkg/manager" +) + +var ( + scheme = runtime.NewScheme() ) func NewServerCommand(f client.Factory) *cobra.Command { @@ -86,6 +98,7 @@ type resticServer struct { ctx context.Context cancelFunc context.CancelFunc fileSystem filesystem.Interface + mgr manager.Manager } func newResticServer(logger logrus.FieldLogger, factory client.Factory) (*resticServer, error) { @@ -130,6 +143,20 @@ func newResticServer(logger logrus.FieldLogger, factory client.Factory) (*restic ctx, cancelFunc := context.WithCancel(context.Background()) + clientConfig, err := factory.ClientConfig() + if err != nil { + return nil, err + } + + ctrl.SetLogger(zap.New(zap.UseDevMode(true))) + velerov1api.AddToScheme(scheme) + mgr, err := ctrl.NewManager(clientConfig, ctrl.Options{ + Scheme: scheme, + }) + if err != nil { + return nil, err + } + s := &resticServer{ kubeClient: kubeClient, veleroClient: veleroClient, @@ -141,6 +168,7 @@ func newResticServer(logger logrus.FieldLogger, factory client.Factory) (*restic ctx: ctx, cancelFunc: cancelFunc, fileSystem: filesystem.NewFileSystem(), + mgr: mgr, } if err := s.validatePodVolumesHostPath(); err != nil { @@ -155,8 +183,6 @@ func (s *resticServer) run() { s.logger.Info("Starting controllers") - var wg sync.WaitGroup - backupController := controller.NewPodVolumeBackupController( s.logger, s.veleroInformerFactory.Velero().V1().PodVolumeBackups(), @@ -165,14 +191,9 @@ func (s *resticServer) run() { s.secretInformer, s.kubeInformerFactory.Core().V1().PersistentVolumeClaims(), s.kubeInformerFactory.Core().V1().PersistentVolumes(), - s.veleroInformerFactory.Velero().V1().BackupStorageLocations(), + s.mgr.GetClient(), os.Getenv("NODE_NAME"), ) - wg.Add(1) - go func() { - defer wg.Done() - backupController.Run(s.ctx, 1) - }() restoreController := controller.NewPodVolumeRestoreController( s.logger, @@ -182,26 +203,30 @@ func (s *resticServer) run() { s.secretInformer, s.kubeInformerFactory.Core().V1().PersistentVolumeClaims(), s.kubeInformerFactory.Core().V1().PersistentVolumes(), - s.veleroInformerFactory.Velero().V1().BackupStorageLocations(), + s.mgr.GetClient(), os.Getenv("NODE_NAME"), ) - wg.Add(1) - go func() { - defer wg.Done() - restoreController.Run(s.ctx, 1) - }() go s.veleroInformerFactory.Start(s.ctx.Done()) go s.kubeInformerFactory.Start(s.ctx.Done()) go s.podInformer.Run(s.ctx.Done()) go s.secretInformer.Run(s.ctx.Done()) - s.logger.Info("Controllers started successfully") + // TODO(2.0): presuming all controllers and resources are converted to runtime-controller + // by v2.0, the block from this line and including the `s.mgr.Start() will be + // deprecated, since the manager auto-starts all the caches. Until then, we need to start the + // cache for them manually. - <-s.ctx.Done() + // Adding the controllers to the manager will register them as a (runtime-controller) runnable, + // so the manager will ensure the cache is started and ready before all controller are started + s.mgr.Add(managercontroller.Runnable(backupController, 1)) + s.mgr.Add(managercontroller.Runnable(restoreController, 1)) - s.logger.Info("Waiting for all controllers to shut down gracefully") - wg.Wait() + s.logger.Info("Controllers starting...") + + if err := s.mgr.Start(ctrl.SetupSignalHandler()); err != nil { + s.logger.Fatal("Problem starting manager", err) + } } // validatePodVolumesHostPath validates that the pod volumes path contains a diff --git a/pkg/cmd/server/server.go b/pkg/cmd/server/server.go index 48c032b09..fb867cb43 100644 --- a/pkg/cmd/server/server.go +++ b/pkg/cmd/server/server.go @@ -25,7 +25,6 @@ import ( "os" "reflect" "strings" - "sync" "time" "github.com/pkg/errors" @@ -34,6 +33,7 @@ import ( "github.com/spf13/cobra" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" kubeerrs "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" @@ -56,6 +56,7 @@ import ( "github.com/vmware-tanzu/velero/pkg/cmd" "github.com/vmware-tanzu/velero/pkg/cmd/util/flag" "github.com/vmware-tanzu/velero/pkg/cmd/util/signals" + "github.com/vmware-tanzu/velero/pkg/controller" velerodiscovery "github.com/vmware-tanzu/velero/pkg/discovery" "github.com/vmware-tanzu/velero/pkg/features" @@ -68,6 +69,14 @@ import ( "github.com/vmware-tanzu/velero/pkg/restic" "github.com/vmware-tanzu/velero/pkg/restore" "github.com/vmware-tanzu/velero/pkg/util/logging" + + ctrl "sigs.k8s.io/controller-runtime" + + kbclient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/manager" + + "github.com/vmware-tanzu/velero/internal/util/managercontroller" + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" ) const ( @@ -240,6 +249,7 @@ type server struct { resticManager restic.RepositoryManager metrics *metrics.ServerMetrics config serverConfig + mgr manager.Manager } func newServer(f client.Factory, config serverConfig, logger *logrus.Logger) (*server, error) { @@ -294,6 +304,16 @@ func newServer(f client.Factory, config serverConfig, logger *logrus.Logger) (*s } } + scheme := runtime.NewScheme() + velerov1api.AddToScheme(scheme) + mgr, err := ctrl.NewManager(clientConfig, ctrl.Options{ + Scheme: scheme, + }) + if err != nil { + cancelFunc() + return nil, err + } + s := &server{ namespace: f.Namespace(), metricsAddress: config.metricsAddress, @@ -311,6 +331,7 @@ func newServer(f client.Factory, config serverConfig, logger *logrus.Logger) (*s logLevel: logger.Level, pluginRegistry: pluginRegistry, config: config, + mgr: mgr, } return s, nil @@ -342,7 +363,13 @@ func (s *server) run() error { return err } - if _, err := s.veleroClient.VeleroV1().BackupStorageLocations(s.namespace).Get(s.config.defaultBackupLocation, metav1.GetOptions{}); err != nil { + // Fetching from the server directly since at this point + // the cache has not yet started + bsl := &velerov1api.BackupStorageLocation{} + if err := s.mgr.GetAPIReader().Get(context.Background(), kbclient.ObjectKey{ + Namespace: s.namespace, + Name: s.config.defaultBackupLocation, + }, bsl); err != nil { s.logger.WithError(errors.WithStack(err)). Warnf("A backup storage location named %s has been specified for the server to use by default, but no corresponding backup storage location exists. Backups with a location not matching the default will need to explicitly specify an existing location", s.config.defaultBackupLocation) } @@ -442,8 +469,12 @@ func (s *server) validateBackupStorageLocations() error { pluginManager := clientmgmt.NewManager(s.logger, s.logLevel, s.pluginRegistry) defer pluginManager.CleanupClients() - locations, err := s.veleroClient.VeleroV1().BackupStorageLocations(s.namespace).List(metav1.ListOptions{}) - if err != nil { + // Fetching from the server directly since at this point + // the cache has not yet started + locations := &velerov1api.BackupStorageLocationList{} + if err := s.mgr.GetAPIReader().List(context.Background(), locations, &kbclient.ListOptions{ + Namespace: s.namespace, + }); err != nil { return errors.WithStack(err) } @@ -543,7 +574,7 @@ func (s *server) initRestic() error { secretsInformer, s.sharedInformerFactory.Velero().V1().ResticRepositories(), s.veleroClient.VeleroV1(), - s.sharedInformerFactory.Velero().V1().BackupStorageLocations(), + s.mgr.GetClient(), s.kubeClient.CoreV1(), s.kubeClient.CoreV1(), s.logger, @@ -588,7 +619,6 @@ func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string s.logger.Info("Starting controllers") ctx := s.ctx - var wg sync.WaitGroup go func() { metricsMux := http.NewServeMux() @@ -611,10 +641,9 @@ func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string backupSyncControllerRunInfo := func() controllerRunInfo { backupSyncContoller := controller.NewBackupSyncController( s.veleroClient.VeleroV1(), - s.veleroClient.VeleroV1(), + s.mgr.GetClient(), s.veleroClient.VeleroV1(), s.sharedInformerFactory.Velero().V1().Backups().Lister(), - s.sharedInformerFactory.Velero().V1().BackupStorageLocations().Lister(), s.config.backupSyncPeriod, s.namespace, s.csiSnapshotClient, @@ -653,7 +682,7 @@ func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string s.logLevel, newPluginManager, backupTracker, - s.sharedInformerFactory.Velero().V1().BackupStorageLocations().Lister(), + s.mgr.GetClient(), s.config.defaultBackupLocation, s.config.defaultVolumesToRestic, s.config.defaultBackupTTL, @@ -693,7 +722,7 @@ func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string s.sharedInformerFactory.Velero().V1().Backups(), s.sharedInformerFactory.Velero().V1().DeleteBackupRequests().Lister(), s.veleroClient.VeleroV1(), - s.sharedInformerFactory.Velero().V1().BackupStorageLocations().Lister(), + s.mgr.GetClient(), ) return controllerRunInfo{ @@ -713,7 +742,7 @@ func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string backupTracker, s.resticManager, s.sharedInformerFactory.Velero().V1().PodVolumeBackups().Lister(), - s.sharedInformerFactory.Velero().V1().BackupStorageLocations().Lister(), + s.mgr.GetClient(), s.sharedInformerFactory.Velero().V1().VolumeSnapshotLocations().Lister(), csiVSLister, csiVSCLister, @@ -748,7 +777,7 @@ func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string s.veleroClient.VeleroV1(), restorer, s.sharedInformerFactory.Velero().V1().Backups().Lister(), - s.sharedInformerFactory.Velero().V1().BackupStorageLocations().Lister(), + s.mgr.GetClient(), s.sharedInformerFactory.Velero().V1().VolumeSnapshotLocations().Lister(), s.logger, s.logLevel, @@ -769,7 +798,7 @@ func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string s.logger, s.sharedInformerFactory.Velero().V1().ResticRepositories(), s.veleroClient.VeleroV1(), - s.sharedInformerFactory.Velero().V1().BackupStorageLocations().Lister(), + s.mgr.GetClient(), s.resticManager, s.config.defaultResticMaintenanceFrequency, ) @@ -785,7 +814,7 @@ func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string s.veleroClient.VeleroV1(), s.sharedInformerFactory.Velero().V1().DownloadRequests(), s.sharedInformerFactory.Velero().V1().Restores().Lister(), - s.sharedInformerFactory.Velero().V1().BackupStorageLocations().Lister(), + s.mgr.GetClient(), s.sharedInformerFactory.Velero().V1().Backups().Lister(), newPluginManager, s.logger, @@ -872,23 +901,22 @@ func (s *server) runControllers(defaultVolumeSnapshotLocations map[string]string s.logger.WithField("informer", informer).Info("Informer cache synced") } - // now that the informer caches have all synced, we can start running the controllers + // TODO(2.0): presuming all controllers and resources are converted to runtime-controller + // by v2.0, the block from this line and including the `s.mgr.Start() will be + // deprecated, since the manager auto-starts all the caches. Until then, we need to start the + // cache for them manually. for i := range controllers { controllerRunInfo := controllers[i] - - wg.Add(1) - go func() { - controllerRunInfo.controller.Run(ctx, controllerRunInfo.numWorkers) - wg.Done() - }() + // Adding the controllers to the manager will register them as a (runtime-controller) runnable, + // so the manager will ensure the cache is started and ready before all controller are started + s.mgr.Add(managercontroller.Runnable(controllerRunInfo.controller, controllerRunInfo.numWorkers)) } - s.logger.Info("Server started successfully") + s.logger.Info("Server starting...") - <-ctx.Done() - - s.logger.Info("Waiting for all controllers to shut down gracefully") - wg.Wait() + if err := s.mgr.Start(s.ctx.Done()); err != nil { + s.logger.Fatal("Problem starting manager", err) + } return nil } diff --git a/pkg/cmd/util/output/backup_storage_location_printer.go b/pkg/cmd/util/output/backup_storage_location_printer.go index 851bc7b73..8eed52f05 100644 --- a/pkg/cmd/util/output/backup_storage_location_printer.go +++ b/pkg/cmd/util/output/backup_storage_location_printer.go @@ -30,7 +30,7 @@ var ( {Name: "Name", Type: "string", Format: "name"}, {Name: "Provider"}, {Name: "Bucket/Prefix"}, - {Name: "Status"}, + {Name: "Phase"}, {Name: "Access Mode"}, } ) diff --git a/pkg/controller/backup_controller.go b/pkg/controller/backup_controller.go index 85380144b..7038cfad7 100644 --- a/pkg/controller/backup_controller.go +++ b/pkg/controller/backup_controller.go @@ -19,6 +19,7 @@ package controller import ( "bytes" "compress/gzip" + "context" "encoding/json" "fmt" "io" @@ -57,6 +58,8 @@ import ( kubeutil "github.com/vmware-tanzu/velero/pkg/util/kube" "github.com/vmware-tanzu/velero/pkg/util/logging" "github.com/vmware-tanzu/velero/pkg/volume" + + kbclient "sigs.k8s.io/controller-runtime/pkg/client" ) type backupController struct { @@ -65,11 +68,11 @@ type backupController struct { backupper pkgbackup.Backupper lister velerov1listers.BackupLister client velerov1client.BackupsGetter + kbClient kbclient.Client clock clock.Clock backupLogLevel logrus.Level newPluginManager func(logrus.FieldLogger) clientmgmt.Manager backupTracker BackupTracker - backupLocationLister velerov1listers.BackupStorageLocationLister defaultBackupLocation string defaultVolumesToRestic bool defaultBackupTTL time.Duration @@ -91,7 +94,7 @@ func NewBackupController( backupLogLevel logrus.Level, newPluginManager func(logrus.FieldLogger) clientmgmt.Manager, backupTracker BackupTracker, - backupLocationLister velerov1listers.BackupStorageLocationLister, + kbClient kbclient.Client, defaultBackupLocation string, defaultVolumesToRestic bool, defaultBackupTTL time.Duration, @@ -112,7 +115,7 @@ func NewBackupController( backupLogLevel: backupLogLevel, newPluginManager: newPluginManager, backupTracker: backupTracker, - backupLocationLister: backupLocationLister, + kbClient: kbClient, defaultBackupLocation: defaultBackupLocation, defaultVolumesToRestic: defaultVolumesToRestic, defaultBackupTTL: defaultBackupTTL, @@ -371,7 +374,11 @@ func (c *backupController) prepareBackupRequest(backup *velerov1api.Backup) *pkg } // validate the storage location, and store the BackupStorageLocation API obj on the request - if storageLocation, err := c.backupLocationLister.BackupStorageLocations(request.Namespace).Get(request.Spec.StorageLocation); err != nil { + storageLocation := &velerov1api.BackupStorageLocation{} + if err := c.kbClient.Get(context.Background(), kbclient.ObjectKey{ + Namespace: request.Namespace, + Name: request.Spec.StorageLocation, + }, storageLocation); err != nil { if apierrors.IsNotFound(err) { request.Status.ValidationErrors = append(request.Status.ValidationErrors, fmt.Sprintf("a BackupStorageLocation CRD with the name specified in the backup spec needs to be created before this backup can be executed. Error: %v", err)) } else { @@ -385,7 +392,6 @@ func (c *backupController) prepareBackupRequest(backup *velerov1api.Backup) *pkg fmt.Sprintf("backup can't be created because backup storage location %s is currently in read-only mode", request.StorageLocation.Name)) } } - // validate and get the backup's VolumeSnapshotLocations, and store the // VolumeSnapshotLocation API objs on the request if locs, errs := c.validateAndGetSnapshotLocations(request.Backup); len(errs) > 0 { diff --git a/pkg/controller/backup_controller_test.go b/pkg/controller/backup_controller_test.go index b746cb734..354a568cc 100644 --- a/pkg/controller/backup_controller_test.go +++ b/pkg/controller/backup_controller_test.go @@ -18,8 +18,10 @@ package controller import ( "bytes" + "context" "fmt" "io" + "sort" "strings" "testing" @@ -32,8 +34,8 @@ import ( "github.com/stretchr/testify/require" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/clock" - "k8s.io/apimachinery/pkg/version" + kbclient "sigs.k8s.io/controller-runtime/pkg/client" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" pkgbackup "github.com/vmware-tanzu/velero/pkg/backup" @@ -154,7 +156,7 @@ func TestProcessBackupValidationFailures(t *testing.T) { { name: "non-existent backup location fails validation", backup: defaultBackup().StorageLocation("nonexistent").Result(), - expectedErrs: []string{"a BackupStorageLocation CRD with the name specified in the backup spec needs to be created before this backup can be executed. Error: backupstoragelocation.velero.io \"nonexistent\" not found"}, + expectedErrs: []string{"a BackupStorageLocation CRD with the name specified in the backup spec needs to be created before this backup can be executed. Error: backupstoragelocations.velero.io \"nonexistent\" not found"}, }, { name: "backup for read-only backup location fails validation", @@ -177,12 +179,19 @@ func TestProcessBackupValidationFailures(t *testing.T) { discoveryHelper, err := discovery.NewHelper(apiServer.DiscoveryClient, logger) require.NoError(t, err) + var fakeClient kbclient.Client + if test.backupLocation != nil { + fakeClient = newFakeClient(t, test.backupLocation) + } else { + fakeClient = newFakeClient(t) + } + c := &backupController{ genericController: newGenericController("backup-test", logger), discoveryHelper: discoveryHelper, client: clientset.VeleroV1(), lister: sharedInformers.Velero().V1().Backups().Lister(), - backupLocationLister: sharedInformers.Velero().V1().BackupStorageLocations().Lister(), + kbClient: fakeClient, snapshotLocationLister: sharedInformers.Velero().V1().VolumeSnapshotLocations().Lister(), defaultBackupLocation: defaultBackupLocation.Name, clock: &clock.RealClock{}, @@ -192,13 +201,6 @@ func TestProcessBackupValidationFailures(t *testing.T) { require.NotNil(t, test.backup) require.NoError(t, sharedInformers.Velero().V1().Backups().Informer().GetStore().Add(test.backup)) - if test.backupLocation != nil { - _, err := clientset.VeleroV1().BackupStorageLocations(test.backupLocation.Namespace).Create(test.backupLocation) - require.NoError(t, err) - - require.NoError(t, sharedInformers.Velero().V1().BackupStorageLocations().Informer().GetStore().Add(test.backupLocation)) - } - require.NoError(t, c.processBackup(fmt.Sprintf("%s/%s", test.backup.Namespace, test.backup.Name))) res, err := clientset.VeleroV1().Backups(test.backup.Namespace).Get(test.backup.Name, metav1.GetOptions{}) @@ -244,6 +246,7 @@ func TestBackupLocationLabel(t *testing.T) { clientset = fake.NewSimpleClientset(test.backup) sharedInformers = informers.NewSharedInformerFactory(clientset, 0) logger = logging.DefaultLogger(logrus.DebugLevel, formatFlag) + fakeClient = newFakeClient(t) ) apiServer := velerotest.NewAPIServer(t) @@ -255,7 +258,7 @@ func TestBackupLocationLabel(t *testing.T) { discoveryHelper: discoveryHelper, client: clientset.VeleroV1(), lister: sharedInformers.Velero().V1().Backups().Lister(), - backupLocationLister: sharedInformers.Velero().V1().BackupStorageLocations().Lister(), + kbClient: fakeClient, snapshotLocationLister: sharedInformers.Velero().V1().VolumeSnapshotLocations().Lister(), defaultBackupLocation: test.backupLocation.Name, clock: &clock.RealClock{}, @@ -303,6 +306,7 @@ func TestDefaultBackupTTL(t *testing.T) { formatFlag := logging.FormatText var ( clientset = fake.NewSimpleClientset(test.backup) + fakeClient = newFakeClient(t) logger = logging.DefaultLogger(logrus.DebugLevel, formatFlag) sharedInformers = informers.NewSharedInformerFactory(clientset, 0) ) @@ -316,7 +320,7 @@ func TestDefaultBackupTTL(t *testing.T) { c := &backupController{ genericController: newGenericController("backup-test", logger), discoveryHelper: discoveryHelper, - backupLocationLister: sharedInformers.Velero().V1().BackupStorageLocations().Lister(), + kbClient: fakeClient, snapshotLocationLister: sharedInformers.Velero().V1().VolumeSnapshotLocations().Lister(), defaultBackupTTL: defaultBackupTTL.Duration, clock: clock.NewFakeClock(now), @@ -776,6 +780,14 @@ func TestProcessBackupCompletions(t *testing.T) { backupper = new(fakeBackupper) ) + var fakeClient kbclient.Client + // add the test's backup storage location if it's different than the default + if test.backupLocation != nil && test.backupLocation != defaultBackupLocation { + fakeClient = newFakeClient(t, test.backupLocation) + } else { + fakeClient = newFakeClient(t) + } + apiServer := velerotest.NewAPIServer(t) apiServer.DiscoveryClient.FakedServerVersion = &version.Info{ @@ -798,7 +810,7 @@ func TestProcessBackupCompletions(t *testing.T) { discoveryHelper: discoveryHelper, client: clientset.VeleroV1(), lister: sharedInformers.Velero().V1().Backups().Lister(), - backupLocationLister: sharedInformers.Velero().V1().BackupStorageLocations().Lister(), + kbClient: fakeClient, snapshotLocationLister: sharedInformers.Velero().V1().VolumeSnapshotLocations().Lister(), defaultBackupLocation: defaultBackupLocation.Name, defaultVolumesToRestic: test.defaultVolumesToRestic, @@ -833,19 +845,7 @@ func TestProcessBackupCompletions(t *testing.T) { require.NoError(t, sharedInformers.Velero().V1().Backups().Informer().GetStore().Add(test.backup)) // add the default backup storage location to the clientset and the informer/lister store - _, err = clientset.VeleroV1().BackupStorageLocations(defaultBackupLocation.Namespace).Create(defaultBackupLocation) - require.NoError(t, err) - - require.NoError(t, sharedInformers.Velero().V1().BackupStorageLocations().Informer().GetStore().Add(defaultBackupLocation)) - - // add the test's backup storage location to the clientset and the informer/lister store - // if it's different than the default - if test.backupLocation != nil && test.backupLocation != defaultBackupLocation { - _, err := clientset.VeleroV1().BackupStorageLocations(test.backupLocation.Namespace).Create(test.backupLocation) - require.NoError(t, err) - - require.NoError(t, sharedInformers.Velero().V1().BackupStorageLocations().Informer().GetStore().Add(test.backupLocation)) - } + require.NoError(t, fakeClient.Create(context.Background(), defaultBackupLocation)) require.NoError(t, c.processBackup(fmt.Sprintf("%s/%s", test.backup.Namespace, test.backup.Name))) diff --git a/pkg/controller/backup_deletion_controller.go b/pkg/controller/backup_deletion_controller.go index 1af193e15..bee84abb0 100644 --- a/pkg/controller/backup_deletion_controller.go +++ b/pkg/controller/backup_deletion_controller.go @@ -49,6 +49,8 @@ import ( "github.com/vmware-tanzu/velero/pkg/plugin/velero" "github.com/vmware-tanzu/velero/pkg/restic" "github.com/vmware-tanzu/velero/pkg/util/kube" + + "sigs.k8s.io/controller-runtime/pkg/client" ) const resticTimeout = time.Minute @@ -64,7 +66,7 @@ type backupDeletionController struct { backupTracker BackupTracker resticMgr restic.RepositoryManager podvolumeBackupLister velerov1listers.PodVolumeBackupLister - backupLocationLister velerov1listers.BackupStorageLocationLister + kbClient client.Client snapshotLocationLister velerov1listers.VolumeSnapshotLocationLister csiSnapshotLister snapshotv1beta1listers.VolumeSnapshotLister csiSnapshotContentLister snapshotv1beta1listers.VolumeSnapshotContentLister @@ -87,7 +89,7 @@ func NewBackupDeletionController( backupTracker BackupTracker, resticMgr restic.RepositoryManager, podvolumeBackupLister velerov1listers.PodVolumeBackupLister, - backupLocationLister velerov1listers.BackupStorageLocationLister, + kbClient client.Client, snapshotLocationLister velerov1listers.VolumeSnapshotLocationLister, csiSnapshotLister snapshotv1beta1listers.VolumeSnapshotLister, csiSnapshotContentLister snapshotv1beta1listers.VolumeSnapshotContentLister, @@ -105,7 +107,7 @@ func NewBackupDeletionController( backupTracker: backupTracker, resticMgr: resticMgr, podvolumeBackupLister: podvolumeBackupLister, - backupLocationLister: backupLocationLister, + kbClient: kbClient, snapshotLocationLister: snapshotLocationLister, csiSnapshotLister: csiSnapshotLister, csiSnapshotContentLister: csiSnapshotContentLister, @@ -214,15 +216,18 @@ func (c *backupDeletionController) processRequest(req *velerov1api.DeleteBackupR } // Don't allow deleting backups in read-only storage locations - location, err := c.backupLocationLister.BackupStorageLocations(backup.Namespace).Get(backup.Spec.StorageLocation) - if apierrors.IsNotFound(err) { - _, err := c.patchDeleteBackupRequest(req, func(r *velerov1api.DeleteBackupRequest) { - r.Status.Phase = velerov1api.DeleteBackupRequestPhaseProcessed - r.Status.Errors = append(r.Status.Errors, fmt.Sprintf("backup storage location %s not found", backup.Spec.StorageLocation)) - }) - return err - } - if err != nil { + location := &velerov1api.BackupStorageLocation{} + if err := c.kbClient.Get(context.Background(), client.ObjectKey{ + Namespace: backup.Namespace, + Name: backup.Spec.StorageLocation, + }, location); err != nil { + if apierrors.IsNotFound(err) { + _, err := c.patchDeleteBackupRequest(req, func(r *velerov1api.DeleteBackupRequest) { + r.Status.Phase = velerov1api.DeleteBackupRequestPhaseProcessed + r.Status.Errors = append(r.Status.Errors, fmt.Sprintf("backup storage location %s not found", backup.Spec.StorageLocation)) + }) + return err + } return errors.Wrap(err, "error getting backup storage location") } diff --git a/pkg/controller/backup_deletion_controller_test.go b/pkg/controller/backup_deletion_controller_test.go index c7a7dd018..7b12b325a 100644 --- a/pkg/controller/backup_deletion_controller_test.go +++ b/pkg/controller/backup_deletion_controller_test.go @@ -17,6 +17,7 @@ limitations under the License. package controller import ( + "context" "fmt" "testing" "time" @@ -34,8 +35,9 @@ import ( "k8s.io/apimachinery/pkg/util/clock" "k8s.io/apimachinery/pkg/util/sets" core "k8s.io/client-go/testing" + "sigs.k8s.io/controller-runtime/pkg/client" - velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" pkgbackup "github.com/vmware-tanzu/velero/pkg/backup" "github.com/vmware-tanzu/velero/pkg/builder" "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned/fake" @@ -63,7 +65,7 @@ func TestBackupDeletionControllerProcessQueueItem(t *testing.T) { NewBackupTracker(), nil, // restic repository manager sharedInformers.Velero().V1().PodVolumeBackups().Lister(), - sharedInformers.Velero().V1().BackupStorageLocations().Lister(), + nil, sharedInformers.Velero().V1().VolumeSnapshotLocations().Lister(), nil, // csiSnapshotLister nil, // csiSnapshotContentLister @@ -84,21 +86,21 @@ func TestBackupDeletionControllerProcessQueueItem(t *testing.T) { req := pkgbackup.NewDeleteBackupRequest("foo", "uid") req.Namespace = "foo" req.Name = "foo-abcde" - req.Status.Phase = velerov1.DeleteBackupRequestPhaseProcessed + req.Status.Phase = velerov1api.DeleteBackupRequestPhaseProcessed err = controller.processQueueItem("foo/bar") assert.NoError(t, err) // Invoke processRequestFunc - for _, phase := range []velerov1.DeleteBackupRequestPhase{"", velerov1.DeleteBackupRequestPhaseNew, velerov1.DeleteBackupRequestPhaseInProgress} { + for _, phase := range []velerov1api.DeleteBackupRequestPhase{"", velerov1api.DeleteBackupRequestPhaseNew, velerov1api.DeleteBackupRequestPhaseInProgress} { t.Run(fmt.Sprintf("phase=%s", phase), func(t *testing.T) { req.Status.Phase = phase sharedInformers.Velero().V1().DeleteBackupRequests().Informer().GetStore().Add(req) var errorToReturn error - var actual *velerov1.DeleteBackupRequest + var actual *velerov1api.DeleteBackupRequest var called bool - controller.processRequestFunc = func(r *velerov1.DeleteBackupRequest) error { + controller.processRequestFunc = func(r *velerov1api.DeleteBackupRequest) error { called = true actual = r return errorToReturn @@ -121,20 +123,22 @@ func TestBackupDeletionControllerProcessQueueItem(t *testing.T) { type backupDeletionControllerTestData struct { client *fake.Clientset + fakeClient client.Client sharedInformers informers.SharedInformerFactory volumeSnapshotter *velerotest.FakeVolumeSnapshotter backupStore *persistencemocks.BackupStore controller *backupDeletionController - req *velerov1.DeleteBackupRequest + req *velerov1api.DeleteBackupRequest } -func setupBackupDeletionControllerTest(objects ...runtime.Object) *backupDeletionControllerTestData { +func setupBackupDeletionControllerTest(t *testing.T, objects ...runtime.Object) *backupDeletionControllerTestData { req := pkgbackup.NewDeleteBackupRequest("foo", "uid") req.Namespace = "velero" req.Name = "foo-abcde" var ( client = fake.NewSimpleClientset(append(objects, req)...) + fakeClient = newFakeClient(t, objects...) sharedInformers = informers.NewSharedInformerFactory(client, 0) volumeSnapshotter = &velerotest.FakeVolumeSnapshotter{SnapshotsTaken: sets.NewString()} pluginManager = &pluginmocks.Manager{} @@ -143,6 +147,7 @@ func setupBackupDeletionControllerTest(objects ...runtime.Object) *backupDeletio data := &backupDeletionControllerTestData{ client: client, + fakeClient: fakeClient, sharedInformers: sharedInformers, volumeSnapshotter: volumeSnapshotter, backupStore: backupStore, @@ -156,7 +161,7 @@ func setupBackupDeletionControllerTest(objects ...runtime.Object) *backupDeletio NewBackupTracker(), nil, // restic repository manager sharedInformers.Velero().V1().PodVolumeBackups().Lister(), - sharedInformers.Velero().V1().BackupStorageLocations().Lister(), + fakeClient, sharedInformers.Velero().V1().VolumeSnapshotLocations().Lister(), nil, // csiSnapshotLister nil, // csiSnapshotContentLister @@ -168,7 +173,7 @@ func setupBackupDeletionControllerTest(objects ...runtime.Object) *backupDeletio req: req, } - data.controller.newBackupStore = func(*velerov1.BackupStorageLocation, persistence.ObjectStoreGetter, logrus.FieldLogger) (persistence.BackupStore, error) { + data.controller.newBackupStore = func(*velerov1api.BackupStorageLocation, persistence.ObjectStoreGetter, logrus.FieldLogger) (persistence.BackupStore, error) { return backupStore, nil } @@ -179,8 +184,7 @@ func setupBackupDeletionControllerTest(objects ...runtime.Object) *backupDeletio func TestBackupDeletionControllerProcessRequest(t *testing.T) { t.Run("missing spec.backupName", func(t *testing.T) { - td := setupBackupDeletionControllerTest() - + td := setupBackupDeletionControllerTest(t) td.req.Spec.BackupName = "" err := td.controller.processRequest(td.req) @@ -188,7 +192,7 @@ func TestBackupDeletionControllerProcessRequest(t *testing.T) { expectedActions := []core.Action{ core.NewPatchAction( - velerov1.SchemeGroupVersion.WithResource("deletebackuprequests"), + velerov1api.SchemeGroupVersion.WithResource("deletebackuprequests"), td.req.Namespace, td.req.Name, types.MergePatchType, @@ -200,7 +204,7 @@ func TestBackupDeletionControllerProcessRequest(t *testing.T) { }) t.Run("existing deletion requests for the backup are deleted", func(t *testing.T) { - td := setupBackupDeletionControllerTest() + td := setupBackupDeletionControllerTest(t) // add the backup to the tracker so the execution of processRequest doesn't progress // past checking for an in-progress backup. this makes validation easier. @@ -208,15 +212,15 @@ func TestBackupDeletionControllerProcessRequest(t *testing.T) { require.NoError(t, td.sharedInformers.Velero().V1().DeleteBackupRequests().Informer().GetStore().Add(td.req)) - existing := &velerov1.DeleteBackupRequest{ + existing := &velerov1api.DeleteBackupRequest{ ObjectMeta: metav1.ObjectMeta{ Namespace: td.req.Namespace, Name: "bar", Labels: map[string]string{ - velerov1.BackupNameLabel: td.req.Spec.BackupName, + velerov1api.BackupNameLabel: td.req.Spec.BackupName, }, }, - Spec: velerov1.DeleteBackupRequestSpec{ + Spec: velerov1api.DeleteBackupRequestSpec{ BackupName: td.req.Spec.BackupName, }, } @@ -225,15 +229,15 @@ func TestBackupDeletionControllerProcessRequest(t *testing.T) { require.NoError(t, err) require.NoError(t, td.sharedInformers.Velero().V1().DeleteBackupRequests().Informer().GetStore().Add( - &velerov1.DeleteBackupRequest{ + &velerov1api.DeleteBackupRequest{ ObjectMeta: metav1.ObjectMeta{ Namespace: td.req.Namespace, Name: "bar-2", Labels: map[string]string{ - velerov1.BackupNameLabel: "some-other-backup", + velerov1api.BackupNameLabel: "some-other-backup", }, }, - Spec: velerov1.DeleteBackupRequestSpec{ + Spec: velerov1api.DeleteBackupRequestSpec{ BackupName: "some-other-backup", }, }, @@ -242,7 +246,7 @@ func TestBackupDeletionControllerProcessRequest(t *testing.T) { assert.NoError(t, td.controller.processRequest(td.req)) expectedDeleteAction := core.NewDeleteAction( - velerov1.SchemeGroupVersion.WithResource("deletebackuprequests"), + velerov1api.SchemeGroupVersion.WithResource("deletebackuprequests"), td.req.Namespace, "bar", ) @@ -255,7 +259,7 @@ func TestBackupDeletionControllerProcessRequest(t *testing.T) { }) t.Run("deleting an in progress backup isn't allowed", func(t *testing.T) { - td := setupBackupDeletionControllerTest() + td := setupBackupDeletionControllerTest(t) td.controller.backupTracker.Add(td.req.Namespace, td.req.Spec.BackupName) @@ -264,7 +268,7 @@ func TestBackupDeletionControllerProcessRequest(t *testing.T) { expectedActions := []core.Action{ core.NewPatchAction( - velerov1.SchemeGroupVersion.WithResource("deletebackuprequests"), + velerov1api.SchemeGroupVersion.WithResource("deletebackuprequests"), td.req.Namespace, td.req.Name, types.MergePatchType, @@ -276,12 +280,10 @@ func TestBackupDeletionControllerProcessRequest(t *testing.T) { }) t.Run("patching to InProgress fails", func(t *testing.T) { - backup := builder.ForBackup(velerov1.DefaultNamespace, "foo").StorageLocation("default").Result() + backup := builder.ForBackup(velerov1api.DefaultNamespace, "foo").StorageLocation("default").Result() location := builder.ForBackupStorageLocation("velero", "default").Result() - td := setupBackupDeletionControllerTest(backup) - - td.sharedInformers.Velero().V1().BackupStorageLocations().Informer().GetStore().Add(location) + td := setupBackupDeletionControllerTest(t, location, backup) td.client.PrependReactor("patch", "deletebackuprequests", func(action core.Action) (bool, runtime.Object, error) { return true, nil, errors.New("bad") @@ -292,12 +294,12 @@ func TestBackupDeletionControllerProcessRequest(t *testing.T) { expectedActions := []core.Action{ core.NewGetAction( - velerov1.SchemeGroupVersion.WithResource("backups"), + velerov1api.SchemeGroupVersion.WithResource("backups"), backup.Namespace, backup.Name, ), core.NewPatchAction( - velerov1.SchemeGroupVersion.WithResource("deletebackuprequests"), + velerov1api.SchemeGroupVersion.WithResource("deletebackuprequests"), td.req.Namespace, td.req.Name, types.MergePatchType, @@ -308,12 +310,10 @@ func TestBackupDeletionControllerProcessRequest(t *testing.T) { }) t.Run("patching backup to Deleting fails", func(t *testing.T) { - backup := builder.ForBackup(velerov1.DefaultNamespace, "foo").StorageLocation("default").Result() + backup := builder.ForBackup(velerov1api.DefaultNamespace, "foo").StorageLocation("default").Result() location := builder.ForBackupStorageLocation("velero", "default").Result() - td := setupBackupDeletionControllerTest(backup) - - td.sharedInformers.Velero().V1().BackupStorageLocations().Informer().GetStore().Add(location) + td := setupBackupDeletionControllerTest(t, location, backup) td.client.PrependReactor("patch", "deletebackuprequests", func(action core.Action) (bool, runtime.Object, error) { return true, td.req, nil @@ -327,19 +327,19 @@ func TestBackupDeletionControllerProcessRequest(t *testing.T) { expectedActions := []core.Action{ core.NewGetAction( - velerov1.SchemeGroupVersion.WithResource("backups"), + velerov1api.SchemeGroupVersion.WithResource("backups"), backup.Namespace, backup.Name, ), core.NewPatchAction( - velerov1.SchemeGroupVersion.WithResource("deletebackuprequests"), + velerov1api.SchemeGroupVersion.WithResource("deletebackuprequests"), td.req.Namespace, td.req.Name, types.MergePatchType, []byte(`{"status":{"phase":"InProgress"}}`), ), core.NewPatchAction( - velerov1.SchemeGroupVersion.WithResource("backups"), + velerov1api.SchemeGroupVersion.WithResource("backups"), backup.Namespace, backup.Name, types.MergePatchType, @@ -350,19 +350,19 @@ func TestBackupDeletionControllerProcessRequest(t *testing.T) { }) t.Run("unable to find backup", func(t *testing.T) { - td := setupBackupDeletionControllerTest() + td := setupBackupDeletionControllerTest(t) err := td.controller.processRequest(td.req) require.NoError(t, err) expectedActions := []core.Action{ core.NewGetAction( - velerov1.SchemeGroupVersion.WithResource("backups"), + velerov1api.SchemeGroupVersion.WithResource("backups"), td.req.Namespace, td.req.Spec.BackupName, ), core.NewPatchAction( - velerov1.SchemeGroupVersion.WithResource("deletebackuprequests"), + velerov1api.SchemeGroupVersion.WithResource("deletebackuprequests"), td.req.Namespace, td.req.Name, types.MergePatchType, @@ -374,21 +374,21 @@ func TestBackupDeletionControllerProcessRequest(t *testing.T) { }) t.Run("unable to find backup storage location", func(t *testing.T) { - backup := builder.ForBackup(velerov1.DefaultNamespace, "foo").StorageLocation("default").Result() + backup := builder.ForBackup(velerov1api.DefaultNamespace, "foo").StorageLocation("default").Result() - td := setupBackupDeletionControllerTest(backup) + td := setupBackupDeletionControllerTest(t, backup) err := td.controller.processRequest(td.req) require.NoError(t, err) expectedActions := []core.Action{ core.NewGetAction( - velerov1.SchemeGroupVersion.WithResource("backups"), + velerov1api.SchemeGroupVersion.WithResource("backups"), td.req.Namespace, td.req.Spec.BackupName, ), core.NewPatchAction( - velerov1.SchemeGroupVersion.WithResource("deletebackuprequests"), + velerov1api.SchemeGroupVersion.WithResource("deletebackuprequests"), td.req.Namespace, td.req.Name, types.MergePatchType, @@ -400,24 +400,22 @@ func TestBackupDeletionControllerProcessRequest(t *testing.T) { }) t.Run("backup storage location is in read-only mode", func(t *testing.T) { - backup := builder.ForBackup(velerov1.DefaultNamespace, "foo").StorageLocation("default").Result() - location := builder.ForBackupStorageLocation("velero", "default").AccessMode(velerov1.BackupStorageLocationAccessModeReadOnly).Result() + backup := builder.ForBackup(velerov1api.DefaultNamespace, "foo").StorageLocation("default").Result() + location := builder.ForBackupStorageLocation("velero", "default").AccessMode(velerov1api.BackupStorageLocationAccessModeReadOnly).Result() - td := setupBackupDeletionControllerTest(backup) - - td.sharedInformers.Velero().V1().BackupStorageLocations().Informer().GetStore().Add(location) + td := setupBackupDeletionControllerTest(t, location, backup) err := td.controller.processRequest(td.req) require.NoError(t, err) expectedActions := []core.Action{ core.NewGetAction( - velerov1.SchemeGroupVersion.WithResource("backups"), + velerov1api.SchemeGroupVersion.WithResource("backups"), td.req.Namespace, td.req.Spec.BackupName, ), core.NewPatchAction( - velerov1.SchemeGroupVersion.WithResource("deletebackuprequests"), + velerov1api.SchemeGroupVersion.WithResource("deletebackuprequests"), td.req.Namespace, td.req.Name, types.MergePatchType, @@ -429,42 +427,43 @@ func TestBackupDeletionControllerProcessRequest(t *testing.T) { }) t.Run("full delete, no errors", func(t *testing.T) { - backup := builder.ForBackup(velerov1.DefaultNamespace, "foo").Result() + backup := builder.ForBackup(velerov1api.DefaultNamespace, "foo").Result() backup.UID = "uid" backup.Spec.StorageLocation = "primary" - restore1 := builder.ForRestore("velero", "restore-1").Phase(velerov1.RestorePhaseCompleted).Backup("foo").Result() - restore2 := builder.ForRestore("velero", "restore-2").Phase(velerov1.RestorePhaseCompleted).Backup("foo").Result() - restore3 := builder.ForRestore("velero", "restore-3").Phase(velerov1.RestorePhaseCompleted).Backup("some-other-backup").Result() + restore1 := builder.ForRestore("velero", "restore-1").Phase(velerov1api.RestorePhaseCompleted).Backup("foo").Result() + restore2 := builder.ForRestore("velero", "restore-2").Phase(velerov1api.RestorePhaseCompleted).Backup("foo").Result() + restore3 := builder.ForRestore("velero", "restore-3").Phase(velerov1api.RestorePhaseCompleted).Backup("some-other-backup").Result() - td := setupBackupDeletionControllerTest(backup, restore1, restore2, restore3) + td := setupBackupDeletionControllerTest(t, backup, restore1, restore2, restore3) td.sharedInformers.Velero().V1().Restores().Informer().GetStore().Add(restore1) td.sharedInformers.Velero().V1().Restores().Informer().GetStore().Add(restore2) td.sharedInformers.Velero().V1().Restores().Informer().GetStore().Add(restore3) - location := &velerov1.BackupStorageLocation{ + location := &velerov1api.BackupStorageLocation{ ObjectMeta: metav1.ObjectMeta{ Namespace: backup.Namespace, Name: backup.Spec.StorageLocation, }, - Spec: velerov1.BackupStorageLocationSpec{ + Spec: velerov1api.BackupStorageLocationSpec{ Provider: "objStoreProvider", - StorageType: velerov1.StorageType{ - ObjectStorage: &velerov1.ObjectStorageLocation{ + StorageType: velerov1api.StorageType{ + ObjectStorage: &velerov1api.ObjectStorageLocation{ Bucket: "bucket", }, }, }, } - require.NoError(t, td.sharedInformers.Velero().V1().BackupStorageLocations().Informer().GetStore().Add(location)) - snapshotLocation := &velerov1.VolumeSnapshotLocation{ + require.NoError(t, td.fakeClient.Create(context.Background(), location)) + + snapshotLocation := &velerov1api.VolumeSnapshotLocation{ ObjectMeta: metav1.ObjectMeta{ Namespace: backup.Namespace, Name: "vsl-1", }, - Spec: velerov1.VolumeSnapshotLocationSpec{ + Spec: velerov1api.VolumeSnapshotLocationSpec{ Provider: "provider-1", }, } @@ -514,55 +513,55 @@ func TestBackupDeletionControllerProcessRequest(t *testing.T) { expectedActions := []core.Action{ core.NewPatchAction( - velerov1.SchemeGroupVersion.WithResource("deletebackuprequests"), + velerov1api.SchemeGroupVersion.WithResource("deletebackuprequests"), td.req.Namespace, td.req.Name, types.MergePatchType, []byte(`{"metadata":{"labels":{"velero.io/backup-name":"foo"}},"status":{"phase":"InProgress"}}`), ), core.NewGetAction( - velerov1.SchemeGroupVersion.WithResource("backups"), + velerov1api.SchemeGroupVersion.WithResource("backups"), td.req.Namespace, td.req.Spec.BackupName, ), core.NewPatchAction( - velerov1.SchemeGroupVersion.WithResource("deletebackuprequests"), + velerov1api.SchemeGroupVersion.WithResource("deletebackuprequests"), td.req.Namespace, td.req.Name, types.MergePatchType, []byte(`{"metadata":{"labels":{"velero.io/backup-uid":"uid"}}}`), ), core.NewPatchAction( - velerov1.SchemeGroupVersion.WithResource("backups"), + velerov1api.SchemeGroupVersion.WithResource("backups"), td.req.Namespace, td.req.Spec.BackupName, types.MergePatchType, []byte(`{"status":{"phase":"Deleting"}}`), ), core.NewDeleteAction( - velerov1.SchemeGroupVersion.WithResource("restores"), + velerov1api.SchemeGroupVersion.WithResource("restores"), td.req.Namespace, "restore-1", ), core.NewDeleteAction( - velerov1.SchemeGroupVersion.WithResource("restores"), + velerov1api.SchemeGroupVersion.WithResource("restores"), td.req.Namespace, "restore-2", ), core.NewDeleteAction( - velerov1.SchemeGroupVersion.WithResource("backups"), + velerov1api.SchemeGroupVersion.WithResource("backups"), td.req.Namespace, td.req.Spec.BackupName, ), core.NewPatchAction( - velerov1.SchemeGroupVersion.WithResource("deletebackuprequests"), + velerov1api.SchemeGroupVersion.WithResource("deletebackuprequests"), td.req.Namespace, td.req.Name, types.MergePatchType, []byte(`{"status":{"phase":"Processed"}}`), ), core.NewDeleteCollectionAction( - velerov1.SchemeGroupVersion.WithResource("deletebackuprequests"), + velerov1api.SchemeGroupVersion.WithResource("deletebackuprequests"), td.req.Namespace, pkgbackup.NewDeleteBackupRequestListOptions(td.req.Spec.BackupName, "uid"), ), @@ -584,19 +583,19 @@ func TestBackupDeletionControllerProcessRequest(t *testing.T) { backup.Spec.StorageLocation = "primary" restore1 := builder.ForRestore("velero", "restore-1"). - Phase(velerov1.RestorePhaseCompleted). + Phase(velerov1api.RestorePhaseCompleted). Backup("the-really-long-backup-name-that-is-much-more-than-63-characters"). Result() restore2 := builder.ForRestore("velero", "restore-2"). - Phase(velerov1.RestorePhaseCompleted). + Phase(velerov1api.RestorePhaseCompleted). Backup("the-really-long-backup-name-that-is-much-more-than-63-characters"). Result() restore3 := builder.ForRestore("velero", "restore-3"). - Phase(velerov1.RestorePhaseCompleted). + Phase(velerov1api.RestorePhaseCompleted). Backup("some-other-backup"). Result() - td := setupBackupDeletionControllerTest(backup, restore1, restore2, restore3) + td := setupBackupDeletionControllerTest(t, backup, restore1, restore2, restore3) td.req = pkgbackup.NewDeleteBackupRequest(backup.Name, string(backup.UID)) td.req.Namespace = "velero" td.req.Name = "foo-abcde" @@ -604,28 +603,28 @@ func TestBackupDeletionControllerProcessRequest(t *testing.T) { td.sharedInformers.Velero().V1().Restores().Informer().GetStore().Add(restore2) td.sharedInformers.Velero().V1().Restores().Informer().GetStore().Add(restore3) - location := &velerov1.BackupStorageLocation{ + location := &velerov1api.BackupStorageLocation{ ObjectMeta: metav1.ObjectMeta{ Namespace: backup.Namespace, Name: backup.Spec.StorageLocation, }, - Spec: velerov1.BackupStorageLocationSpec{ + Spec: velerov1api.BackupStorageLocationSpec{ Provider: "objStoreProvider", - StorageType: velerov1.StorageType{ - ObjectStorage: &velerov1.ObjectStorageLocation{ + StorageType: velerov1api.StorageType{ + ObjectStorage: &velerov1api.ObjectStorageLocation{ Bucket: "bucket", }, }, }, } - require.NoError(t, td.sharedInformers.Velero().V1().BackupStorageLocations().Informer().GetStore().Add(location)) + require.NoError(t, td.fakeClient.Create(context.Background(), location)) - snapshotLocation := &velerov1.VolumeSnapshotLocation{ + snapshotLocation := &velerov1api.VolumeSnapshotLocation{ ObjectMeta: metav1.ObjectMeta{ Namespace: backup.Namespace, Name: "vsl-1", }, - Spec: velerov1.VolumeSnapshotLocationSpec{ + Spec: velerov1api.VolumeSnapshotLocationSpec{ Provider: "provider-1", }, } @@ -673,55 +672,55 @@ func TestBackupDeletionControllerProcessRequest(t *testing.T) { expectedActions := []core.Action{ core.NewPatchAction( - velerov1.SchemeGroupVersion.WithResource("deletebackuprequests"), + velerov1api.SchemeGroupVersion.WithResource("deletebackuprequests"), td.req.Namespace, td.req.Name, types.MergePatchType, []byte(`{"metadata":{"labels":{"velero.io/backup-name":"the-really-long-backup-name-that-is-much-more-than-63-cha6ca4bc"}},"status":{"phase":"InProgress"}}`), ), core.NewGetAction( - velerov1.SchemeGroupVersion.WithResource("backups"), + velerov1api.SchemeGroupVersion.WithResource("backups"), td.req.Namespace, td.req.Spec.BackupName, ), core.NewPatchAction( - velerov1.SchemeGroupVersion.WithResource("deletebackuprequests"), + velerov1api.SchemeGroupVersion.WithResource("deletebackuprequests"), td.req.Namespace, td.req.Name, types.MergePatchType, []byte(`{"metadata":{"labels":{"velero.io/backup-uid":"uid"}}}`), ), core.NewPatchAction( - velerov1.SchemeGroupVersion.WithResource("backups"), + velerov1api.SchemeGroupVersion.WithResource("backups"), td.req.Namespace, td.req.Spec.BackupName, types.MergePatchType, []byte(`{"status":{"phase":"Deleting"}}`), ), core.NewDeleteAction( - velerov1.SchemeGroupVersion.WithResource("restores"), + velerov1api.SchemeGroupVersion.WithResource("restores"), td.req.Namespace, "restore-1", ), core.NewDeleteAction( - velerov1.SchemeGroupVersion.WithResource("restores"), + velerov1api.SchemeGroupVersion.WithResource("restores"), td.req.Namespace, "restore-2", ), core.NewDeleteAction( - velerov1.SchemeGroupVersion.WithResource("backups"), + velerov1api.SchemeGroupVersion.WithResource("backups"), td.req.Namespace, td.req.Spec.BackupName, ), core.NewPatchAction( - velerov1.SchemeGroupVersion.WithResource("deletebackuprequests"), + velerov1api.SchemeGroupVersion.WithResource("deletebackuprequests"), td.req.Namespace, td.req.Name, types.MergePatchType, []byte(`{"status":{"phase":"Processed"}}`), ), core.NewDeleteCollectionAction( - velerov1.SchemeGroupVersion.WithResource("deletebackuprequests"), + velerov1api.SchemeGroupVersion.WithResource("deletebackuprequests"), td.req.Namespace, pkgbackup.NewDeleteBackupRequestListOptions(td.req.Spec.BackupName, "uid"), ), @@ -735,6 +734,7 @@ func TestBackupDeletionControllerProcessRequest(t *testing.T) { } func TestBackupDeletionControllerDeleteExpiredRequests(t *testing.T) { + now := time.Date(2018, 4, 4, 12, 0, 0, 0, time.UTC) unexpired1 := time.Date(2018, 4, 4, 11, 0, 0, 0, time.UTC) unexpired2 := time.Date(2018, 4, 3, 12, 0, 1, 0, time.UTC) @@ -743,7 +743,7 @@ func TestBackupDeletionControllerDeleteExpiredRequests(t *testing.T) { tests := []struct { name string - requests []*velerov1.DeleteBackupRequest + requests []*velerov1api.DeleteBackupRequest expectedDeletions []string }{ { @@ -751,14 +751,14 @@ func TestBackupDeletionControllerDeleteExpiredRequests(t *testing.T) { }, { name: "older than max age, phase = '', don't delete", - requests: []*velerov1.DeleteBackupRequest{ + requests: []*velerov1api.DeleteBackupRequest{ { ObjectMeta: metav1.ObjectMeta{ Namespace: "ns", Name: "name", CreationTimestamp: metav1.Time{Time: expired1}, }, - Status: velerov1.DeleteBackupRequestStatus{ + Status: velerov1api.DeleteBackupRequestStatus{ Phase: "", }, }, @@ -766,45 +766,45 @@ func TestBackupDeletionControllerDeleteExpiredRequests(t *testing.T) { }, { name: "older than max age, phase = New, don't delete", - requests: []*velerov1.DeleteBackupRequest{ + requests: []*velerov1api.DeleteBackupRequest{ { ObjectMeta: metav1.ObjectMeta{ Namespace: "ns", Name: "name", CreationTimestamp: metav1.Time{Time: expired1}, }, - Status: velerov1.DeleteBackupRequestStatus{ - Phase: velerov1.DeleteBackupRequestPhaseNew, + Status: velerov1api.DeleteBackupRequestStatus{ + Phase: velerov1api.DeleteBackupRequestPhaseNew, }, }, }, }, { name: "older than max age, phase = InProcess, don't delete", - requests: []*velerov1.DeleteBackupRequest{ + requests: []*velerov1api.DeleteBackupRequest{ { ObjectMeta: metav1.ObjectMeta{ Namespace: "ns", Name: "name", CreationTimestamp: metav1.Time{Time: expired1}, }, - Status: velerov1.DeleteBackupRequestStatus{ - Phase: velerov1.DeleteBackupRequestPhaseInProgress, + Status: velerov1api.DeleteBackupRequestStatus{ + Phase: velerov1api.DeleteBackupRequestPhaseInProgress, }, }, }, }, { name: "some expired, some not", - requests: []*velerov1.DeleteBackupRequest{ + requests: []*velerov1api.DeleteBackupRequest{ { ObjectMeta: metav1.ObjectMeta{ Namespace: "ns", Name: "unexpired-1", CreationTimestamp: metav1.Time{Time: unexpired1}, }, - Status: velerov1.DeleteBackupRequestStatus{ - Phase: velerov1.DeleteBackupRequestPhaseProcessed, + Status: velerov1api.DeleteBackupRequestStatus{ + Phase: velerov1api.DeleteBackupRequestPhaseProcessed, }, }, { @@ -813,8 +813,8 @@ func TestBackupDeletionControllerDeleteExpiredRequests(t *testing.T) { Name: "expired-1", CreationTimestamp: metav1.Time{Time: expired1}, }, - Status: velerov1.DeleteBackupRequestStatus{ - Phase: velerov1.DeleteBackupRequestPhaseProcessed, + Status: velerov1api.DeleteBackupRequestStatus{ + Phase: velerov1api.DeleteBackupRequestPhaseProcessed, }, }, { @@ -823,8 +823,8 @@ func TestBackupDeletionControllerDeleteExpiredRequests(t *testing.T) { Name: "unexpired-2", CreationTimestamp: metav1.Time{Time: unexpired2}, }, - Status: velerov1.DeleteBackupRequestStatus{ - Phase: velerov1.DeleteBackupRequestPhaseProcessed, + Status: velerov1api.DeleteBackupRequestStatus{ + Phase: velerov1api.DeleteBackupRequestPhaseProcessed, }, }, { @@ -833,8 +833,8 @@ func TestBackupDeletionControllerDeleteExpiredRequests(t *testing.T) { Name: "expired-2", CreationTimestamp: metav1.Time{Time: expired2}, }, - Status: velerov1.DeleteBackupRequestStatus{ - Phase: velerov1.DeleteBackupRequestPhaseProcessed, + Status: velerov1api.DeleteBackupRequestStatus{ + Phase: velerov1api.DeleteBackupRequestPhaseProcessed, }, }, }, @@ -845,6 +845,7 @@ func TestBackupDeletionControllerDeleteExpiredRequests(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { client := fake.NewSimpleClientset() + fakeClient := newFakeClient(t) sharedInformers := informers.NewSharedInformerFactory(client, 0) controller := NewBackupDeletionController( @@ -857,7 +858,7 @@ func TestBackupDeletionControllerDeleteExpiredRequests(t *testing.T) { NewBackupTracker(), nil, sharedInformers.Velero().V1().PodVolumeBackups().Lister(), - sharedInformers.Velero().V1().BackupStorageLocations().Lister(), + fakeClient, sharedInformers.Velero().V1().VolumeSnapshotLocations().Lister(), nil, // csiSnapshotLister nil, // csiSnapshotContentLister @@ -878,7 +879,7 @@ func TestBackupDeletionControllerDeleteExpiredRequests(t *testing.T) { expectedActions := []core.Action{} for _, name := range test.expectedDeletions { - expectedActions = append(expectedActions, core.NewDeleteAction(velerov1.SchemeGroupVersion.WithResource("deletebackuprequests"), "ns", name)) + expectedActions = append(expectedActions, core.NewDeleteAction(velerov1api.SchemeGroupVersion.WithResource("deletebackuprequests"), "ns", name)) } velerotest.CompareActions(t, expectedActions, client.Actions()) @@ -981,7 +982,7 @@ func TestDeleteCSIVolumeSnapshots(t *testing.T) { Name: "vs1", Namespace: "ns1", Labels: map[string]string{ - velerov1.BackupNameLabel: "backup1", + velerov1api.BackupNameLabel: "backup1", }, }, Status: &snapshotv1beta1api.VolumeSnapshotStatus{ @@ -1003,7 +1004,7 @@ func TestDeleteCSIVolumeSnapshots(t *testing.T) { Name: "vs2", Namespace: "ns1", Labels: map[string]string{ - velerov1.BackupNameLabel: "backup1", + velerov1api.BackupNameLabel: "backup1", }, }, Status: &snapshotv1beta1api.VolumeSnapshotStatus{ @@ -1025,7 +1026,7 @@ func TestDeleteCSIVolumeSnapshots(t *testing.T) { Name: "vs1", Namespace: "ns2", Labels: map[string]string{ - velerov1.BackupNameLabel: "backup1", + velerov1api.BackupNameLabel: "backup1", }, }, Status: &snapshotv1beta1api.VolumeSnapshotStatus{ @@ -1047,7 +1048,7 @@ func TestDeleteCSIVolumeSnapshots(t *testing.T) { Name: "vs2", Namespace: "ns2", Labels: map[string]string{ - velerov1.BackupNameLabel: "backup1", + velerov1api.BackupNameLabel: "backup1", }, }, Status: &snapshotv1beta1api.VolumeSnapshotStatus{ @@ -1061,7 +1062,7 @@ func TestDeleteCSIVolumeSnapshots(t *testing.T) { Name: "ns1NilStatusVS", Namespace: "ns2", Labels: map[string]string{ - velerov1.BackupNameLabel: "backup2", + velerov1api.BackupNameLabel: "backup2", }, }, Status: nil, @@ -1073,7 +1074,7 @@ func TestDeleteCSIVolumeSnapshots(t *testing.T) { Name: "ns1NilBoundVSCVS", Namespace: "ns2", Labels: map[string]string{ - velerov1.BackupNameLabel: "backup3", + velerov1api.BackupNameLabel: "backup3", }, }, Status: &snapshotv1beta1api.VolumeSnapshotStatus{ @@ -1088,7 +1089,7 @@ func TestDeleteCSIVolumeSnapshots(t *testing.T) { Name: "ns1NonExistentVSCVS", Namespace: "ns2", Labels: map[string]string{ - velerov1.BackupNameLabel: "backup3", + velerov1api.BackupNameLabel: "backup3", }, }, Status: &snapshotv1beta1api.VolumeSnapshotStatus{ @@ -1151,7 +1152,7 @@ func TestDeleteCSIVolumeSnapshotContents(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "retainVSC", Labels: map[string]string{ - velerov1.BackupNameLabel: "backup1", + velerov1api.BackupNameLabel: "backup1", }, }, Spec: snapshotv1beta1api.VolumeSnapshotContentSpec{ @@ -1162,7 +1163,7 @@ func TestDeleteCSIVolumeSnapshotContents(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "deleteVSC", Labels: map[string]string{ - velerov1.BackupNameLabel: "backup2", + velerov1api.BackupNameLabel: "backup2", }, }, Spec: snapshotv1beta1api.VolumeSnapshotContentSpec{ @@ -1174,7 +1175,7 @@ func TestDeleteCSIVolumeSnapshotContents(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "nothingVSC", Labels: map[string]string{ - velerov1.BackupNameLabel: "backup3", + velerov1api.BackupNameLabel: "backup3", }, }, Spec: snapshotv1beta1api.VolumeSnapshotContentSpec{}, diff --git a/pkg/controller/backup_sync_controller.go b/pkg/controller/backup_sync_controller.go index 6caa6039c..de05f2145 100644 --- a/pkg/controller/backup_sync_controller.go +++ b/pkg/controller/backup_sync_controller.go @@ -17,15 +17,15 @@ limitations under the License. package controller import ( - "encoding/json" + "context" "time" snapshotterClientSet "github.com/kubernetes-csi/external-snapshotter/v2/pkg/client/clientset/versioned" "github.com/pkg/errors" "github.com/sirupsen/logrus" kuberrs "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" - "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/kubernetes" @@ -36,31 +36,31 @@ import ( "github.com/vmware-tanzu/velero/pkg/label" "github.com/vmware-tanzu/velero/pkg/persistence" "github.com/vmware-tanzu/velero/pkg/plugin/clientmgmt" + + "sigs.k8s.io/controller-runtime/pkg/client" ) type backupSyncController struct { *genericController - backupClient velerov1client.BackupsGetter - backupLocationClient velerov1client.BackupStorageLocationsGetter - podVolumeBackupClient velerov1client.PodVolumeBackupsGetter - backupLister velerov1listers.BackupLister - csiSnapshotClient *snapshotterClientSet.Clientset - kubeClient kubernetes.Interface - backupStorageLocationLister velerov1listers.BackupStorageLocationLister - namespace string - defaultBackupLocation string - defaultBackupSyncPeriod time.Duration - newPluginManager func(logrus.FieldLogger) clientmgmt.Manager - newBackupStore func(*velerov1api.BackupStorageLocation, persistence.ObjectStoreGetter, logrus.FieldLogger) (persistence.BackupStore, error) + backupClient velerov1client.BackupsGetter + kbClient client.Client + podVolumeBackupClient velerov1client.PodVolumeBackupsGetter + backupLister velerov1listers.BackupLister + csiSnapshotClient *snapshotterClientSet.Clientset + kubeClient kubernetes.Interface + namespace string + defaultBackupLocation string + defaultBackupSyncPeriod time.Duration + newPluginManager func(logrus.FieldLogger) clientmgmt.Manager + newBackupStore func(*velerov1api.BackupStorageLocation, persistence.ObjectStoreGetter, logrus.FieldLogger) (persistence.BackupStore, error) } func NewBackupSyncController( backupClient velerov1client.BackupsGetter, - backupLocationClient velerov1client.BackupStorageLocationsGetter, + kbClient client.Client, podVolumeBackupClient velerov1client.PodVolumeBackupsGetter, backupLister velerov1listers.BackupLister, - backupStorageLocationLister velerov1listers.BackupStorageLocationLister, syncPeriod time.Duration, namespace string, csiSnapshotClient *snapshotterClientSet.Clientset, @@ -75,17 +75,16 @@ func NewBackupSyncController( logger.Infof("Backup sync period is %v", syncPeriod) c := &backupSyncController{ - genericController: newGenericController("backup-sync", logger), - backupClient: backupClient, - backupLocationClient: backupLocationClient, - podVolumeBackupClient: podVolumeBackupClient, - namespace: namespace, - defaultBackupLocation: defaultBackupLocation, - defaultBackupSyncPeriod: syncPeriod, - backupLister: backupLister, - backupStorageLocationLister: backupStorageLocationLister, - csiSnapshotClient: csiSnapshotClient, - kubeClient: kubeClient, + genericController: newGenericController("backup-sync", logger), + backupClient: backupClient, + kbClient: kbClient, + podVolumeBackupClient: podVolumeBackupClient, + namespace: namespace, + defaultBackupLocation: defaultBackupLocation, + defaultBackupSyncPeriod: syncPeriod, + backupLister: backupLister, + csiSnapshotClient: csiSnapshotClient, + kubeClient: kubeClient, // use variables to refer to these functions so they can be // replaced with fakes for testing. @@ -101,35 +100,38 @@ func NewBackupSyncController( // orderedBackupLocations returns a new slice with the default backup location first (if it exists), // followed by the rest of the locations in no particular order. -func orderedBackupLocations(locations []*velerov1api.BackupStorageLocation, defaultLocationName string) []*velerov1api.BackupStorageLocation { - var result []*velerov1api.BackupStorageLocation +func orderedBackupLocations(locationList *velerov1api.BackupStorageLocationList, defaultLocationName string) []velerov1api.BackupStorageLocation { + var result []velerov1api.BackupStorageLocation - for i := range locations { - if locations[i].Name == defaultLocationName { + for i := range locationList.Items { + if locationList.Items[i].Name == defaultLocationName { // put the default location first - result = append(result, locations[i]) + result = append(result, locationList.Items[i]) // append everything before the default - result = append(result, locations[:i]...) + result = append(result, locationList.Items[:i]...) // append everything after the default - result = append(result, locations[i+1:]...) + result = append(result, locationList.Items[i+1:]...) return result } } - return locations + return locationList.Items } func (c *backupSyncController) run() { c.logger.Debug("Checking for existing backup storage locations to sync into cluster") - locations, err := c.backupStorageLocationLister.BackupStorageLocations(c.namespace).List(labels.Everything()) - if err != nil { + locationList := &velerov1api.BackupStorageLocationList{} + if err := c.kbClient.List(context.Background(), locationList, &client.ListOptions{ + Namespace: c.namespace, + }); err != nil { c.logger.WithError(errors.WithStack(err)).Error("Error getting backup storage locations from lister") return } + // sync the default location first, if it exists - locations = orderedBackupLocations(locations, c.defaultBackupLocation) + locations := orderedBackupLocations(locationList, c.defaultBackupLocation) pluginManager := c.newPluginManager(c.logger) defer pluginManager.CleanupClients() @@ -162,7 +164,7 @@ func (c *backupSyncController) run() { log.Debug("Checking backup location for backups to sync into cluster") - backupStore, err := c.newBackupStore(location, pluginManager, log) + backupStore, err := c.newBackupStore(&location, pluginManager, log) if err != nil { log.WithError(err).Error("Error getting backup store for this location") continue @@ -304,23 +306,9 @@ func (c *backupSyncController) run() { c.deleteOrphanedBackups(location.Name, backupStoreBackups, log) // update the location's last-synced time field - patch := map[string]interface{}{ - "status": map[string]interface{}{ - "lastSyncedTime": time.Now().UTC(), - }, - } - - patchBytes, err := json.Marshal(patch) - if err != nil { - log.WithError(errors.WithStack(err)).Error("Error marshaling last-synced patch to JSON") - continue - } - - if _, err = c.backupLocationClient.BackupStorageLocations(c.namespace).Patch( - location.Name, - types.MergePatchType, - patchBytes, - ); err != nil { + statusPatch := client.MergeFrom(location.DeepCopyObject()) + location.Status.LastSyncedTime = &metav1.Time{Time: time.Now().UTC()} + if err := c.kbClient.Status().Patch(context.Background(), &location, statusPatch); err != nil { log.WithError(errors.WithStack(err)).Error("Error patching backup location's last-synced time") continue } diff --git a/pkg/controller/backup_sync_controller_test.go b/pkg/controller/backup_sync_controller_test.go index 47661e50d..8ff93736c 100644 --- a/pkg/controller/backup_sync_controller_test.go +++ b/pkg/controller/backup_sync_controller_test.go @@ -17,6 +17,7 @@ limitations under the License. package controller import ( + "context" "testing" "time" @@ -332,6 +333,7 @@ func TestBackupSyncControllerRun(t *testing.T) { t.Run(test.name, func(t *testing.T) { var ( client = fake.NewSimpleClientset() + fakeClient = newFakeClient(t) sharedInformers = informers.NewSharedInformerFactory(client, 0) pluginManager = &pluginmocks.Manager{} backupStores = make(map[string]*persistencemocks.BackupStore) @@ -339,10 +341,9 @@ func TestBackupSyncControllerRun(t *testing.T) { c := NewBackupSyncController( client.VeleroV1(), - client.VeleroV1(), + fakeClient, client.VeleroV1(), sharedInformers.Velero().V1().Backups().Lister(), - sharedInformers.Velero().V1().BackupStorageLocations().Lister(), time.Duration(0), test.namespace, nil, // csiSnapshotClient @@ -360,7 +361,7 @@ func TestBackupSyncControllerRun(t *testing.T) { pluginManager.On("CleanupClients").Return(nil) for _, location := range test.locations { - require.NoError(t, sharedInformers.Velero().V1().BackupStorageLocations().Informer().GetStore().Add(location)) + require.NoError(t, fakeClient.Create(context.Background(), location)) backupStores[location.Name] = &persistencemocks.BackupStore{} } @@ -559,15 +560,15 @@ func TestDeleteOrphanedBackups(t *testing.T) { t.Run(test.name, func(t *testing.T) { var ( client = fake.NewSimpleClientset() + fakeClient = newFakeClient(t) sharedInformers = informers.NewSharedInformerFactory(client, 0) ) c := NewBackupSyncController( client.VeleroV1(), - client.VeleroV1(), + fakeClient, client.VeleroV1(), sharedInformers.Velero().V1().Backups().Lister(), - sharedInformers.Velero().V1().BackupStorageLocations().Lister(), time.Duration(0), test.namespace, nil, // csiSnapshotClient @@ -652,15 +653,15 @@ func TestStorageLabelsInDeleteOrphanedBackups(t *testing.T) { t.Run(test.name, func(t *testing.T) { var ( client = fake.NewSimpleClientset() + fakeClient = newFakeClient(t) sharedInformers = informers.NewSharedInformerFactory(client, 0) ) c := NewBackupSyncController( client.VeleroV1(), - client.VeleroV1(), + fakeClient, client.VeleroV1(), sharedInformers.Velero().V1().Backups().Lister(), - sharedInformers.Velero().V1().BackupStorageLocations().Lister(), time.Duration(0), test.namespace, nil, // csiSnapshotClient diff --git a/pkg/controller/download_request_controller.go b/pkg/controller/download_request_controller.go index 7bd84cdce..51125c92f 100644 --- a/pkg/controller/download_request_controller.go +++ b/pkg/controller/download_request_controller.go @@ -17,6 +17,7 @@ limitations under the License. package controller import ( + "context" "encoding/json" "time" @@ -29,8 +30,9 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/clock" "k8s.io/client-go/tools/cache" + "sigs.k8s.io/controller-runtime/pkg/client" - v1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velerov1client "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned/typed/velero/v1" velerov1informers "github.com/vmware-tanzu/velero/pkg/generated/informers/externalversions/velero/v1" velerov1listers "github.com/vmware-tanzu/velero/pkg/generated/listers/velero/v1" @@ -46,10 +48,10 @@ type downloadRequestController struct { downloadRequestLister velerov1listers.DownloadRequestLister restoreLister velerov1listers.RestoreLister clock clock.Clock - backupLocationLister velerov1listers.BackupStorageLocationLister + kbClient client.Client backupLister velerov1listers.BackupLister newPluginManager func(logrus.FieldLogger) clientmgmt.Manager - newBackupStore func(*v1.BackupStorageLocation, persistence.ObjectStoreGetter, logrus.FieldLogger) (persistence.BackupStore, error) + newBackupStore func(*velerov1api.BackupStorageLocation, persistence.ObjectStoreGetter, logrus.FieldLogger) (persistence.BackupStore, error) } // NewDownloadRequestController creates a new DownloadRequestController. @@ -57,7 +59,7 @@ func NewDownloadRequestController( downloadRequestClient velerov1client.DownloadRequestsGetter, downloadRequestInformer velerov1informers.DownloadRequestInformer, restoreLister velerov1listers.RestoreLister, - backupLocationLister velerov1listers.BackupStorageLocationLister, + kbClient client.Client, backupLister velerov1listers.BackupLister, newPluginManager func(logrus.FieldLogger) clientmgmt.Manager, logger logrus.FieldLogger, @@ -67,7 +69,7 @@ func NewDownloadRequestController( downloadRequestClient: downloadRequestClient, downloadRequestLister: downloadRequestInformer.Lister(), restoreLister: restoreLister, - backupLocationLister: backupLocationLister, + kbClient: kbClient, backupLister: backupLister, // use variables to refer to these functions so they can be @@ -85,7 +87,7 @@ func NewDownloadRequestController( AddFunc: func(obj interface{}) { key, err := cache.MetaNamespaceKeyFunc(obj) if err != nil { - downloadRequest := obj.(*v1.DownloadRequest) + downloadRequest := obj.(*velerov1api.DownloadRequest) c.logger.WithError(errors.WithStack(err)). WithField("downloadRequest", downloadRequest.Name). Error("Error creating queue key, item not added to queue") @@ -121,9 +123,9 @@ func (c *downloadRequestController) processDownloadRequest(key string) error { } switch downloadRequest.Status.Phase { - case "", v1.DownloadRequestPhaseNew: + case "", velerov1api.DownloadRequestPhaseNew: return c.generatePreSignedURL(downloadRequest, log) - case v1.DownloadRequestPhaseProcessed: + case velerov1api.DownloadRequestPhaseProcessed: return c.deleteIfExpired(downloadRequest) } @@ -134,7 +136,7 @@ const signedURLTTL = 10 * time.Minute // generatePreSignedURL generates a pre-signed URL for downloadRequest, changes the phase to // Processed, and persists the changes to storage. -func (c *downloadRequestController) generatePreSignedURL(downloadRequest *v1.DownloadRequest, log logrus.FieldLogger) error { +func (c *downloadRequestController) generatePreSignedURL(downloadRequest *velerov1api.DownloadRequest, log logrus.FieldLogger) error { update := downloadRequest.DeepCopy() var ( @@ -143,7 +145,7 @@ func (c *downloadRequestController) generatePreSignedURL(downloadRequest *v1.Dow ) switch downloadRequest.Spec.Target.Kind { - case v1.DownloadTargetKindRestoreLog, v1.DownloadTargetKindRestoreResults: + case velerov1api.DownloadTargetKindRestoreLog, velerov1api.DownloadTargetKindRestoreResults: restore, err := c.restoreLister.Restores(downloadRequest.Namespace).Get(downloadRequest.Spec.Target.Name) if err != nil { return errors.Wrap(err, "error getting Restore") @@ -159,8 +161,11 @@ func (c *downloadRequestController) generatePreSignedURL(downloadRequest *v1.Dow return errors.WithStack(err) } - backupLocation, err := c.backupLocationLister.BackupStorageLocations(backup.Namespace).Get(backup.Spec.StorageLocation) - if err != nil { + backupLocation := &velerov1api.BackupStorageLocation{} + if err := c.kbClient.Get(context.Background(), client.ObjectKey{ + Namespace: backup.Namespace, + Name: backup.Spec.StorageLocation, + }, backupLocation); err != nil { return errors.WithStack(err) } @@ -176,7 +181,7 @@ func (c *downloadRequestController) generatePreSignedURL(downloadRequest *v1.Dow return err } - update.Status.Phase = v1.DownloadRequestPhaseProcessed + update.Status.Phase = velerov1api.DownloadRequestPhaseProcessed update.Status.Expiration = &metav1.Time{Time: c.clock.Now().Add(persistence.DownloadURLTTL)} _, err = patchDownloadRequest(downloadRequest, update, c.downloadRequestClient) @@ -184,7 +189,7 @@ func (c *downloadRequestController) generatePreSignedURL(downloadRequest *v1.Dow } // deleteIfExpired deletes downloadRequest if it has expired. -func (c *downloadRequestController) deleteIfExpired(downloadRequest *v1.DownloadRequest) error { +func (c *downloadRequestController) deleteIfExpired(downloadRequest *velerov1api.DownloadRequest) error { log := c.logger.WithField("key", kube.NamespaceAndName(downloadRequest)) log.Info("checking for expiration of DownloadRequest") if downloadRequest.Status.Expiration.Time.After(c.clock.Now()) { @@ -216,7 +221,7 @@ func (c *downloadRequestController) resync() { } } -func patchDownloadRequest(original, updated *v1.DownloadRequest, client velerov1client.DownloadRequestsGetter) (*v1.DownloadRequest, error) { +func patchDownloadRequest(original, updated *velerov1api.DownloadRequest, client velerov1client.DownloadRequestsGetter) (*velerov1api.DownloadRequest, error) { origBytes, err := json.Marshal(original) if err != nil { return nil, errors.Wrap(err, "error marshalling original download request") diff --git a/pkg/controller/download_request_controller_test.go b/pkg/controller/download_request_controller_test.go index 89d987776..c17bd20fb 100644 --- a/pkg/controller/download_request_controller_test.go +++ b/pkg/controller/download_request_controller_test.go @@ -27,7 +27,9 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/clock" - v1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/builder" "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned/fake" informers "github.com/vmware-tanzu/velero/pkg/generated/informers/externalversions" @@ -48,7 +50,7 @@ type downloadRequestTestHarness struct { controller *downloadRequestController } -func newDownloadRequestTestHarness(t *testing.T) *downloadRequestTestHarness { +func newDownloadRequestTestHarness(t *testing.T, fakeClient client.Client) *downloadRequestTestHarness { var ( client = fake.NewSimpleClientset() informerFactory = informers.NewSharedInformerFactory(client, 0) @@ -58,7 +60,7 @@ func newDownloadRequestTestHarness(t *testing.T) *downloadRequestTestHarness { client.VeleroV1(), informerFactory.Velero().V1().DownloadRequests(), informerFactory.Velero().V1().Restores().Lister(), - informerFactory.Velero().V1().BackupStorageLocations().Lister(), + fakeClient, informerFactory.Velero().V1().Backups().Lister(), func(logrus.FieldLogger) clientmgmt.Manager { return pluginManager }, velerotest.NewLogger(), @@ -69,7 +71,7 @@ func newDownloadRequestTestHarness(t *testing.T) *downloadRequestTestHarness { require.NoError(t, err) controller.clock = clock.NewFakeClock(clockTime) - controller.newBackupStore = func(*v1.BackupStorageLocation, persistence.ObjectStoreGetter, logrus.FieldLogger) (persistence.BackupStore, error) { + controller.newBackupStore = func(*velerov1api.BackupStorageLocation, persistence.ObjectStoreGetter, logrus.FieldLogger) (persistence.BackupStore, error) { return backupStore, nil } @@ -84,34 +86,34 @@ func newDownloadRequestTestHarness(t *testing.T) *downloadRequestTestHarness { } } -func newDownloadRequest(phase v1.DownloadRequestPhase, targetKind v1.DownloadTargetKind, targetName string) *v1.DownloadRequest { - return &v1.DownloadRequest{ +func newDownloadRequest(phase velerov1api.DownloadRequestPhase, targetKind velerov1api.DownloadTargetKind, targetName string) *velerov1api.DownloadRequest { + return &velerov1api.DownloadRequest{ ObjectMeta: metav1.ObjectMeta{ Name: "a-download-request", - Namespace: v1.DefaultNamespace, + Namespace: velerov1api.DefaultNamespace, }, - Spec: v1.DownloadRequestSpec{ - Target: v1.DownloadTarget{ + Spec: velerov1api.DownloadRequestSpec{ + Target: velerov1api.DownloadTarget{ Kind: targetKind, Name: targetName, }, }, - Status: v1.DownloadRequestStatus{ + Status: velerov1api.DownloadRequestStatus{ Phase: phase, }, } } -func newBackupLocation(name, provider, bucket string) *v1.BackupStorageLocation { - return &v1.BackupStorageLocation{ +func newBackupLocation(name, provider, bucket string) *velerov1api.BackupStorageLocation { + return &velerov1api.BackupStorageLocation{ ObjectMeta: metav1.ObjectMeta{ Name: name, - Namespace: v1.DefaultNamespace, + Namespace: velerov1api.DefaultNamespace, }, - Spec: v1.BackupStorageLocationSpec{ + Spec: velerov1api.BackupStorageLocationSpec{ Provider: provider, - StorageType: v1.StorageType{ - ObjectStorage: &v1.ObjectStorageLocation{ + StorageType: velerov1api.StorageType{ + ObjectStorage: &velerov1api.ObjectStorageLocation{ Bucket: bucket, }, }, @@ -120,17 +122,18 @@ func newBackupLocation(name, provider, bucket string) *v1.BackupStorageLocation } func TestProcessDownloadRequest(t *testing.T) { - defaultBackup := func() *v1.Backup { - return builder.ForBackup(v1.DefaultNamespace, "a-backup").StorageLocation("a-location").Result() + + defaultBackup := func() *velerov1api.Backup { + return builder.ForBackup(velerov1api.DefaultNamespace, "a-backup").StorageLocation("a-location").Result() } tests := []struct { name string key string - downloadRequest *v1.DownloadRequest - backup *v1.Backup - restore *v1.Restore - backupLocation *v1.BackupStorageLocation + downloadRequest *velerov1api.DownloadRequest + backup *velerov1api.Backup + restore *velerov1api.Restore + backupLocation *velerov1api.BackupStorageLocation expired bool expectedErr string expectGetsURL bool @@ -149,94 +152,94 @@ func TestProcessDownloadRequest(t *testing.T) { }, { name: "backup contents request for nonexistent backup returns an error", - downloadRequest: newDownloadRequest("", v1.DownloadTargetKindBackupContents, "a-backup"), - backup: builder.ForBackup(v1.DefaultNamespace, "non-matching-backup").StorageLocation("a-location").Result(), + downloadRequest: newDownloadRequest("", velerov1api.DownloadTargetKindBackupContents, "a-backup"), + backup: builder.ForBackup(velerov1api.DefaultNamespace, "non-matching-backup").StorageLocation("a-location").Result(), backupLocation: newBackupLocation("a-location", "a-provider", "a-bucket"), expectedErr: "backup.velero.io \"a-backup\" not found", }, { name: "restore log request for nonexistent restore returns an error", - downloadRequest: newDownloadRequest("", v1.DownloadTargetKindRestoreLog, "a-backup-20170912150214"), - restore: builder.ForRestore(v1.DefaultNamespace, "non-matching-restore").Phase(v1.RestorePhaseCompleted).Backup("a-backup").Result(), + downloadRequest: newDownloadRequest("", velerov1api.DownloadTargetKindRestoreLog, "a-backup-20170912150214"), + restore: builder.ForRestore(velerov1api.DefaultNamespace, "non-matching-restore").Phase(velerov1api.RestorePhaseCompleted).Backup("a-backup").Result(), backup: defaultBackup(), backupLocation: newBackupLocation("a-location", "a-provider", "a-bucket"), expectedErr: "error getting Restore: restore.velero.io \"a-backup-20170912150214\" not found", }, { name: "backup contents request for backup with nonexistent location returns an error", - downloadRequest: newDownloadRequest("", v1.DownloadTargetKindBackupContents, "a-backup"), + downloadRequest: newDownloadRequest("", velerov1api.DownloadTargetKindBackupContents, "a-backup"), backup: defaultBackup(), backupLocation: newBackupLocation("non-matching-location", "a-provider", "a-bucket"), - expectedErr: "backupstoragelocation.velero.io \"a-location\" not found", + expectedErr: "backupstoragelocations.velero.io \"a-location\" not found", }, { name: "backup contents request with phase '' gets a url", - downloadRequest: newDownloadRequest("", v1.DownloadTargetKindBackupContents, "a-backup"), + downloadRequest: newDownloadRequest("", velerov1api.DownloadTargetKindBackupContents, "a-backup"), backup: defaultBackup(), backupLocation: newBackupLocation("a-location", "a-provider", "a-bucket"), expectGetsURL: true, }, { name: "backup contents request with phase 'New' gets a url", - downloadRequest: newDownloadRequest(v1.DownloadRequestPhaseNew, v1.DownloadTargetKindBackupContents, "a-backup"), + downloadRequest: newDownloadRequest(velerov1api.DownloadRequestPhaseNew, velerov1api.DownloadTargetKindBackupContents, "a-backup"), backup: defaultBackup(), backupLocation: newBackupLocation("a-location", "a-provider", "a-bucket"), expectGetsURL: true, }, { name: "backup log request with phase '' gets a url", - downloadRequest: newDownloadRequest("", v1.DownloadTargetKindBackupLog, "a-backup"), + downloadRequest: newDownloadRequest("", velerov1api.DownloadTargetKindBackupLog, "a-backup"), backup: defaultBackup(), backupLocation: newBackupLocation("a-location", "a-provider", "a-bucket"), expectGetsURL: true, }, { name: "backup log request with phase 'New' gets a url", - downloadRequest: newDownloadRequest(v1.DownloadRequestPhaseNew, v1.DownloadTargetKindBackupLog, "a-backup"), + downloadRequest: newDownloadRequest(velerov1api.DownloadRequestPhaseNew, velerov1api.DownloadTargetKindBackupLog, "a-backup"), backup: defaultBackup(), backupLocation: newBackupLocation("a-location", "a-provider", "a-bucket"), expectGetsURL: true, }, { name: "restore log request with phase '' gets a url", - downloadRequest: newDownloadRequest("", v1.DownloadTargetKindRestoreLog, "a-backup-20170912150214"), - restore: builder.ForRestore(v1.DefaultNamespace, "a-backup-20170912150214").Phase(v1.RestorePhaseCompleted).Backup("a-backup").Result(), + downloadRequest: newDownloadRequest("", velerov1api.DownloadTargetKindRestoreLog, "a-backup-20170912150214"), + restore: builder.ForRestore(velerov1api.DefaultNamespace, "a-backup-20170912150214").Phase(velerov1api.RestorePhaseCompleted).Backup("a-backup").Result(), backup: defaultBackup(), backupLocation: newBackupLocation("a-location", "a-provider", "a-bucket"), expectGetsURL: true, }, { name: "restore log request with phase 'New' gets a url", - downloadRequest: newDownloadRequest(v1.DownloadRequestPhaseNew, v1.DownloadTargetKindRestoreLog, "a-backup-20170912150214"), - restore: builder.ForRestore(v1.DefaultNamespace, "a-backup-20170912150214").Phase(v1.RestorePhaseCompleted).Backup("a-backup").Result(), + downloadRequest: newDownloadRequest(velerov1api.DownloadRequestPhaseNew, velerov1api.DownloadTargetKindRestoreLog, "a-backup-20170912150214"), + restore: builder.ForRestore(velerov1api.DefaultNamespace, "a-backup-20170912150214").Phase(velerov1api.RestorePhaseCompleted).Backup("a-backup").Result(), backup: defaultBackup(), backupLocation: newBackupLocation("a-location", "a-provider", "a-bucket"), expectGetsURL: true, }, { name: "restore results request with phase '' gets a url", - downloadRequest: newDownloadRequest("", v1.DownloadTargetKindRestoreResults, "a-backup-20170912150214"), - restore: builder.ForRestore(v1.DefaultNamespace, "a-backup-20170912150214").Phase(v1.RestorePhaseCompleted).Backup("a-backup").Result(), + downloadRequest: newDownloadRequest("", velerov1api.DownloadTargetKindRestoreResults, "a-backup-20170912150214"), + restore: builder.ForRestore(velerov1api.DefaultNamespace, "a-backup-20170912150214").Phase(velerov1api.RestorePhaseCompleted).Backup("a-backup").Result(), backup: defaultBackup(), backupLocation: newBackupLocation("a-location", "a-provider", "a-bucket"), expectGetsURL: true, }, { name: "restore results request with phase 'New' gets a url", - downloadRequest: newDownloadRequest(v1.DownloadRequestPhaseNew, v1.DownloadTargetKindRestoreResults, "a-backup-20170912150214"), - restore: builder.ForRestore(v1.DefaultNamespace, "a-backup-20170912150214").Phase(v1.RestorePhaseCompleted).Backup("a-backup").Result(), + downloadRequest: newDownloadRequest(velerov1api.DownloadRequestPhaseNew, velerov1api.DownloadTargetKindRestoreResults, "a-backup-20170912150214"), + restore: builder.ForRestore(velerov1api.DefaultNamespace, "a-backup-20170912150214").Phase(velerov1api.RestorePhaseCompleted).Backup("a-backup").Result(), backup: defaultBackup(), backupLocation: newBackupLocation("a-location", "a-provider", "a-bucket"), expectGetsURL: true, }, { name: "request with phase 'Processed' is not deleted if not expired", - downloadRequest: newDownloadRequest(v1.DownloadRequestPhaseProcessed, v1.DownloadTargetKindBackupLog, "a-backup-20170912150214"), + downloadRequest: newDownloadRequest(velerov1api.DownloadRequestPhaseProcessed, velerov1api.DownloadTargetKindBackupLog, "a-backup-20170912150214"), backup: defaultBackup(), }, { name: "request with phase 'Processed' is deleted if expired", - downloadRequest: newDownloadRequest(v1.DownloadRequestPhaseProcessed, v1.DownloadTargetKindBackupLog, "a-backup-20170912150214"), + downloadRequest: newDownloadRequest(velerov1api.DownloadRequestPhaseProcessed, velerov1api.DownloadTargetKindBackupLog, "a-backup-20170912150214"), backup: defaultBackup(), expired: true, }, @@ -244,13 +247,20 @@ func TestProcessDownloadRequest(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - harness := newDownloadRequestTestHarness(t) + var fakeClient client.Client + if tc.backupLocation != nil { + fakeClient = newFakeClient(t, tc.backupLocation) + } else { + fakeClient = newFakeClient(t) + } + + harness := newDownloadRequestTestHarness(t, fakeClient) // set up test case data // Set .status.expiration properly for processed requests. Since "expired" is relative to the controller's // clock time, it's easier to do this here than as part of the test case definitions. - if tc.downloadRequest != nil && tc.downloadRequest.Status.Phase == v1.DownloadRequestPhaseProcessed { + if tc.downloadRequest != nil && tc.downloadRequest.Status.Phase == velerov1api.DownloadRequestPhaseProcessed { if tc.expired { tc.downloadRequest.Status.Expiration = &metav1.Time{Time: harness.controller.clock.Now().Add(-1 * time.Minute)} } else { @@ -273,10 +283,6 @@ func TestProcessDownloadRequest(t *testing.T) { require.NoError(t, harness.informerFactory.Velero().V1().Backups().Informer().GetStore().Add(tc.backup)) } - if tc.backupLocation != nil { - require.NoError(t, harness.informerFactory.Velero().V1().BackupStorageLocations().Informer().GetStore().Add(tc.backupLocation)) - } - if tc.expectGetsURL { harness.backupStore.On("GetDownloadURL", tc.downloadRequest.Spec.Target).Return("a-url", nil) } @@ -300,12 +306,12 @@ func TestProcessDownloadRequest(t *testing.T) { output, err := harness.client.VeleroV1().DownloadRequests(tc.downloadRequest.Namespace).Get(tc.downloadRequest.Name, metav1.GetOptions{}) require.NoError(t, err) - assert.Equal(t, string(v1.DownloadRequestPhaseProcessed), string(output.Status.Phase)) + assert.Equal(t, string(velerov1api.DownloadRequestPhaseProcessed), string(output.Status.Phase)) assert.Equal(t, "a-url", output.Status.DownloadURL) assert.True(t, velerotest.TimesAreEqual(harness.controller.clock.Now().Add(signedURLTTL), output.Status.Expiration.Time), "expiration does not match") } - if tc.downloadRequest != nil && tc.downloadRequest.Status.Phase == v1.DownloadRequestPhaseProcessed { + if tc.downloadRequest != nil && tc.downloadRequest.Status.Phase == velerov1api.DownloadRequestPhaseProcessed { res, err := harness.client.VeleroV1().DownloadRequests(tc.downloadRequest.Namespace).Get(tc.downloadRequest.Name, metav1.GetOptions{}) if tc.expired { diff --git a/pkg/controller/gc_controller.go b/pkg/controller/gc_controller.go index f262e2e8a..488589aa6 100644 --- a/pkg/controller/gc_controller.go +++ b/pkg/controller/gc_controller.go @@ -17,6 +17,7 @@ limitations under the License. package controller import ( + "context" "time" "github.com/pkg/errors" @@ -26,6 +27,8 @@ import ( "k8s.io/apimachinery/pkg/util/clock" "k8s.io/client-go/tools/cache" + "sigs.k8s.io/controller-runtime/pkg/client" + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" pkgbackup "github.com/vmware-tanzu/velero/pkg/backup" velerov1client "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned/typed/velero/v1" @@ -45,7 +48,7 @@ type gcController struct { backupLister velerov1listers.BackupLister deleteBackupRequestLister velerov1listers.DeleteBackupRequestLister deleteBackupRequestClient velerov1client.DeleteBackupRequestsGetter - backupLocationLister velerov1listers.BackupStorageLocationLister + kbClient client.Client clock clock.Clock } @@ -56,7 +59,7 @@ func NewGCController( backupInformer velerov1informers.BackupInformer, deleteBackupRequestLister velerov1listers.DeleteBackupRequestLister, deleteBackupRequestClient velerov1client.DeleteBackupRequestsGetter, - backupLocationLister velerov1listers.BackupStorageLocationLister, + kbClient client.Client, ) Interface { c := &gcController{ genericController: newGenericController("gc-controller", logger), @@ -64,7 +67,7 @@ func NewGCController( backupLister: backupInformer.Lister(), deleteBackupRequestLister: deleteBackupRequestLister, deleteBackupRequestClient: deleteBackupRequestClient, - backupLocationLister: backupLocationLister, + kbClient: kbClient, } c.syncHandler = c.processQueueItem @@ -130,11 +133,14 @@ func (c *gcController) processQueueItem(key string) error { log.Info("Backup has expired") - loc, err := c.backupLocationLister.BackupStorageLocations(ns).Get(backup.Spec.StorageLocation) - if apierrors.IsNotFound(err) { - log.Warnf("Backup cannot be garbage-collected because backup storage location %s does not exist", backup.Spec.StorageLocation) - } - if err != nil { + loc := &velerov1api.BackupStorageLocation{} + if err := c.kbClient.Get(context.Background(), client.ObjectKey{ + Namespace: ns, + Name: backup.Spec.StorageLocation, + }, loc); err != nil { + if apierrors.IsNotFound(err) { + log.Warnf("Backup cannot be garbage-collected because backup storage location %s does not exist", backup.Spec.StorageLocation) + } return errors.Wrap(err, "error getting backup storage location") } diff --git a/pkg/controller/gc_controller_test.go b/pkg/controller/gc_controller_test.go index c8e053c62..7663b8951 100644 --- a/pkg/controller/gc_controller_test.go +++ b/pkg/controller/gc_controller_test.go @@ -32,7 +32,9 @@ import ( "k8s.io/apimachinery/pkg/watch" core "k8s.io/client-go/testing" - api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + kbclient "sigs.k8s.io/controller-runtime/pkg/client" + + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/builder" "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned/fake" informers "github.com/vmware-tanzu/velero/pkg/generated/informers/externalversions" @@ -50,7 +52,7 @@ func TestGCControllerEnqueueAllBackups(t *testing.T) { sharedInformers.Velero().V1().Backups(), sharedInformers.Velero().V1().DeleteBackupRequests().Lister(), client.VeleroV1(), - sharedInformers.Velero().V1().BackupStorageLocations().Lister(), + nil, ).(*gcController) ) @@ -64,7 +66,7 @@ func TestGCControllerEnqueueAllBackups(t *testing.T) { var expected []string for i := 0; i < 3; i++ { - backup := builder.ForBackup(api.DefaultNamespace, fmt.Sprintf("backup-%d", i)).Result() + backup := builder.ForBackup(velerov1api.DefaultNamespace, fmt.Sprintf("backup-%d", i)).Result() sharedInformers.Velero().V1().Backups().Informer().GetStore().Add(backup) expected = append(expected, kube.NamespaceAndName(backup)) } @@ -111,7 +113,7 @@ func TestGCControllerHasUpdateFunc(t *testing.T) { sharedInformers.Velero().V1().Backups(), sharedInformers.Velero().V1().DeleteBackupRequests().Lister(), client.VeleroV1(), - sharedInformers.Velero().V1().BackupStorageLocations().Lister(), + nil, ).(*gcController) keys := make(chan string) @@ -148,14 +150,15 @@ func TestGCControllerHasUpdateFunc(t *testing.T) { } func TestGCControllerProcessQueueItem(t *testing.T) { + fakeClock := clock.NewFakeClock(time.Now()) defaultBackupLocation := builder.ForBackupStorageLocation("velero", "default").Result() tests := []struct { name string - backup *api.Backup - deleteBackupRequests []*api.DeleteBackupRequest - backupLocation *api.BackupStorageLocation + backup *velerov1api.Backup + deleteBackupRequests []*velerov1api.DeleteBackupRequest + backupLocation *velerov1api.BackupStorageLocation expectDeletion bool createDeleteBackupRequestError bool expectError bool @@ -172,13 +175,13 @@ func TestGCControllerProcessQueueItem(t *testing.T) { { name: "expired backup in read-only storage location is not deleted", backup: defaultBackup().Expiration(fakeClock.Now().Add(-time.Minute)).StorageLocation("read-only").Result(), - backupLocation: builder.ForBackupStorageLocation("velero", "read-only").AccessMode(api.BackupStorageLocationAccessModeReadOnly).Result(), + backupLocation: builder.ForBackupStorageLocation("velero", "read-only").AccessMode(velerov1api.BackupStorageLocationAccessModeReadOnly).Result(), expectDeletion: false, }, { name: "expired backup in read-write storage location is deleted", backup: defaultBackup().Expiration(fakeClock.Now().Add(-time.Minute)).StorageLocation("read-write").Result(), - backupLocation: builder.ForBackupStorageLocation("velero", "read-write").AccessMode(api.BackupStorageLocationAccessModeReadWrite).Result(), + backupLocation: builder.ForBackupStorageLocation("velero", "read-write").AccessMode(velerov1api.BackupStorageLocationAccessModeReadWrite).Result(), expectDeletion: true, }, { @@ -191,18 +194,18 @@ func TestGCControllerProcessQueueItem(t *testing.T) { name: "expired backup with a pending deletion request is not deleted", backup: defaultBackup().Expiration(fakeClock.Now().Add(-time.Second)).StorageLocation("default").Result(), backupLocation: defaultBackupLocation, - deleteBackupRequests: []*api.DeleteBackupRequest{ + deleteBackupRequests: []*velerov1api.DeleteBackupRequest{ { ObjectMeta: metav1.ObjectMeta{ - Namespace: api.DefaultNamespace, + Namespace: velerov1api.DefaultNamespace, Name: "foo", Labels: map[string]string{ - api.BackupNameLabel: "backup-1", - api.BackupUIDLabel: "", + velerov1api.BackupNameLabel: "backup-1", + velerov1api.BackupUIDLabel: "", }, }, - Status: api.DeleteBackupRequestStatus{ - Phase: api.DeleteBackupRequestPhaseInProgress, + Status: velerov1api.DeleteBackupRequestStatus{ + Phase: velerov1api.DeleteBackupRequestPhaseInProgress, }, }, }, @@ -212,18 +215,18 @@ func TestGCControllerProcessQueueItem(t *testing.T) { name: "expired backup with only processed deletion requests is deleted", backup: defaultBackup().Expiration(fakeClock.Now().Add(-time.Second)).StorageLocation("default").Result(), backupLocation: defaultBackupLocation, - deleteBackupRequests: []*api.DeleteBackupRequest{ + deleteBackupRequests: []*velerov1api.DeleteBackupRequest{ { ObjectMeta: metav1.ObjectMeta{ - Namespace: api.DefaultNamespace, + Namespace: velerov1api.DefaultNamespace, Name: "foo", Labels: map[string]string{ - api.BackupNameLabel: "backup-1", - api.BackupUIDLabel: "", + velerov1api.BackupNameLabel: "backup-1", + velerov1api.BackupUIDLabel: "", }, }, - Status: api.DeleteBackupRequestStatus{ - Phase: api.DeleteBackupRequestPhaseProcessed, + Status: velerov1api.DeleteBackupRequestStatus{ + Phase: velerov1api.DeleteBackupRequestPhaseProcessed, }, }, }, @@ -246,12 +249,19 @@ func TestGCControllerProcessQueueItem(t *testing.T) { sharedInformers = informers.NewSharedInformerFactory(client, 0) ) + var fakeClient kbclient.Client + if test.backupLocation != nil { + fakeClient = newFakeClient(t, test.backupLocation) + } else { + fakeClient = newFakeClient(t) + } + controller := NewGCController( velerotest.NewLogger(), sharedInformers.Velero().V1().Backups(), sharedInformers.Velero().V1().DeleteBackupRequests().Lister(), client.VeleroV1(), - sharedInformers.Velero().V1().BackupStorageLocations().Lister(), + fakeClient, ).(*gcController) controller.clock = fakeClock @@ -261,10 +271,6 @@ func TestGCControllerProcessQueueItem(t *testing.T) { sharedInformers.Velero().V1().Backups().Informer().GetStore().Add(test.backup) } - if test.backupLocation != nil { - sharedInformers.Velero().V1().BackupStorageLocations().Informer().GetStore().Add(test.backupLocation) - } - for _, dbr := range test.deleteBackupRequests { sharedInformers.Velero().V1().DeleteBackupRequests().Informer().GetStore().Add(dbr) } diff --git a/pkg/controller/pod_volume_backup_controller.go b/pkg/controller/pod_volume_backup_controller.go index ae57546bd..754fb2c41 100644 --- a/pkg/controller/pod_volume_backup_controller.go +++ b/pkg/controller/pod_volume_backup_controller.go @@ -42,6 +42,8 @@ import ( "github.com/vmware-tanzu/velero/pkg/restic" "github.com/vmware-tanzu/velero/pkg/util/filesystem" "github.com/vmware-tanzu/velero/pkg/util/kube" + + "sigs.k8s.io/controller-runtime/pkg/client" ) type podVolumeBackupController struct { @@ -53,7 +55,7 @@ type podVolumeBackupController struct { podLister corev1listers.PodLister pvcLister corev1listers.PersistentVolumeClaimLister pvLister corev1listers.PersistentVolumeLister - backupLocationLister listers.BackupStorageLocationLister + kbClient client.Client nodeName string processBackupFunc func(*velerov1api.PodVolumeBackup) error @@ -70,7 +72,7 @@ func NewPodVolumeBackupController( secretInformer cache.SharedIndexInformer, pvcInformer corev1informers.PersistentVolumeClaimInformer, pvInformer corev1informers.PersistentVolumeInformer, - backupLocationInformer informers.BackupStorageLocationInformer, + kbClient client.Client, nodeName string, ) Interface { c := &podVolumeBackupController{ @@ -81,7 +83,7 @@ func NewPodVolumeBackupController( secretLister: corev1listers.NewSecretLister(secretInformer.GetIndexer()), pvcLister: pvcInformer.Lister(), pvLister: pvInformer.Lister(), - backupLocationLister: backupLocationInformer.Lister(), + kbClient: kbClient, nodeName: nodeName, fileSystem: filesystem.NewFileSystem(), @@ -95,7 +97,6 @@ func NewPodVolumeBackupController( podInformer.HasSynced, secretInformer.HasSynced, pvcInformer.Informer().HasSynced, - backupLocationInformer.Informer().HasSynced, ) c.processBackupFunc = c.processBackup @@ -228,10 +229,11 @@ func (c *podVolumeBackupController) processBackup(req *velerov1api.PodVolumeBack ) // if there's a caCert on the ObjectStorage, write it to disk so that it can be passed to restic - caCert, err := restic.GetCACert(c.backupLocationLister, req.Namespace, req.Spec.BackupStorageLocation) + caCert, err := restic.GetCACert(c.kbClient, req.Namespace, req.Spec.BackupStorageLocation) if err != nil { log.WithError(err).Error("Error getting caCert") } + var caCertFile string if caCert != nil { caCertFile, err = restic.TempCACertFile(caCert, req.Spec.BackupStorageLocation, c.fileSystem) @@ -247,12 +249,12 @@ func (c *podVolumeBackupController) processBackup(req *velerov1api.PodVolumeBack // set resticCmd.Env appropriately (currently for Azure and S3 based backuplocations) var env []string if strings.HasPrefix(req.Spec.RepoIdentifier, "azure") { - if env, err = restic.AzureCmdEnv(c.backupLocationLister, req.Namespace, req.Spec.BackupStorageLocation); err != nil { + if env, err = restic.AzureCmdEnv(c.kbClient, req.Namespace, req.Spec.BackupStorageLocation); err != nil { return c.fail(req, errors.Wrap(err, "error setting restic cmd env").Error(), log) } resticCmd.Env = env } else if strings.HasPrefix(req.Spec.RepoIdentifier, "s3") { - if env, err = restic.S3CmdEnv(c.backupLocationLister, req.Namespace, req.Spec.BackupStorageLocation); err != nil { + if env, err = restic.S3CmdEnv(c.kbClient, req.Namespace, req.Spec.BackupStorageLocation); err != nil { return c.fail(req, errors.Wrap(err, "error setting restic cmd env").Error(), log) } resticCmd.Env = env diff --git a/pkg/controller/pod_volume_restore_controller.go b/pkg/controller/pod_volume_restore_controller.go index 37f5b6060..a2b995e8f 100644 --- a/pkg/controller/pod_volume_restore_controller.go +++ b/pkg/controller/pod_volume_restore_controller.go @@ -36,6 +36,8 @@ import ( corev1informers "k8s.io/client-go/informers/core/v1" corev1listers "k8s.io/client-go/listers/core/v1" "k8s.io/client-go/tools/cache" + k8scache "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/client" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velerov1client "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned/typed/velero/v1" @@ -56,7 +58,8 @@ type podVolumeRestoreController struct { secretLister corev1listers.SecretLister pvcLister corev1listers.PersistentVolumeClaimLister pvLister corev1listers.PersistentVolumeLister - backupLocationLister listers.BackupStorageLocationLister + backupLocationInformer k8scache.Informer + kbClient client.Client nodeName string processRestoreFunc func(*velerov1api.PodVolumeRestore) error @@ -73,7 +76,7 @@ func NewPodVolumeRestoreController( secretInformer cache.SharedIndexInformer, pvcInformer corev1informers.PersistentVolumeClaimInformer, pvInformer corev1informers.PersistentVolumeInformer, - backupLocationInformer informers.BackupStorageLocationInformer, + kbClient client.Client, nodeName string, ) Interface { c := &podVolumeRestoreController{ @@ -84,7 +87,7 @@ func NewPodVolumeRestoreController( secretLister: corev1listers.NewSecretLister(secretInformer.GetIndexer()), pvcLister: pvcInformer.Lister(), pvLister: pvInformer.Lister(), - backupLocationLister: backupLocationInformer.Lister(), + kbClient: kbClient, nodeName: nodeName, fileSystem: filesystem.NewFileSystem(), @@ -98,7 +101,6 @@ func NewPodVolumeRestoreController( podInformer.HasSynced, secretInformer.HasSynced, pvcInformer.Informer().HasSynced, - backupLocationInformer.Informer().HasSynced, ) c.processRestoreFunc = c.processRestore @@ -294,10 +296,11 @@ func (c *podVolumeRestoreController) processRestore(req *velerov1api.PodVolumeRe defer os.Remove(credsFile) // if there's a caCert on the ObjectStorage, write it to disk so that it can be passed to restic - caCert, err := restic.GetCACert(c.backupLocationLister, req.Namespace, req.Spec.BackupStorageLocation) + caCert, err := restic.GetCACert(c.kbClient, req.Namespace, req.Spec.BackupStorageLocation) if err != nil { log.WithError(err).Error("Error getting caCert") } + var caCertFile string if caCert != nil { caCertFile, err = restic.TempCACertFile(caCert, req.Spec.BackupStorageLocation, c.fileSystem) @@ -347,13 +350,13 @@ func (c *podVolumeRestoreController) restorePodVolume(req *velerov1api.PodVolume // Running restic command might need additional provider specific environment variables. Based on the provider, we // set resticCmd.Env appropriately (currently for Azure and S3 based backuplocations) if strings.HasPrefix(req.Spec.RepoIdentifier, "azure") { - env, err := restic.AzureCmdEnv(c.backupLocationLister, req.Namespace, req.Spec.BackupStorageLocation) + env, err := restic.AzureCmdEnv(c.kbClient, req.Namespace, req.Spec.BackupStorageLocation) if err != nil { return c.failRestore(req, errors.Wrap(err, "error setting restic cmd env").Error(), log) } resticCmd.Env = env } else if strings.HasPrefix(req.Spec.RepoIdentifier, "s3") { - env, err := restic.S3CmdEnv(c.backupLocationLister, req.Namespace, req.Spec.BackupStorageLocation) + env, err := restic.S3CmdEnv(c.kbClient, req.Namespace, req.Spec.BackupStorageLocation) if err != nil { return c.failRestore(req, errors.Wrap(err, "error setting restic cmd env").Error(), log) } diff --git a/pkg/controller/restic_repository_controller.go b/pkg/controller/restic_repository_controller.go index 871077036..3baf1df57 100644 --- a/pkg/controller/restic_repository_controller.go +++ b/pkg/controller/restic_repository_controller.go @@ -17,6 +17,7 @@ limitations under the License. package controller import ( + "context" "encoding/json" "strings" "time" @@ -31,11 +32,13 @@ import ( "k8s.io/apimachinery/pkg/util/clock" "k8s.io/client-go/tools/cache" - v1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velerov1client "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned/typed/velero/v1" velerov1informers "github.com/vmware-tanzu/velero/pkg/generated/informers/externalversions/velero/v1" velerov1listers "github.com/vmware-tanzu/velero/pkg/generated/listers/velero/v1" "github.com/vmware-tanzu/velero/pkg/restic" + + "sigs.k8s.io/controller-runtime/pkg/client" ) type resticRepositoryController struct { @@ -43,7 +46,7 @@ type resticRepositoryController struct { resticRepositoryClient velerov1client.ResticRepositoriesGetter resticRepositoryLister velerov1listers.ResticRepositoryLister - backupLocationLister velerov1listers.BackupStorageLocationLister + kbClient client.Client repositoryManager restic.RepositoryManager defaultMaintenanceFrequency time.Duration @@ -55,7 +58,7 @@ func NewResticRepositoryController( logger logrus.FieldLogger, resticRepositoryInformer velerov1informers.ResticRepositoryInformer, resticRepositoryClient velerov1client.ResticRepositoriesGetter, - backupLocationLister velerov1listers.BackupStorageLocationLister, + kbClient client.Client, repositoryManager restic.RepositoryManager, defaultMaintenanceFrequency time.Duration, ) Interface { @@ -63,7 +66,7 @@ func NewResticRepositoryController( genericController: newGenericController("restic-repository", logger), resticRepositoryClient: resticRepositoryClient, resticRepositoryLister: resticRepositoryInformer.Lister(), - backupLocationLister: backupLocationLister, + kbClient: kbClient, repositoryManager: repositoryManager, defaultMaintenanceFrequency: defaultMaintenanceFrequency, @@ -129,7 +132,7 @@ func (c *resticRepositoryController) processQueueItem(key string) error { // Don't mutate the shared cache reqCopy := req.DeepCopy() - if req.Status.Phase == "" || req.Status.Phase == v1.ResticRepositoryPhaseNew { + if req.Status.Phase == "" || req.Status.Phase == velerov1api.ResticRepositoryPhaseNew { return c.initializeRepo(reqCopy, log) } @@ -142,29 +145,32 @@ func (c *resticRepositoryController) processQueueItem(key string) error { } switch req.Status.Phase { - case v1.ResticRepositoryPhaseReady: + case velerov1api.ResticRepositoryPhaseReady: return c.runMaintenanceIfDue(reqCopy, log) - case v1.ResticRepositoryPhaseNotReady: + case velerov1api.ResticRepositoryPhaseNotReady: return c.checkNotReadyRepo(reqCopy, log) } return nil } -func (c *resticRepositoryController) initializeRepo(req *v1.ResticRepository, log logrus.FieldLogger) error { +func (c *resticRepositoryController) initializeRepo(req *velerov1api.ResticRepository, log logrus.FieldLogger) error { log.Info("Initializing restic repository") // confirm the repo's BackupStorageLocation is valid - loc, err := c.backupLocationLister.BackupStorageLocations(req.Namespace).Get(req.Spec.BackupStorageLocation) - if err != nil { + loc := &velerov1api.BackupStorageLocation{} + if err := c.kbClient.Get(context.Background(), client.ObjectKey{ + Namespace: req.Namespace, + Name: req.Spec.BackupStorageLocation, + }, loc); err != nil { return c.patchResticRepository(req, repoNotReady(err.Error())) } repoIdentifier, err := restic.GetRepoIdentifier(loc, req.Spec.VolumeNamespace) if err != nil { - return c.patchResticRepository(req, func(r *v1.ResticRepository) { + return c.patchResticRepository(req, func(r *velerov1api.ResticRepository) { r.Status.Message = err.Error() - r.Status.Phase = v1.ResticRepositoryPhaseNotReady + r.Status.Phase = velerov1api.ResticRepositoryPhaseNotReady if r.Spec.MaintenanceFrequency.Duration <= 0 { r.Spec.MaintenanceFrequency = metav1.Duration{Duration: c.defaultMaintenanceFrequency} @@ -173,7 +179,7 @@ func (c *resticRepositoryController) initializeRepo(req *v1.ResticRepository, lo } // defaulting - if the patch fails, return an error so the item is returned to the queue - if err := c.patchResticRepository(req, func(r *v1.ResticRepository) { + if err := c.patchResticRepository(req, func(r *velerov1api.ResticRepository) { r.Spec.ResticIdentifier = repoIdentifier if r.Spec.MaintenanceFrequency.Duration <= 0 { @@ -187,8 +193,8 @@ func (c *resticRepositoryController) initializeRepo(req *v1.ResticRepository, lo return c.patchResticRepository(req, repoNotReady(err.Error())) } - return c.patchResticRepository(req, func(req *v1.ResticRepository) { - req.Status.Phase = v1.ResticRepositoryPhaseReady + return c.patchResticRepository(req, func(req *velerov1api.ResticRepository) { + req.Status.Phase = velerov1api.ResticRepositoryPhaseReady req.Status.LastMaintenanceTime = &metav1.Time{Time: time.Now()} }) } @@ -196,7 +202,7 @@ func (c *resticRepositoryController) initializeRepo(req *v1.ResticRepository, lo // ensureRepo checks to see if a repository exists, and attempts to initialize it if // it does not exist. An error is returned if the repository can't be connected to // or initialized. -func ensureRepo(repo *v1.ResticRepository, repoManager restic.RepositoryManager) error { +func ensureRepo(repo *velerov1api.ResticRepository, repoManager restic.RepositoryManager) error { if err := repoManager.ConnectToRepo(repo); err != nil { // If the repository has not yet been initialized, the error message will always include // the following string. This is the only scenario where we should try to initialize it. @@ -212,7 +218,7 @@ func ensureRepo(repo *v1.ResticRepository, repoManager restic.RepositoryManager) return nil } -func (c *resticRepositoryController) runMaintenanceIfDue(req *v1.ResticRepository, log logrus.FieldLogger) error { +func (c *resticRepositoryController) runMaintenanceIfDue(req *velerov1api.ResticRepository, log logrus.FieldLogger) error { log.Debug("resticRepositoryController.runMaintenanceIfDue") now := c.clock.Now() @@ -229,23 +235,23 @@ func (c *resticRepositoryController) runMaintenanceIfDue(req *v1.ResticRepositor log.Debug("Pruning repo") if err := c.repositoryManager.PruneRepo(req); err != nil { log.WithError(err).Warn("error pruning repository") - if patchErr := c.patchResticRepository(req, func(r *v1.ResticRepository) { + if patchErr := c.patchResticRepository(req, func(r *velerov1api.ResticRepository) { r.Status.Message = err.Error() }); patchErr != nil { return patchErr } } - return c.patchResticRepository(req, func(req *v1.ResticRepository) { + return c.patchResticRepository(req, func(req *velerov1api.ResticRepository) { req.Status.LastMaintenanceTime = &metav1.Time{Time: now} }) } -func dueForMaintenance(req *v1.ResticRepository, now time.Time) bool { +func dueForMaintenance(req *velerov1api.ResticRepository, now time.Time) bool { return req.Status.LastMaintenanceTime == nil || req.Status.LastMaintenanceTime.Add(req.Spec.MaintenanceFrequency.Duration).Before(now) } -func (c *resticRepositoryController) checkNotReadyRepo(req *v1.ResticRepository, log logrus.FieldLogger) error { +func (c *resticRepositoryController) checkNotReadyRepo(req *velerov1api.ResticRepository, log logrus.FieldLogger) error { // no identifier: can't possibly be ready, so just return if req.Spec.ResticIdentifier == "" { return nil @@ -262,16 +268,16 @@ func (c *resticRepositoryController) checkNotReadyRepo(req *v1.ResticRepository, return c.patchResticRepository(req, repoReady()) } -func repoNotReady(msg string) func(*v1.ResticRepository) { - return func(r *v1.ResticRepository) { - r.Status.Phase = v1.ResticRepositoryPhaseNotReady +func repoNotReady(msg string) func(*velerov1api.ResticRepository) { + return func(r *velerov1api.ResticRepository) { + r.Status.Phase = velerov1api.ResticRepositoryPhaseNotReady r.Status.Message = msg } } -func repoReady() func(*v1.ResticRepository) { - return func(r *v1.ResticRepository) { - r.Status.Phase = v1.ResticRepositoryPhaseReady +func repoReady() func(*velerov1api.ResticRepository) { + return func(r *velerov1api.ResticRepository) { + r.Status.Phase = velerov1api.ResticRepositoryPhaseReady r.Status.Message = "" } } @@ -279,7 +285,7 @@ func repoReady() func(*v1.ResticRepository) { // patchResticRepository mutates req with the provided mutate function, and patches it // through the Kube API. After executing this function, req will be updated with both // the mutation and the results of the Patch() API call. -func (c *resticRepositoryController) patchResticRepository(req *v1.ResticRepository, mutate func(*v1.ResticRepository)) error { +func (c *resticRepositoryController) patchResticRepository(req *velerov1api.ResticRepository, mutate func(*velerov1api.ResticRepository)) error { // Record original json oldData, err := json.Marshal(req) if err != nil { @@ -305,7 +311,7 @@ func (c *resticRepositoryController) patchResticRepository(req *v1.ResticReposit } // patch, and if successful, update req - var patched *v1.ResticRepository + var patched *velerov1api.ResticRepository if patched, err = c.resticRepositoryClient.ResticRepositories(req.Namespace).Patch(req.Name, types.MergePatchType, patchBytes); err != nil { return errors.Wrap(err, "error patching ResticRepository") } diff --git a/pkg/controller/restore_controller.go b/pkg/controller/restore_controller.go index 343af150f..756a49eec 100644 --- a/pkg/controller/restore_controller.go +++ b/pkg/controller/restore_controller.go @@ -19,6 +19,7 @@ package controller import ( "bytes" "compress/gzip" + "context" "encoding/json" "fmt" "io" @@ -48,6 +49,8 @@ import ( "github.com/vmware-tanzu/velero/pkg/util/collections" kubeutil "github.com/vmware-tanzu/velero/pkg/util/kube" "github.com/vmware-tanzu/velero/pkg/util/logging" + + "sigs.k8s.io/controller-runtime/pkg/client" ) // nonRestorableResources is a blacklist for the restoration process. Any resources @@ -80,7 +83,7 @@ type restoreController struct { restorer pkgrestore.Restorer backupLister velerov1listers.BackupLister restoreLister velerov1listers.RestoreLister - backupLocationLister velerov1listers.BackupStorageLocationLister + kbClient client.Client snapshotLocationLister velerov1listers.VolumeSnapshotLocationLister restoreLogLevel logrus.Level defaultBackupLocation string @@ -88,7 +91,7 @@ type restoreController struct { logFormat logging.Format newPluginManager func(logger logrus.FieldLogger) clientmgmt.Manager - newBackupStore func(*api.BackupStorageLocation, persistence.ObjectStoreGetter, logrus.FieldLogger) (persistence.BackupStore, error) + newBackupStore func(*velerov1api.BackupStorageLocation, persistence.ObjectStoreGetter, logrus.FieldLogger) (persistence.BackupStore, error) } func NewRestoreController( @@ -98,7 +101,7 @@ func NewRestoreController( podVolumeBackupClient velerov1client.PodVolumeBackupsGetter, restorer pkgrestore.Restorer, backupLister velerov1listers.BackupLister, - backupLocationLister velerov1listers.BackupStorageLocationLister, + kbClient client.Client, snapshotLocationLister velerov1listers.VolumeSnapshotLocationLister, logger logrus.FieldLogger, restoreLogLevel logrus.Level, @@ -115,7 +118,7 @@ func NewRestoreController( restorer: restorer, backupLister: backupLister, restoreLister: restoreInformer.Lister(), - backupLocationLister: backupLocationLister, + kbClient: kbClient, snapshotLocationLister: snapshotLocationLister, restoreLogLevel: restoreLogLevel, defaultBackupLocation: defaultBackupLocation, @@ -274,7 +277,7 @@ func (c *restoreController) processRestore(restore *api.Restore) error { type backupInfo struct { backup *api.Backup - location *api.BackupStorageLocation + location *velerov1api.BackupStorageLocation backupStore persistence.BackupStore } @@ -396,8 +399,11 @@ func (c *restoreController) fetchBackupInfo(backupName string, pluginManager cli return backupInfo{}, err } - location, err := c.backupLocationLister.BackupStorageLocations(c.namespace).Get(backup.Spec.StorageLocation) - if err != nil { + location := &velerov1api.BackupStorageLocation{} + if err := c.kbClient.Get(context.Background(), client.ObjectKey{ + Namespace: c.namespace, + Name: backup.Spec.StorageLocation, + }, location); err != nil { return backupInfo{}, errors.WithStack(err) } diff --git a/pkg/controller/restore_controller_test.go b/pkg/controller/restore_controller_test.go index ae414872a..2a3ebd043 100644 --- a/pkg/controller/restore_controller_test.go +++ b/pkg/controller/restore_controller_test.go @@ -18,6 +18,7 @@ package controller import ( "bytes" + "context" "encoding/json" "io/ioutil" "testing" @@ -34,7 +35,7 @@ import ( core "k8s.io/client-go/testing" "k8s.io/client-go/tools/cache" - api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/builder" "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned/fake" informers "github.com/vmware-tanzu/velero/pkg/generated/informers/externalversions" @@ -52,29 +53,30 @@ import ( ) func TestFetchBackupInfo(t *testing.T) { + tests := []struct { name string backupName string - informerLocations []*api.BackupStorageLocation - informerBackups []*api.Backup - backupStoreBackup *api.Backup + informerLocations []*velerov1api.BackupStorageLocation + informerBackups []*velerov1api.Backup + backupStoreBackup *velerov1api.Backup backupStoreError error - expectedRes *api.Backup + expectedRes *velerov1api.Backup expectedErr bool }{ { name: "lister has backup", backupName: "backup-1", - informerLocations: []*api.BackupStorageLocation{builder.ForBackupStorageLocation("velero", "default").Provider("myCloud").Bucket("bucket").Result()}, - informerBackups: []*api.Backup{defaultBackup().StorageLocation("default").Result()}, + informerLocations: []*velerov1api.BackupStorageLocation{builder.ForBackupStorageLocation("velero", "default").Provider("myCloud").Bucket("bucket").Result()}, + informerBackups: []*velerov1api.Backup{defaultBackup().StorageLocation("default").Result()}, expectedRes: defaultBackup().StorageLocation("default").Result(), }, { name: "lister does not have a backup, but backupSvc does", backupName: "backup-1", backupStoreBackup: defaultBackup().StorageLocation("default").Result(), - informerLocations: []*api.BackupStorageLocation{builder.ForBackupStorageLocation("velero", "default").Provider("myCloud").Bucket("bucket").Result()}, - informerBackups: []*api.Backup{defaultBackup().StorageLocation("default").Result()}, + informerLocations: []*velerov1api.BackupStorageLocation{builder.ForBackupStorageLocation("velero", "default").Provider("myCloud").Bucket("bucket").Result()}, + informerBackups: []*velerov1api.Backup{defaultBackup().StorageLocation("default").Result()}, expectedRes: defaultBackup().StorageLocation("default").Result(), }, { @@ -91,6 +93,7 @@ func TestFetchBackupInfo(t *testing.T) { t.Run(test.name, func(t *testing.T) { var ( client = fake.NewSimpleClientset() + fakeClient = newFakeClient(t) restorer = &fakeRestorer{} sharedInformers = informers.NewSharedInformerFactory(client, 0) logger = velerotest.NewLogger() @@ -102,13 +105,13 @@ func TestFetchBackupInfo(t *testing.T) { defer backupStore.AssertExpectations(t) c := NewRestoreController( - api.DefaultNamespace, + velerov1api.DefaultNamespace, sharedInformers.Velero().V1().Restores(), client.VeleroV1(), client.VeleroV1(), restorer, sharedInformers.Velero().V1().Backups().Lister(), - sharedInformers.Velero().V1().BackupStorageLocations().Lister(), + fakeClient, sharedInformers.Velero().V1().VolumeSnapshotLocations().Lister(), logger, logrus.InfoLevel, @@ -118,13 +121,13 @@ func TestFetchBackupInfo(t *testing.T) { formatFlag, ).(*restoreController) - c.newBackupStore = func(*api.BackupStorageLocation, persistence.ObjectStoreGetter, logrus.FieldLogger) (persistence.BackupStore, error) { + c.newBackupStore = func(*velerov1api.BackupStorageLocation, persistence.ObjectStoreGetter, logrus.FieldLogger) (persistence.BackupStore, error) { return backupStore, nil } if test.backupStoreError == nil { for _, itm := range test.informerLocations { - sharedInformers.Velero().V1().BackupStorageLocations().Informer().GetStore().Add(itm) + require.NoError(t, fakeClient.Create(context.Background(), itm)) } for _, itm := range test.informerBackups { @@ -157,7 +160,7 @@ func TestProcessQueueItemSkips(t *testing.T) { tests := []struct { name string restoreKey string - restore *api.Restore + restore *velerov1api.Restore expectError bool }{ { @@ -172,17 +175,17 @@ func TestProcessQueueItemSkips(t *testing.T) { { name: "restore with phase InProgress does not get processed", restoreKey: "foo/bar", - restore: builder.ForRestore("foo", "bar").Phase(api.RestorePhaseInProgress).Result(), + restore: builder.ForRestore("foo", "bar").Phase(velerov1api.RestorePhaseInProgress).Result(), }, { name: "restore with phase Completed does not get processed", restoreKey: "foo/bar", - restore: builder.ForRestore("foo", "bar").Phase(api.RestorePhaseCompleted).Result(), + restore: builder.ForRestore("foo", "bar").Phase(velerov1api.RestorePhaseCompleted).Result(), }, { name: "restore with phase FailedValidation does not get processed", restoreKey: "foo/bar", - restore: builder.ForRestore("foo", "bar").Phase(api.RestorePhaseFailedValidation).Result(), + restore: builder.ForRestore("foo", "bar").Phase(velerov1api.RestorePhaseFailedValidation).Result(), }, } @@ -198,13 +201,13 @@ func TestProcessQueueItemSkips(t *testing.T) { ) c := NewRestoreController( - api.DefaultNamespace, + velerov1api.DefaultNamespace, sharedInformers.Velero().V1().Restores(), client.VeleroV1(), client.VeleroV1(), restorer, sharedInformers.Velero().V1().Backups().Lister(), - sharedInformers.Velero().V1().BackupStorageLocations().Lister(), + nil, sharedInformers.Velero().V1().VolumeSnapshotLocations().Lister(), logger, logrus.InfoLevel, @@ -226,20 +229,21 @@ func TestProcessQueueItemSkips(t *testing.T) { } func TestProcessQueueItem(t *testing.T) { + defaultStorageLocation := builder.ForBackupStorageLocation("velero", "default").Provider("myCloud").Bucket("bucket").Result() tests := []struct { name string restoreKey string - location *api.BackupStorageLocation - restore *api.Restore - backup *api.Backup + location *velerov1api.BackupStorageLocation + restore *velerov1api.Restore + backup *velerov1api.Backup restorerError error expectedErr bool expectedPhase string expectedValidationErrors []string expectedRestoreErrors int - expectedRestorerCall *api.Restore + expectedRestorerCall *velerov1api.Restore backupStoreGetBackupMetadataErr error backupStoreGetBackupContentsErr error putRestoreLogErr error @@ -248,80 +252,80 @@ func TestProcessQueueItem(t *testing.T) { { name: "restore with both namespace in both includedNamespaces and excludedNamespaces fails validation", location: defaultStorageLocation, - restore: NewRestore("foo", "bar", "backup-1", "another-1", "*", api.RestorePhaseNew).ExcludedNamespaces("another-1").Result(), + restore: NewRestore("foo", "bar", "backup-1", "another-1", "*", velerov1api.RestorePhaseNew).ExcludedNamespaces("another-1").Result(), backup: defaultBackup().StorageLocation("default").Result(), expectedErr: false, - expectedPhase: string(api.RestorePhaseFailedValidation), + expectedPhase: string(velerov1api.RestorePhaseFailedValidation), expectedValidationErrors: []string{"Invalid included/excluded namespace lists: excludes list cannot contain an item in the includes list: another-1"}, }, { name: "restore with resource in both includedResources and excludedResources fails validation", location: defaultStorageLocation, - restore: NewRestore("foo", "bar", "backup-1", "*", "a-resource", api.RestorePhaseNew).ExcludedResources("a-resource").Result(), + restore: NewRestore("foo", "bar", "backup-1", "*", "a-resource", velerov1api.RestorePhaseNew).ExcludedResources("a-resource").Result(), backup: defaultBackup().StorageLocation("default").Result(), expectedErr: false, - expectedPhase: string(api.RestorePhaseFailedValidation), + expectedPhase: string(velerov1api.RestorePhaseFailedValidation), expectedValidationErrors: []string{"Invalid included/excluded resource lists: excludes list cannot contain an item in the includes list: a-resource"}, }, { name: "new restore with empty backup and schedule names fails validation", - restore: NewRestore("foo", "bar", "", "ns-1", "", api.RestorePhaseNew).Result(), + restore: NewRestore("foo", "bar", "", "ns-1", "", velerov1api.RestorePhaseNew).Result(), expectedErr: false, - expectedPhase: string(api.RestorePhaseFailedValidation), + expectedPhase: string(velerov1api.RestorePhaseFailedValidation), expectedValidationErrors: []string{"Either a backup or schedule must be specified as a source for the restore, but not both"}, }, { name: "new restore with backup and schedule names provided fails validation", - restore: NewRestore("foo", "bar", "backup-1", "ns-1", "", api.RestorePhaseNew).Schedule("sched-1").Result(), + restore: NewRestore("foo", "bar", "backup-1", "ns-1", "", velerov1api.RestorePhaseNew).Schedule("sched-1").Result(), expectedErr: false, - expectedPhase: string(api.RestorePhaseFailedValidation), + expectedPhase: string(velerov1api.RestorePhaseFailedValidation), expectedValidationErrors: []string{"Either a backup or schedule must be specified as a source for the restore, but not both"}, }, { name: "valid restore with schedule name gets executed", location: defaultStorageLocation, - restore: NewRestore("foo", "bar", "", "ns-1", "", api.RestorePhaseNew).Schedule("sched-1").Result(), - backup: defaultBackup().StorageLocation("default").ObjectMeta(builder.WithLabels(api.ScheduleNameLabel, "sched-1")).Phase(api.BackupPhaseCompleted).Result(), + restore: NewRestore("foo", "bar", "", "ns-1", "", velerov1api.RestorePhaseNew).Schedule("sched-1").Result(), + backup: defaultBackup().StorageLocation("default").ObjectMeta(builder.WithLabels(velerov1api.ScheduleNameLabel, "sched-1")).Phase(velerov1api.BackupPhaseCompleted).Result(), expectedErr: false, - expectedPhase: string(api.RestorePhaseInProgress), - expectedRestorerCall: NewRestore("foo", "bar", "backup-1", "ns-1", "", api.RestorePhaseInProgress).Schedule("sched-1").Result(), + expectedPhase: string(velerov1api.RestorePhaseInProgress), + expectedRestorerCall: NewRestore("foo", "bar", "backup-1", "ns-1", "", velerov1api.RestorePhaseInProgress).Schedule("sched-1").Result(), }, { name: "restore with non-existent backup name fails", - restore: NewRestore("foo", "bar", "backup-1", "ns-1", "*", api.RestorePhaseNew).Result(), + restore: NewRestore("foo", "bar", "backup-1", "ns-1", "*", velerov1api.RestorePhaseNew).Result(), expectedErr: false, - expectedPhase: string(api.RestorePhaseFailedValidation), + expectedPhase: string(velerov1api.RestorePhaseFailedValidation), expectedValidationErrors: []string{"Error retrieving backup: backup.velero.io \"backup-1\" not found"}, backupStoreGetBackupMetadataErr: errors.New("no backup here"), }, { name: "restorer throwing an error causes the restore to fail", location: defaultStorageLocation, - restore: NewRestore("foo", "bar", "backup-1", "ns-1", "", api.RestorePhaseNew).Result(), + restore: NewRestore("foo", "bar", "backup-1", "ns-1", "", velerov1api.RestorePhaseNew).Result(), backup: defaultBackup().StorageLocation("default").Result(), restorerError: errors.New("blarg"), expectedErr: false, - expectedPhase: string(api.RestorePhaseInProgress), - expectedFinalPhase: string(api.RestorePhasePartiallyFailed), + expectedPhase: string(velerov1api.RestorePhaseInProgress), + expectedFinalPhase: string(velerov1api.RestorePhasePartiallyFailed), expectedRestoreErrors: 1, - expectedRestorerCall: NewRestore("foo", "bar", "backup-1", "ns-1", "", api.RestorePhaseInProgress).Result(), + expectedRestorerCall: NewRestore("foo", "bar", "backup-1", "ns-1", "", velerov1api.RestorePhaseInProgress).Result(), }, { name: "valid restore gets executed", location: defaultStorageLocation, - restore: NewRestore("foo", "bar", "backup-1", "ns-1", "", api.RestorePhaseNew).Result(), + restore: NewRestore("foo", "bar", "backup-1", "ns-1", "", velerov1api.RestorePhaseNew).Result(), backup: defaultBackup().StorageLocation("default").Result(), expectedErr: false, - expectedPhase: string(api.RestorePhaseInProgress), - expectedRestorerCall: NewRestore("foo", "bar", "backup-1", "ns-1", "", api.RestorePhaseInProgress).Result(), + expectedPhase: string(velerov1api.RestorePhaseInProgress), + expectedRestorerCall: NewRestore("foo", "bar", "backup-1", "ns-1", "", velerov1api.RestorePhaseInProgress).Result(), }, { name: "restoration of nodes is not supported", location: defaultStorageLocation, - restore: NewRestore("foo", "bar", "backup-1", "ns-1", "nodes", api.RestorePhaseNew).Result(), + restore: NewRestore("foo", "bar", "backup-1", "ns-1", "nodes", velerov1api.RestorePhaseNew).Result(), backup: defaultBackup().StorageLocation("default").Result(), expectedErr: false, - expectedPhase: string(api.RestorePhaseFailedValidation), + expectedPhase: string(velerov1api.RestorePhaseFailedValidation), expectedValidationErrors: []string{ "nodes are non-restorable resources", "Invalid included/excluded resource lists: excludes list cannot contain an item in the includes list: nodes", @@ -330,10 +334,10 @@ func TestProcessQueueItem(t *testing.T) { { name: "restoration of events is not supported", location: defaultStorageLocation, - restore: NewRestore("foo", "bar", "backup-1", "ns-1", "events", api.RestorePhaseNew).Result(), + restore: NewRestore("foo", "bar", "backup-1", "ns-1", "events", velerov1api.RestorePhaseNew).Result(), backup: defaultBackup().StorageLocation("default").Result(), expectedErr: false, - expectedPhase: string(api.RestorePhaseFailedValidation), + expectedPhase: string(velerov1api.RestorePhaseFailedValidation), expectedValidationErrors: []string{ "events are non-restorable resources", "Invalid included/excluded resource lists: excludes list cannot contain an item in the includes list: events", @@ -342,10 +346,10 @@ func TestProcessQueueItem(t *testing.T) { { name: "restoration of events.events.k8s.io is not supported", location: defaultStorageLocation, - restore: NewRestore("foo", "bar", "backup-1", "ns-1", "events.events.k8s.io", api.RestorePhaseNew).Result(), + restore: NewRestore("foo", "bar", "backup-1", "ns-1", "events.events.k8s.io", velerov1api.RestorePhaseNew).Result(), backup: defaultBackup().StorageLocation("default").Result(), expectedErr: false, - expectedPhase: string(api.RestorePhaseFailedValidation), + expectedPhase: string(velerov1api.RestorePhaseFailedValidation), expectedValidationErrors: []string{ "events.events.k8s.io are non-restorable resources", "Invalid included/excluded resource lists: excludes list cannot contain an item in the includes list: events.events.k8s.io", @@ -354,10 +358,10 @@ func TestProcessQueueItem(t *testing.T) { { name: "restoration of backups.velero.io is not supported", location: defaultStorageLocation, - restore: NewRestore("foo", "bar", "backup-1", "ns-1", "backups.velero.io", api.RestorePhaseNew).Result(), + restore: NewRestore("foo", "bar", "backup-1", "ns-1", "backups.velero.io", velerov1api.RestorePhaseNew).Result(), backup: defaultBackup().StorageLocation("default").Result(), expectedErr: false, - expectedPhase: string(api.RestorePhaseFailedValidation), + expectedPhase: string(velerov1api.RestorePhaseFailedValidation), expectedValidationErrors: []string{ "backups.velero.io are non-restorable resources", "Invalid included/excluded resource lists: excludes list cannot contain an item in the includes list: backups.velero.io", @@ -366,10 +370,10 @@ func TestProcessQueueItem(t *testing.T) { { name: "restoration of restores.velero.io is not supported", location: defaultStorageLocation, - restore: NewRestore("foo", "bar", "backup-1", "ns-1", "restores.velero.io", api.RestorePhaseNew).Result(), + restore: NewRestore("foo", "bar", "backup-1", "ns-1", "restores.velero.io", velerov1api.RestorePhaseNew).Result(), backup: defaultBackup().StorageLocation("default").Result(), expectedErr: false, - expectedPhase: string(api.RestorePhaseFailedValidation), + expectedPhase: string(velerov1api.RestorePhaseFailedValidation), expectedValidationErrors: []string{ "restores.velero.io are non-restorable resources", "Invalid included/excluded resource lists: excludes list cannot contain an item in the includes list: restores.velero.io", @@ -378,9 +382,9 @@ func TestProcessQueueItem(t *testing.T) { { name: "backup download error results in failed restore", location: defaultStorageLocation, - restore: NewRestore(api.DefaultNamespace, "bar", "backup-1", "ns-1", "", api.RestorePhaseNew).Result(), - expectedPhase: string(api.RestorePhaseInProgress), - expectedFinalPhase: string(api.RestorePhaseFailed), + restore: NewRestore(velerov1api.DefaultNamespace, "bar", "backup-1", "ns-1", "", velerov1api.RestorePhaseNew).Result(), + expectedPhase: string(velerov1api.RestorePhaseInProgress), + expectedFinalPhase: string(velerov1api.RestorePhaseFailed), backupStoreGetBackupContentsErr: errors.New("Couldn't download backup"), backup: defaultBackup().StorageLocation("default").Result(), }, @@ -392,6 +396,7 @@ func TestProcessQueueItem(t *testing.T) { t.Run(test.name, func(t *testing.T) { var ( client = fake.NewSimpleClientset() + fakeClient = newFakeClient(t) restorer = &fakeRestorer{} sharedInformers = informers.NewSharedInformerFactory(client, 0) logger = velerotest.NewLogger() @@ -403,13 +408,13 @@ func TestProcessQueueItem(t *testing.T) { defer backupStore.AssertExpectations(t) c := NewRestoreController( - api.DefaultNamespace, + velerov1api.DefaultNamespace, sharedInformers.Velero().V1().Restores(), client.VeleroV1(), client.VeleroV1(), restorer, sharedInformers.Velero().V1().Backups().Lister(), - sharedInformers.Velero().V1().BackupStorageLocations().Lister(), + fakeClient, sharedInformers.Velero().V1().VolumeSnapshotLocations().Lister(), logger, logrus.InfoLevel, @@ -419,12 +424,12 @@ func TestProcessQueueItem(t *testing.T) { formatFlag, ).(*restoreController) - c.newBackupStore = func(*api.BackupStorageLocation, persistence.ObjectStoreGetter, logrus.FieldLogger) (persistence.BackupStore, error) { + c.newBackupStore = func(*velerov1api.BackupStorageLocation, persistence.ObjectStoreGetter, logrus.FieldLogger) (persistence.BackupStore, error) { return backupStore, nil } if test.location != nil { - sharedInformers.Velero().V1().BackupStorageLocations().Informer().GetStore().Add(test.location) + require.NoError(t, fakeClient.Create(context.Background(), test.location)) } if test.backup != nil { sharedInformers.Velero().V1().Backups().Informer().GetStore().Add(test.backup) @@ -462,7 +467,7 @@ func TestProcessQueueItem(t *testing.T) { // these are the fields that we expect to be set by // the controller - res.Status.Phase = api.RestorePhase(phase) + res.Status.Phase = velerov1api.RestorePhase(phase) backupName, found, err := unstructured.NestedString(patchMap, "spec", "backupName") if found { @@ -546,9 +551,9 @@ func TestProcessQueueItem(t *testing.T) { } type StatusPatch struct { - Phase api.RestorePhase `json:"phase"` - ValidationErrors []string `json:"validationErrors"` - Errors int `json:"errors"` + Phase velerov1api.RestorePhase `json:"phase"` + ValidationErrors []string `json:"validationErrors"` + Errors int `json:"errors"` } type Patch struct { @@ -568,7 +573,7 @@ func TestProcessQueueItem(t *testing.T) { expected := Patch{ Status: StatusPatch{ - Phase: api.RestorePhase(test.expectedPhase), + Phase: velerov1api.RestorePhase(test.expectedPhase), ValidationErrors: test.expectedValidationErrors, }, } @@ -593,7 +598,7 @@ func TestProcessQueueItem(t *testing.T) { expected = Patch{ Status: StatusPatch{ - Phase: api.RestorePhaseCompleted, + Phase: velerov1api.RestorePhaseCompleted, Errors: test.expectedRestoreErrors, }, } @@ -601,7 +606,7 @@ func TestProcessQueueItem(t *testing.T) { if test.expectedFinalPhase != "" { expected = Patch{ Status: StatusPatch{ - Phase: api.RestorePhase(test.expectedFinalPhase), + Phase: velerov1api.RestorePhase(test.expectedFinalPhase), Errors: test.expectedRestoreErrors, }, } @@ -628,13 +633,13 @@ func TestvalidateAndCompleteWhenScheduleNameSpecified(t *testing.T) { ) c := NewRestoreController( - api.DefaultNamespace, + velerov1api.DefaultNamespace, sharedInformers.Velero().V1().Restores(), client.VeleroV1(), client.VeleroV1(), nil, sharedInformers.Velero().V1().Backups().Lister(), - sharedInformers.Velero().V1().BackupStorageLocations().Lister(), + nil, sharedInformers.Velero().V1().VolumeSnapshotLocations().Lister(), logger, logrus.DebugLevel, @@ -644,12 +649,12 @@ func TestvalidateAndCompleteWhenScheduleNameSpecified(t *testing.T) { formatFlag, ).(*restoreController) - restore := &api.Restore{ + restore := &velerov1api.Restore{ ObjectMeta: metav1.ObjectMeta{ - Namespace: api.DefaultNamespace, + Namespace: velerov1api.DefaultNamespace, Name: "restore-1", }, - Spec: api.RestoreSpec{ + Spec: velerov1api.RestoreSpec{ ScheduleName: "schedule-1", }, } @@ -657,8 +662,8 @@ func TestvalidateAndCompleteWhenScheduleNameSpecified(t *testing.T) { // no backups created from the schedule: fail validation require.NoError(t, sharedInformers.Velero().V1().Backups().Informer().GetStore().Add( defaultBackup(). - ObjectMeta(builder.WithLabels(api.ScheduleNameLabel, "non-matching-schedule")). - Phase(api.BackupPhaseCompleted). + ObjectMeta(builder.WithLabels(velerov1api.ScheduleNameLabel, "non-matching-schedule")). + Phase(velerov1api.BackupPhaseCompleted). Result(), )) @@ -671,9 +676,9 @@ func TestvalidateAndCompleteWhenScheduleNameSpecified(t *testing.T) { defaultBackup(). ObjectMeta( builder.WithName("backup-2"), - builder.WithLabels(api.ScheduleNameLabel, "schedule-1"), + builder.WithLabels(velerov1api.ScheduleNameLabel, "schedule-1"), ). - Phase(api.BackupPhaseInProgress). + Phase(velerov1api.BackupPhaseInProgress). Result(), )) @@ -688,9 +693,9 @@ func TestvalidateAndCompleteWhenScheduleNameSpecified(t *testing.T) { defaultBackup(). ObjectMeta( builder.WithName("foo"), - builder.WithLabels(api.ScheduleNameLabel, "schedule-1"), + builder.WithLabels(velerov1api.ScheduleNameLabel, "schedule-1"), ). - Phase(api.BackupPhaseCompleted). + Phase(velerov1api.BackupPhaseCompleted). StartTimestamp(now). Result(), )) @@ -698,9 +703,9 @@ func TestvalidateAndCompleteWhenScheduleNameSpecified(t *testing.T) { defaultBackup(). ObjectMeta( builder.WithName("foo"), - builder.WithLabels(api.ScheduleNameLabel, "schedule-1"), + builder.WithLabels(velerov1api.ScheduleNameLabel, "schedule-1"), ). - Phase(api.BackupPhaseCompleted). + Phase(velerov1api.BackupPhaseCompleted). StartTimestamp(now.Add(time.Second)). Result(), )) @@ -711,7 +716,7 @@ func TestvalidateAndCompleteWhenScheduleNameSpecified(t *testing.T) { } func TestBackupXorScheduleProvided(t *testing.T) { - r := &api.Restore{} + r := &velerov1api.Restore{} assert.False(t, backupXorScheduleProvided(r)) r.Spec.BackupName = "backup-1" @@ -728,12 +733,12 @@ func TestBackupXorScheduleProvided(t *testing.T) { } func TestMostRecentCompletedBackup(t *testing.T) { - backups := []*api.Backup{ + backups := []*velerov1api.Backup{ { ObjectMeta: metav1.ObjectMeta{ Name: "a", }, - Status: api.BackupStatus{ + Status: velerov1api.BackupStatus{ Phase: "", }, }, @@ -741,32 +746,32 @@ func TestMostRecentCompletedBackup(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "b", }, - Status: api.BackupStatus{ - Phase: api.BackupPhaseNew, + Status: velerov1api.BackupStatus{ + Phase: velerov1api.BackupPhaseNew, }, }, { ObjectMeta: metav1.ObjectMeta{ Name: "c", }, - Status: api.BackupStatus{ - Phase: api.BackupPhaseInProgress, + Status: velerov1api.BackupStatus{ + Phase: velerov1api.BackupPhaseInProgress, }, }, { ObjectMeta: metav1.ObjectMeta{ Name: "d", }, - Status: api.BackupStatus{ - Phase: api.BackupPhaseFailedValidation, + Status: velerov1api.BackupStatus{ + Phase: velerov1api.BackupPhaseFailedValidation, }, }, { ObjectMeta: metav1.ObjectMeta{ Name: "e", }, - Status: api.BackupStatus{ - Phase: api.BackupPhaseFailed, + Status: velerov1api.BackupStatus{ + Phase: velerov1api.BackupPhaseFailed, }, }, } @@ -775,22 +780,22 @@ func TestMostRecentCompletedBackup(t *testing.T) { now := time.Now() - backups = append(backups, &api.Backup{ + backups = append(backups, &velerov1api.Backup{ ObjectMeta: metav1.ObjectMeta{ Name: "foo", }, - Status: api.BackupStatus{ - Phase: api.BackupPhaseCompleted, + Status: velerov1api.BackupStatus{ + Phase: velerov1api.BackupPhaseCompleted, StartTimestamp: &metav1.Time{Time: now}, }, }) - expected := &api.Backup{ + expected := &velerov1api.Backup{ ObjectMeta: metav1.ObjectMeta{ Name: "bar", }, - Status: api.BackupStatus{ - Phase: api.BackupPhaseCompleted, + Status: velerov1api.BackupStatus{ + Phase: velerov1api.BackupPhaseCompleted, StartTimestamp: &metav1.Time{Time: now.Add(time.Second)}, }, } @@ -799,7 +804,7 @@ func TestMostRecentCompletedBackup(t *testing.T) { assert.Equal(t, expected, mostRecentCompletedBackup(backups)) } -func NewRestore(ns, name, backup, includeNS, includeResource string, phase api.RestorePhase) *builder.RestoreBuilder { +func NewRestore(ns, name, backup, includeNS, includeResource string, phase velerov1api.RestorePhase) *builder.RestoreBuilder { restore := builder.ForRestore(ns, name).Phase(phase).Backup(backup) if includeNS != "" { @@ -817,7 +822,7 @@ func NewRestore(ns, name, backup, includeNS, includeResource string, phase api.R type fakeRestorer struct { mock.Mock - calledWithArg api.Restore + calledWithArg velerov1api.Restore } func (r *fakeRestorer) Restore( diff --git a/pkg/controller/suite_test.go b/pkg/controller/suite_test.go new file mode 100644 index 000000000..8fe4412c4 --- /dev/null +++ b/pkg/controller/suite_test.go @@ -0,0 +1,37 @@ +/* +Copyright 2020 the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + + "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func newFakeClient(t *testing.T, initObjs ...runtime.Object) client.Client { + err := velerov1api.AddToScheme(scheme.Scheme) + require.NoError(t, err) + return fake.NewFakeClientWithScheme(scheme.Scheme, initObjs...) +} diff --git a/pkg/install/resources.go b/pkg/install/resources.go index b88dbb183..2c5a0d9a1 100644 --- a/pkg/install/resources.go +++ b/pkg/install/resources.go @@ -19,6 +19,8 @@ package install import ( "time" + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + corev1 "k8s.io/api/core/v1" rbacv1beta1 "k8s.io/api/rbac/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -26,9 +28,8 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" - v1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + "github.com/vmware-tanzu/velero/config/crd/crds" "github.com/vmware-tanzu/velero/pkg/buildinfo" - "github.com/vmware-tanzu/velero/pkg/generated/crds" ) // Use "latest" if the build process didn't supply a version @@ -137,17 +138,17 @@ func Namespace(namespace string) *corev1.Namespace { } } -func BackupStorageLocation(namespace, provider, bucket, prefix string, config map[string]string, caCert []byte) *v1.BackupStorageLocation { - return &v1.BackupStorageLocation{ +func BackupStorageLocation(namespace, provider, bucket, prefix string, config map[string]string, caCert []byte) *velerov1api.BackupStorageLocation { + return &velerov1api.BackupStorageLocation{ ObjectMeta: objectMeta(namespace, "default"), TypeMeta: metav1.TypeMeta{ Kind: "BackupStorageLocation", - APIVersion: v1.SchemeGroupVersion.String(), + APIVersion: velerov1api.SchemeGroupVersion.String(), }, - Spec: v1.BackupStorageLocationSpec{ + Spec: velerov1api.BackupStorageLocationSpec{ Provider: provider, - StorageType: v1.StorageType{ - ObjectStorage: &v1.ObjectStorageLocation{ + StorageType: velerov1api.StorageType{ + ObjectStorage: &velerov1api.ObjectStorageLocation{ Bucket: bucket, Prefix: prefix, CACert: caCert, @@ -158,14 +159,14 @@ func BackupStorageLocation(namespace, provider, bucket, prefix string, config ma } } -func VolumeSnapshotLocation(namespace, provider string, config map[string]string) *v1.VolumeSnapshotLocation { - return &v1.VolumeSnapshotLocation{ +func VolumeSnapshotLocation(namespace, provider string, config map[string]string) *velerov1api.VolumeSnapshotLocation { + return &velerov1api.VolumeSnapshotLocation{ ObjectMeta: objectMeta(namespace, "default"), TypeMeta: metav1.TypeMeta{ Kind: "VolumeSnapshotLocation", - APIVersion: v1.SchemeGroupVersion.String(), + APIVersion: velerov1api.SchemeGroupVersion.String(), }, - Spec: v1.VolumeSnapshotLocationSpec{ + Spec: velerov1api.VolumeSnapshotLocationSpec{ Provider: provider, Config: config, }, diff --git a/pkg/restic/common.go b/pkg/restic/common.go index ffb2c1167..fc8f20ca0 100644 --- a/pkg/restic/common.go +++ b/pkg/restic/common.go @@ -17,11 +17,14 @@ limitations under the License. package restic import ( + "context" "fmt" "os" "strings" "time" + kbclient "sigs.k8s.io/controller-runtime/pkg/client" + "github.com/pkg/errors" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -284,17 +287,20 @@ func TempCACertFile(caCert []byte, bsl string, fs filesystem.Interface) (string, return name, nil } -func GetCACert(backupLocationLister velerov1listers.BackupStorageLocationLister, namespace, bsl string) ([]byte, error) { - location, err := backupLocationLister.BackupStorageLocations(namespace).Get(bsl) - if err != nil { - return nil, errors.Wrap(err, "error getting backup storage location") +func GetCACert(client kbclient.Client, namespace, backupLocation string) ([]byte, error) { + location := &velerov1api.BackupStorageLocation{} + if err := client.Get(context.Background(), kbclient.ObjectKey{ + Namespace: namespace, + Name: backupLocation, + }, location); err != nil { + return nil, err } - if location.Spec.ObjectStorage != nil { - return location.Spec.ObjectStorage.CACert, nil + if location.Spec.ObjectStorage == nil { + return nil, nil } - return nil, nil + return location.Spec.ObjectStorage.CACert, nil } // NewPodVolumeRestoreListOptions creates a ListOptions with a label selector configured to @@ -309,10 +315,13 @@ func NewPodVolumeRestoreListOptions(name string) metav1.ListOptions { // should be used when running a restic command for an Azure backend. This list is // the current environment, plus the Azure-specific variables restic needs, namely // a storage account name and key. -func AzureCmdEnv(backupLocationLister velerov1listers.BackupStorageLocationLister, namespace, backupLocation string) ([]string, error) { - loc, err := backupLocationLister.BackupStorageLocations(namespace).Get(backupLocation) - if err != nil { - return nil, errors.Wrap(err, "error getting backup storage location") +func AzureCmdEnv(client kbclient.Client, namespace, backupLocation string) ([]string, error) { + loc := &velerov1api.BackupStorageLocation{} + if err := client.Get(context.Background(), kbclient.ObjectKey{ + Namespace: namespace, + Name: backupLocation, + }, loc); err != nil { + return nil, err } azureVars, err := getAzureResticEnvVars(loc.Spec.Config) @@ -332,10 +341,13 @@ func AzureCmdEnv(backupLocationLister velerov1listers.BackupStorageLocationListe // should be used when running a restic command for an S3 backend. This list is // the current environment, plus the AWS-specific variables restic needs, namely // a credential profile. -func S3CmdEnv(backupLocationLister velerov1listers.BackupStorageLocationLister, namespace, backupLocation string) ([]string, error) { - loc, err := backupLocationLister.BackupStorageLocations(namespace).Get(backupLocation) - if err != nil { - return nil, errors.Wrap(err, "error getting backup storage location") +func S3CmdEnv(client kbclient.Client, namespace, backupLocation string) ([]string, error) { + loc := &velerov1api.BackupStorageLocation{} + if err := client.Get(context.Background(), kbclient.ObjectKey{ + Namespace: namespace, + Name: backupLocation, + }, loc); err != nil { + return nil, err } awsVars, err := getS3ResticEnvVars(loc.Spec.Config) diff --git a/pkg/restic/common_test.go b/pkg/restic/common_test.go index c69a1a44e..b4dec4929 100644 --- a/pkg/restic/common_test.go +++ b/pkg/restic/common_test.go @@ -17,18 +17,22 @@ limitations under the License. package restic import ( + "context" "os" "sort" "testing" - velerov1listers "github.com/vmware-tanzu/velero/pkg/generated/listers/velero/v1" + "k8s.io/apimachinery/pkg/runtime" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/scheme" corev1listers "k8s.io/client-go/listers/core/v1" "k8s.io/client-go/tools/cache" + "sigs.k8s.io/controller-runtime/pkg/client" + k8sfake "sigs.k8s.io/controller-runtime/pkg/client/fake" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/builder" @@ -381,10 +385,8 @@ func TestTempCredentialsFile(t *testing.T) { func TestTempCACertFile(t *testing.T) { var ( - bslInformer = cache.NewSharedIndexInformer(nil, new(velerov1api.BackupStorageLocation), 0, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) - bslLister = velerov1listers.NewBackupStorageLocationLister(bslInformer.GetIndexer()) - fs = velerotest.NewFakeFileSystem() - bsl = &velerov1api.BackupStorageLocation{ + fs = velerotest.NewFakeFileSystem() + bsl = &velerov1api.BackupStorageLocation{ TypeMeta: metav1.TypeMeta{}, ObjectMeta: metav1.ObjectMeta{ Namespace: "velero", @@ -398,15 +400,11 @@ func TestTempCACertFile(t *testing.T) { } ) - // bsl not in lister: expect an error - caCert, err := GetCACert(bslLister, "velero", "default") - assert.Error(t, err) + fakeClient := newFakeClient(t) + fakeClient.Create(context.Background(), bsl) - // now add bsl to lister - require.NoError(t, bslInformer.GetStore().Add(bsl)) - - // bsl in lister: expect temp file to be created with cacert value - caCert, err = GetCACert(bslLister, "velero", "default") + // expect temp file to be created with cacert value + caCert, err := GetCACert(fakeClient, bsl.Namespace, bsl.Name) require.NoError(t, err) fileName, err := TempCACertFile(caCert, "default", fs) @@ -521,3 +519,9 @@ func TestGetPodVolumesUsingRestic(t *testing.T) { }) } } + +func newFakeClient(t *testing.T, initObjs ...runtime.Object) client.Client { + err := velerov1api.AddToScheme(scheme.Scheme) + require.NoError(t, err) + return k8sfake.NewFakeClientWithScheme(scheme.Scheme, initObjs...) +} diff --git a/pkg/restic/repository_manager.go b/pkg/restic/repository_manager.go index 6ae87f022..08b23ffd9 100644 --- a/pkg/restic/repository_manager.go +++ b/pkg/restic/repository_manager.go @@ -29,6 +29,8 @@ import ( corev1listers "k8s.io/client-go/listers/core/v1" "k8s.io/client-go/tools/cache" + kbclient "sigs.k8s.io/controller-runtime/pkg/client" + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" clientset "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned" velerov1client "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned/typed/velero/v1" @@ -79,20 +81,19 @@ type RestorerFactory interface { } type repositoryManager struct { - namespace string - veleroClient clientset.Interface - secretsLister corev1listers.SecretLister - repoLister velerov1listers.ResticRepositoryLister - repoInformerSynced cache.InformerSynced - backupLocationLister velerov1listers.BackupStorageLocationLister - backupLocationInformerSynced cache.InformerSynced - log logrus.FieldLogger - repoLocker *repoLocker - repoEnsurer *repositoryEnsurer - fileSystem filesystem.Interface - ctx context.Context - pvcClient corev1client.PersistentVolumeClaimsGetter - pvClient corev1client.PersistentVolumesGetter + namespace string + veleroClient clientset.Interface + secretsLister corev1listers.SecretLister + repoLister velerov1listers.ResticRepositoryLister + repoInformerSynced cache.InformerSynced + kbClient kbclient.Client + log logrus.FieldLogger + repoLocker *repoLocker + repoEnsurer *repositoryEnsurer + fileSystem filesystem.Interface + ctx context.Context + pvcClient corev1client.PersistentVolumeClaimsGetter + pvClient corev1client.PersistentVolumesGetter } // NewRepositoryManager constructs a RepositoryManager. @@ -103,23 +104,22 @@ func NewRepositoryManager( secretsInformer cache.SharedIndexInformer, repoInformer velerov1informers.ResticRepositoryInformer, repoClient velerov1client.ResticRepositoriesGetter, - backupLocationInformer velerov1informers.BackupStorageLocationInformer, + kbClient kbclient.Client, pvcClient corev1client.PersistentVolumeClaimsGetter, pvClient corev1client.PersistentVolumesGetter, log logrus.FieldLogger, ) (RepositoryManager, error) { rm := &repositoryManager{ - namespace: namespace, - veleroClient: veleroClient, - secretsLister: corev1listers.NewSecretLister(secretsInformer.GetIndexer()), - repoLister: repoInformer.Lister(), - repoInformerSynced: repoInformer.Informer().HasSynced, - backupLocationLister: backupLocationInformer.Lister(), - backupLocationInformerSynced: backupLocationInformer.Informer().HasSynced, - pvcClient: pvcClient, - pvClient: pvClient, - log: log, - ctx: ctx, + namespace: namespace, + veleroClient: veleroClient, + secretsLister: corev1listers.NewSecretLister(secretsInformer.GetIndexer()), + repoLister: repoInformer.Lister(), + repoInformerSynced: repoInformer.Informer().HasSynced, + kbClient: kbClient, + pvcClient: pvcClient, + pvClient: pvClient, + log: log, + ctx: ctx, repoLocker: newRepoLocker(), repoEnsurer: newRepositoryEnsurer(repoInformer, repoClient, log), @@ -245,10 +245,11 @@ func (rm *repositoryManager) exec(cmd *Command, backupLocation string) error { cmd.PasswordFile = file // if there's a caCert on the ObjectStorage, write it to disk so that it can be passed to restic - caCert, err := GetCACert(rm.backupLocationLister, rm.namespace, backupLocation) + caCert, err := GetCACert(rm.kbClient, rm.namespace, backupLocation) if err != nil { return err } + var caCertFile string if caCert != nil { caCertFile, err = TempCACertFile(caCert, backupLocation, rm.fileSystem) @@ -261,21 +262,13 @@ func (rm *repositoryManager) exec(cmd *Command, backupLocation string) error { cmd.CACertFile = caCertFile if strings.HasPrefix(cmd.RepoIdentifier, "azure") { - if !cache.WaitForCacheSync(rm.ctx.Done(), rm.backupLocationInformerSynced) { - return errors.New("timed out waiting for cache to sync") - } - - env, err := AzureCmdEnv(rm.backupLocationLister, rm.namespace, backupLocation) + env, err := AzureCmdEnv(rm.kbClient, rm.namespace, backupLocation) if err != nil { return err } cmd.Env = env } else if strings.HasPrefix(cmd.RepoIdentifier, "s3") { - if !cache.WaitForCacheSync(rm.ctx.Done(), rm.backupLocationInformerSynced) { - return errors.New("timed out waiting for cache to sync") - } - - env, err := S3CmdEnv(rm.backupLocationLister, rm.namespace, backupLocation) + env, err := S3CmdEnv(rm.kbClient, rm.namespace, backupLocation) if err != nil { return err } diff --git a/site/docs/master/code-standards.md b/site/docs/master/code-standards.md index cc28a0f0d..49abc2941 100644 --- a/site/docs/master/code-standards.md +++ b/site/docs/master/code-standards.md @@ -53,8 +53,8 @@ Example: metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" corev1client "k8s.io/client-go/kubernetes/typed/core/v1" corev1listers "k8s.io/client-go/listers/core/v1" - - velerov1api ""github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velerov1client "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned/typed/velero/v1" ) diff --git a/site/docs/v1.3.1/code-standards.md b/site/docs/v1.3.1/code-standards.md index bc2f09de7..586a865fa 100644 --- a/site/docs/v1.3.1/code-standards.md +++ b/site/docs/v1.3.1/code-standards.md @@ -43,8 +43,7 @@ Example: metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" corev1client "k8s.io/client-go/kubernetes/typed/core/v1" corev1listers "k8s.io/client-go/listers/core/v1" - - velerov1api ""github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velerov1client "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned/typed/velero/v1" ) diff --git a/site/docs/v1.3.2/code-standards.md b/site/docs/v1.3.2/code-standards.md index 4dbe03dbc..4323756bf 100644 --- a/site/docs/v1.3.2/code-standards.md +++ b/site/docs/v1.3.2/code-standards.md @@ -43,8 +43,8 @@ Example: metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" corev1client "k8s.io/client-go/kubernetes/typed/core/v1" corev1listers "k8s.io/client-go/listers/core/v1" - - velerov1api ""github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velerov1client "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned/typed/velero/v1" ) diff --git a/site/docs/v1.4/code-standards.md b/site/docs/v1.4/code-standards.md index e212ad98c..00a179546 100644 --- a/site/docs/v1.4/code-standards.md +++ b/site/docs/v1.4/code-standards.md @@ -51,8 +51,8 @@ Example: metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" corev1client "k8s.io/client-go/kubernetes/typed/core/v1" corev1listers "k8s.io/client-go/listers/core/v1" - - velerov1api ""github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" velerov1client "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned/typed/velero/v1" )