/* Copyright the Velero contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package install import ( "fmt" "time" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" v1crds "github.com/vmware-tanzu/velero/config/crd/v1/crds" v2alpha1crds "github.com/vmware-tanzu/velero/config/crd/v2alpha1/crds" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/util/kube" ) const ( defaultServiceAccountName = "velero" podSecurityLevel = "privileged" podSecurityVersion = "latest" ) var ( // default values for Velero server pod resource request/limit DefaultVeleroPodCPURequest = "500m" DefaultVeleroPodMemRequest = "128Mi" DefaultVeleroPodCPULimit = "1000m" DefaultVeleroPodMemLimit = "512Mi" // default values for node-agent pod resource request/limit, // "0" means no request/limit is set, so as to make the QoS as BestEffort DefaultNodeAgentPodCPURequest = "0" DefaultNodeAgentPodMemRequest = "0" DefaultNodeAgentPodCPULimit = "0" DefaultNodeAgentPodMemLimit = "0" DefaultVeleroNamespace = "velero" ) func Labels() map[string]string { return map[string]string{ "component": "velero", } } func podLabels(userLabels ...map[string]string) map[string]string { // Use the default labels as a starting point base := Labels() // Merge base labels with user labels to enforce CLI precedence for _, labels := range userLabels { for k, v := range labels { base[k] = v } } return base } func podAnnotations(userAnnotations map[string]string) map[string]string { // Use the default annotations as a starting point base := map[string]string{ "prometheus.io/scrape": "true", "prometheus.io/port": "8085", "prometheus.io/path": "/metrics", } // Merge base annotations with user annotations to enforce CLI precedence for k, v := range userAnnotations { base[k] = v } return base } func containerPorts() []corev1.ContainerPort { return []corev1.ContainerPort{ { Name: "metrics", ContainerPort: 8085, }, } } func objectMeta(namespace, name string) metav1.ObjectMeta { return metav1.ObjectMeta{ Name: name, Namespace: namespace, Labels: Labels(), } } func ServiceAccount(namespace string, annotations map[string]string) *corev1.ServiceAccount { objMeta := objectMeta(namespace, defaultServiceAccountName) objMeta.Annotations = annotations return &corev1.ServiceAccount{ ObjectMeta: objMeta, TypeMeta: metav1.TypeMeta{ Kind: "ServiceAccount", APIVersion: corev1.SchemeGroupVersion.String(), }, } } func ClusterRoleBinding(namespace string) *rbacv1.ClusterRoleBinding { crbName := "velero" if namespace != DefaultVeleroNamespace { crbName = "velero-" + namespace } crb := &rbacv1.ClusterRoleBinding{ ObjectMeta: objectMeta("", crbName), TypeMeta: metav1.TypeMeta{ Kind: "ClusterRoleBinding", APIVersion: rbacv1.SchemeGroupVersion.String(), }, Subjects: []rbacv1.Subject{ { Kind: "ServiceAccount", Namespace: namespace, Name: "velero", }, }, RoleRef: rbacv1.RoleRef{ Kind: "ClusterRole", Name: "cluster-admin", APIGroup: "rbac.authorization.k8s.io", }, } return crb } func Namespace(namespace string) *corev1.Namespace { ns := &corev1.Namespace{ ObjectMeta: objectMeta("", namespace), TypeMeta: metav1.TypeMeta{ Kind: "Namespace", APIVersion: corev1.SchemeGroupVersion.String(), }, } ns.Labels["pod-security.kubernetes.io/enforce"] = podSecurityLevel ns.Labels["pod-security.kubernetes.io/enforce-version"] = podSecurityVersion ns.Labels["pod-security.kubernetes.io/audit"] = podSecurityLevel ns.Labels["pod-security.kubernetes.io/audit-version"] = podSecurityVersion ns.Labels["pod-security.kubernetes.io/warn"] = podSecurityLevel ns.Labels["pod-security.kubernetes.io/warn-version"] = podSecurityVersion return ns } func BackupStorageLocation(namespace, provider, bucket, prefix string, config map[string]string, caCert []byte) *velerov1api.BackupStorageLocation { return &velerov1api.BackupStorageLocation{ ObjectMeta: objectMeta(namespace, "default"), TypeMeta: metav1.TypeMeta{ Kind: "BackupStorageLocation", APIVersion: velerov1api.SchemeGroupVersion.String(), }, Spec: velerov1api.BackupStorageLocationSpec{ Provider: provider, StorageType: velerov1api.StorageType{ ObjectStorage: &velerov1api.ObjectStorageLocation{ Bucket: bucket, Prefix: prefix, CACert: caCert, }, }, Config: config, Default: true, }, } } func VolumeSnapshotLocation(namespace, provider string, config map[string]string) *velerov1api.VolumeSnapshotLocation { return &velerov1api.VolumeSnapshotLocation{ ObjectMeta: objectMeta(namespace, "default"), TypeMeta: metav1.TypeMeta{ Kind: "VolumeSnapshotLocation", APIVersion: velerov1api.SchemeGroupVersion.String(), }, Spec: velerov1api.VolumeSnapshotLocationSpec{ Provider: provider, Config: config, }, } } func Secret(namespace string, data []byte) *corev1.Secret { return &corev1.Secret{ ObjectMeta: objectMeta(namespace, "cloud-credentials"), TypeMeta: metav1.TypeMeta{ Kind: "Secret", APIVersion: corev1.SchemeGroupVersion.String(), }, Data: map[string][]byte{ "cloud": data, }, Type: corev1.SecretTypeOpaque, } } func appendUnstructured(list *unstructured.UnstructuredList, obj runtime.Object) error { u, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&obj) // Remove the status field so we're not sending blank data to the server. // On CRDs, having an empty status is actually a validation error. delete(u, "status") if err != nil { return err } list.Items = append(list.Items, unstructured.Unstructured{Object: u}) return nil } type VeleroOptions struct { Namespace string Image string ProviderName string Bucket string Prefix string PodAnnotations map[string]string PodLabels map[string]string ServiceAccountAnnotations map[string]string ServiceAccountName string VeleroPodResources corev1.ResourceRequirements NodeAgentPodResources corev1.ResourceRequirements SecretData []byte RestoreOnly bool UseNodeAgent bool PrivilegedNodeAgent bool UseVolumeSnapshots bool BSLConfig map[string]string VSLConfig map[string]string DefaultRepoMaintenanceFrequency time.Duration GarbageCollectionFrequency time.Duration PodVolumeOperationTimeout time.Duration Plugins []string NoDefaultBackupLocation bool CACertData []byte Features []string DefaultVolumesToFsBackup bool UploaderType string DefaultSnapshotMoveData bool DisableInformerCache bool ScheduleSkipImmediately bool PodResources kube.PodResources KeepLatestMaintenanceJobs int BackupRepoConfigMap string RepoMaintenanceJobConfigMap string NodeAgentConfigMap string } func AllCRDs() *unstructured.UnstructuredList { resources := new(unstructured.UnstructuredList) // Set the GVK so that the serialization framework outputs the list properly resources.SetGroupVersionKind(schema.GroupVersionKind{Group: "", Version: "v1", Kind: "List"}) for _, crd := range v1crds.CRDs { crd.SetLabels(Labels()) if err := appendUnstructured(resources, crd); err != nil { fmt.Printf("error appending v1 CRD %s: %s\n", crd.GetName(), err.Error()) } } for _, crd := range v2alpha1crds.CRDs { crd.SetLabels(Labels()) if err := appendUnstructured(resources, crd); err != nil { fmt.Printf("error appending v2alpha1 CRD %s: %s\n", crd.GetName(), err.Error()) } } return resources } // AllResources returns a list of all resources necessary to install Velero, in the appropriate order, into a Kubernetes cluster. // Items are unstructured, since there are different data types returned. func AllResources(o *VeleroOptions) *unstructured.UnstructuredList { resources := AllCRDs() ns := Namespace(o.Namespace) if err := appendUnstructured(resources, ns); err != nil { fmt.Printf("error appending Namespace %s: %s\n", ns.GetName(), err.Error()) } serviceAccountName := defaultServiceAccountName if o.ServiceAccountName == "" { crb := ClusterRoleBinding(o.Namespace) if err := appendUnstructured(resources, crb); err != nil { fmt.Printf("error appending ClusterRoleBinding %s: %s\n", crb.GetName(), err.Error()) } sa := ServiceAccount(o.Namespace, o.ServiceAccountAnnotations) if err := appendUnstructured(resources, sa); err != nil { fmt.Printf("error appending ServiceAccount %s: %s\n", sa.GetName(), err.Error()) } } else { serviceAccountName = o.ServiceAccountName } if o.SecretData != nil { sec := Secret(o.Namespace, o.SecretData) if err := appendUnstructured(resources, sec); err != nil { fmt.Printf("error appending Secret %s: %s\n", sec.GetName(), err.Error()) } } if !o.NoDefaultBackupLocation { bsl := BackupStorageLocation(o.Namespace, o.ProviderName, o.Bucket, o.Prefix, o.BSLConfig, o.CACertData) if err := appendUnstructured(resources, bsl); err != nil { fmt.Printf("error appending BackupStorageLocation %s: %s\n", bsl.GetName(), err.Error()) } } // A snapshot location may not be desirable for users relying on pod volume backup/restore if o.UseVolumeSnapshots { vsl := VolumeSnapshotLocation(o.Namespace, o.ProviderName, o.VSLConfig) if err := appendUnstructured(resources, vsl); err != nil { fmt.Printf("error appending VolumeSnapshotLocation %s: %s\n", vsl.GetName(), err.Error()) } } secretPresent := o.SecretData != nil deployOpts := []podTemplateOption{ WithAnnotations(o.PodAnnotations), WithLabels(o.PodLabels), WithImage(o.Image), WithResources(o.VeleroPodResources), WithSecret(secretPresent), WithDefaultRepoMaintenanceFrequency(o.DefaultRepoMaintenanceFrequency), WithServiceAccountName(serviceAccountName), WithGarbageCollectionFrequency(o.GarbageCollectionFrequency), WithPodVolumeOperationTimeout(o.PodVolumeOperationTimeout), WithUploaderType(o.UploaderType), WithScheduleSkipImmediately(o.ScheduleSkipImmediately), WithPodResources(o.PodResources), WithKeepLatestMaintenanceJobs(o.KeepLatestMaintenanceJobs), } if len(o.Features) > 0 { deployOpts = append(deployOpts, WithFeatures(o.Features)) } if o.RestoreOnly { deployOpts = append(deployOpts, WithRestoreOnly(true)) } if len(o.Plugins) > 0 { deployOpts = append(deployOpts, WithPlugins(o.Plugins)) } if o.DefaultVolumesToFsBackup { deployOpts = append(deployOpts, WithDefaultVolumesToFsBackup(true)) } if o.DefaultSnapshotMoveData { deployOpts = append(deployOpts, WithDefaultSnapshotMoveData(true)) } if o.DisableInformerCache { deployOpts = append(deployOpts, WithDisableInformerCache(true)) } if len(o.BackupRepoConfigMap) > 0 { deployOpts = append(deployOpts, WithBackupRepoConfigMap(o.BackupRepoConfigMap)) } if len(o.RepoMaintenanceJobConfigMap) > 0 { deployOpts = append(deployOpts, WithRepoMaintenanceJobConfigMap(o.RepoMaintenanceJobConfigMap)) } deploy := Deployment(o.Namespace, deployOpts...) if err := appendUnstructured(resources, deploy); err != nil { fmt.Printf("error appending Deployment %s: %s\n", deploy.GetName(), err.Error()) } if o.UseNodeAgent { dsOpts := []podTemplateOption{ WithAnnotations(o.PodAnnotations), WithLabels(o.PodLabels), WithImage(o.Image), WithResources(o.NodeAgentPodResources), WithSecret(secretPresent), WithServiceAccountName(serviceAccountName), } if len(o.Features) > 0 { dsOpts = append(dsOpts, WithFeatures(o.Features)) } if o.PrivilegedNodeAgent { dsOpts = append(dsOpts, WithPrivilegedNodeAgent(true)) } if len(o.NodeAgentConfigMap) > 0 { dsOpts = append(dsOpts, WithNodeAgentConfigMap(o.NodeAgentConfigMap)) } ds := DaemonSet(o.Namespace, dsOpts...) if err := appendUnstructured(resources, ds); err != nil { fmt.Printf("error appending DaemonSet %s: %s\n", ds.GetName(), err.Error()) } } return resources }