From ed4437ad22f82ad42e2a24c74ac67a7be8028db6 Mon Sep 17 00:00:00 2001 From: Anshul Ahuja Date: Wed, 5 Jul 2023 13:51:32 +0530 Subject: [PATCH] initial draft Signed-off-by: Anshul Ahuja --- .../resourcemodifiers/resource_modifiers.go | 148 ++++++++++++++ .../resource_modifiers_test.go | 103 ++++++++++ .../resource_modifiers_validator.go | 55 ++++++ .../resource_modifiers_validator_test.go | 181 ++++++++++++++++++ pkg/controller/restore_controller.go | 53 +++-- pkg/restore/restore.go | 6 +- 6 files changed, 529 insertions(+), 17 deletions(-) create mode 100644 internal/resourcemodifiers/resource_modifiers.go create mode 100644 internal/resourcemodifiers/resource_modifiers_test.go create mode 100644 internal/resourcemodifiers/resource_modifiers_validator.go create mode 100644 internal/resourcemodifiers/resource_modifiers_validator_test.go diff --git a/internal/resourcemodifiers/resource_modifiers.go b/internal/resourcemodifiers/resource_modifiers.go new file mode 100644 index 000000000..c23e4d6f7 --- /dev/null +++ b/internal/resourcemodifiers/resource_modifiers.go @@ -0,0 +1,148 @@ +package resourcemodifiers + +import ( + "fmt" + "io" + "regexp" + "strings" + + jsonpatch "github.com/evanphx/json-patch" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "github.com/vmware-tanzu/velero/pkg/util/collections" + "gopkg.in/yaml.v3" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +const ( + ConfigmapRefType = "configmap" + ResourceModifierSupportedVersionV1 = "v1" +) + +type JsonPatch struct { + Operation string `yaml:"operation"` + Path string `yaml:"path"` + NewValue string `yaml:"newValue,omitempty"` +} + +type Conditions struct { + Namespaces []string `yaml:"namespaces,omitempty"` + GroupKind string `yaml:"groupKind"` + ResourceNameRegex string `yaml:"resourceNameRegex"` +} + +type ResourceModifierRule struct { + Conditions Conditions `yaml:"conditions"` + Patches []JsonPatch `yaml:"patches"` +} + +type ResourceModifiers struct { + Version string `yaml:"version"` + ResourceModifierRules []ResourceModifierRule `yaml:"resourceModifierRules"` +} + +func GetResourceModifiersFromConfig(cm *v1.ConfigMap) (*ResourceModifiers, error) { + if cm == nil { + return nil, fmt.Errorf("could not parse config from nil configmap") + } + if len(cm.Data) != 1 { + return nil, fmt.Errorf("illegal resource modifiers %s/%s configmap", cm.Name, cm.Namespace) + } + + var yamlData string + for _, v := range cm.Data { + yamlData = v + } + + resModifiers, err := unmarshalResourceModifiers(&yamlData) + if err != nil { + return nil, errors.WithStack(err) + } + + return resModifiers, nil +} + +func (p *ResourceModifiers) ApplyResourceModifierRules(obj *unstructured.Unstructured, log logrus.FieldLogger) []error { + var errs []error + for _, rule := range p.ResourceModifierRules { + errs = append(errs, rule.Apply(obj, log)) + } + + return errs +} + +func (r *ResourceModifierRule) Apply(obj *unstructured.Unstructured, log logrus.FieldLogger) error { + namespaceInclusion := collections.NewIncludesExcludes().Includes(r.Conditions.Namespaces...) + if !namespaceInclusion.ShouldInclude(obj.GetNamespace()) { + return nil + } + if !strings.EqualFold(obj.GroupVersionKind().GroupKind().String(), r.Conditions.GroupKind) { + return nil + } + if r.Conditions.ResourceNameRegex != "" { + match, _ := regexp.MatchString(r.Conditions.ResourceNameRegex, obj.GetName()) + if !match { + return nil + } + } + patches, err := r.PatchArrayToByteArray() + if err != nil { + return err + } + err = ApplyPatch(patches, obj, log) + if err != nil { + return err + } + return nil +} + +// convert all JsonPatch to string array with the format of jsonpatch.Patch and then convert it to byte array +func (r *ResourceModifierRule) PatchArrayToByteArray() ([]byte, error) { + var patches []string + for _, patch := range r.Patches { + patches = append(patches, patch.ToString()) + } + patchesStr := strings.Join(patches, ",\n\t") + return []byte(fmt.Sprintf(`[%s]`, patchesStr)), nil +} + +func (p *JsonPatch) ToString() string { + return fmt.Sprintf(`{"op": "%s", "path": "%s", "value": "%s"}`, p.Operation, p.Path, p.NewValue) +} + +func ApplyPatch(patch []byte, obj *unstructured.Unstructured, log logrus.FieldLogger) error { + jsonPatch, err := jsonpatch.DecodePatch(patch) + if err != nil { + return fmt.Errorf("error in decoding json patch %s", err.Error()) + } + objBytes, err := obj.MarshalJSON() + if err != nil { + return fmt.Errorf("error in marshalling object %s", err.Error()) + } + modifiedObjBytes, err := jsonPatch.Apply(objBytes) + if err != nil { + return fmt.Errorf("error in applying JSON Patch, could be due to test operator failing %s", err.Error()) + } + err = obj.UnmarshalJSON(modifiedObjBytes) + if err != nil { + return fmt.Errorf("error in unmarshalling modified object %s", err.Error()) + } + return nil +} + +func unmarshalResourceModifiers(yamlData *string) (*ResourceModifiers, error) { + resModifiers := &ResourceModifiers{} + err := decodeStruct(strings.NewReader(*yamlData), resModifiers) + if err != nil { + return nil, fmt.Errorf("failed to decode yaml data into resource modifiers %v", err) + } + return resModifiers, nil +} + +// decodeStruct restrict validate the keys in decoded mappings to exist as fields in the struct being decoded into +func decodeStruct(r io.Reader, s interface{}) error { + dec := yaml.NewDecoder(r) + dec.KnownFields(true) + return dec.Decode(s) +} diff --git a/internal/resourcemodifiers/resource_modifiers_test.go b/internal/resourcemodifiers/resource_modifiers_test.go new file mode 100644 index 000000000..54e3d6714 --- /dev/null +++ b/internal/resourcemodifiers/resource_modifiers_test.go @@ -0,0 +1,103 @@ +package resourcemodifiers + +import ( + "testing" + + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +func TestGetResourceModifiersFromConfig(t *testing.T) { + // Create a test ConfigMap + cm := &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-configmap", + Namespace: "test-namespace", + }, + Data: map[string]string{ + "sub.yml": "# kubectl delete cm jsonsub-configmap -n velero\n# kubectl create cm jsonsub-configmap --from-file /home/anshulahuja/workspace/fork-velero/tilt-resources/examples/sub.yml -nvelero\nversion: v1\nresourceModifierRules:\n- conditions:\n groupKind: persistentvolumeclaims.storage.k8s.io\n resourceNameRegex: \".*\"\n namespaces:\n - bar\n - foo\n patches:\n - operation: replace\n path: \"/spec/storageClassName\"\n newValue: \"premium\"\n - operation: remove\n path: \"/metadata/labels/test\"\n\n\n", + }, + } + + // Call the function and check for errors + original, err := GetResourceModifiersFromConfig(cm) + assert.Nil(t, err) + + // Check that the returned resourceModifiers object contains the expected data + assert.Equal(t, "v1", original.Version) + assert.Len(t, original.ResourceModifierRules, 1) + assert.Len(t, original.ResourceModifierRules[0].Patches, 2) + + expected := &ResourceModifiers{ + Version: "v1", + ResourceModifierRules: []ResourceModifierRule{ + { + Conditions: Conditions{ + GroupKind: "persistentvolumeclaims.storage.k8s.io", + ResourceNameRegex: ".*", + Namespaces: []string{"bar", "foo"}, + }, + Patches: []JsonPatch{ + { + Operation: "replace", + Path: "/spec/storageClassName", + NewValue: "premium", + }, + { + Operation: "remove", + Path: "/metadata/labels/test", + }, + }, + }, + }, + } + if err != nil { + t.Fatalf("failed to build policy with error %v", err) + } + assert.Equal(t, original, expected) +} + +// pvc1 := &unstructured.Unstructured{ +// Object: map[string]interface{}{ +// "kind": "PersistentVolumeClaim", +// "metadata": map[string]interface{}{ +// "name": "test-pvc", +// "namespace": "foo", +// }, +// }, +// } + +func TestResourceModifiers_ApplyResourceModifierRules(t *testing.T) { + type fields struct { + Version string + ResourceModifierRules []ResourceModifierRule + } + type args struct { + obj *unstructured.Unstructured + log logrus.FieldLogger + } + tests := []struct { + name string + fields fields + args args + want []error + wantObj *unstructured.Unstructured + }{ + // TODO: Add test cases. + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + p := &ResourceModifiers{ + Version: tt.fields.Version, + ResourceModifierRules: tt.fields.ResourceModifierRules, + } + // comapre lenght of got and want { + got := p.ApplyResourceModifierRules(tt.args.obj, tt.args.log) + assert.Equal(t, len(got), len(tt.want)) + assert.Equal(t, *tt.args.obj, *tt.wantObj) + }) + } +} diff --git a/internal/resourcemodifiers/resource_modifiers_validator.go b/internal/resourcemodifiers/resource_modifiers_validator.go new file mode 100644 index 000000000..705c0c37a --- /dev/null +++ b/internal/resourcemodifiers/resource_modifiers_validator.go @@ -0,0 +1,55 @@ +package resourcemodifiers + +import ( + "strings" + + "fmt" + + "github.com/pkg/errors" +) + +func (r *ResourceModifierRule) Validate() error { + if err := r.Conditions.Validate(); err != nil { + return errors.WithStack(err) + } + for _, patch := range r.Patches { + if err := patch.Validate(); err != nil { + return errors.WithStack(err) + } + } + return nil +} + +func (p *ResourceModifiers) Validate() error { + if !strings.EqualFold(p.Version, ResourceModifierSupportedVersionV1) { + return fmt.Errorf("unsupported resource modifier version %s", p.Version) + } + if len(p.ResourceModifierRules) == 0 { + return fmt.Errorf("resource modifier rules cannot be empty") + } + for _, rule := range p.ResourceModifierRules { + if err := rule.Validate(); err != nil { + return errors.WithStack(err) + } + } + + return nil +} + +func (p *JsonPatch) Validate() error { + // TODO validate allowed operation + if p.Operation == "" { + return fmt.Errorf("operation cannot be empty") + } + if p.Path == "" { + return fmt.Errorf("path cannot be empty") + } + return nil +} + +func (c *Conditions) Validate() error { + if c.GroupKind == "" { + return fmt.Errorf("groupkind cannot be empty") + } + return nil +} diff --git a/internal/resourcemodifiers/resource_modifiers_validator_test.go b/internal/resourcemodifiers/resource_modifiers_validator_test.go new file mode 100644 index 000000000..5eaa8b3ac --- /dev/null +++ b/internal/resourcemodifiers/resource_modifiers_validator_test.go @@ -0,0 +1,181 @@ +package resourcemodifiers + +import ( + "testing" +) + +func TestResourceModifiers_Validate(t *testing.T) { + type fields struct { + Version string + ResourceModifierRules []ResourceModifierRule + } + tests := []struct { + name string + fields fields + wantErr bool + }{ + { + name: "correct version, non 0 length ResourceModifierRules", + fields: fields{ + Version: "v1", + ResourceModifierRules: []ResourceModifierRule{ + { + Conditions: Conditions{ + GroupKind: "persistentvolumeclaims.storage.k8s.io", + ResourceNameRegex: ".*", + Namespaces: []string{"bar", "foo"}, + }, + Patches: []JsonPatch{ + { + Operation: "replace", + Path: "/spec/storageClassName", + NewValue: "premium", + }, + }, + }, + }, + }, + wantErr: false, + }, + { + name: "incorrect version, non 0 length ResourceModifierRules", + fields: fields{ + Version: "v2", + ResourceModifierRules: []ResourceModifierRule{ + { + Conditions: Conditions{ + GroupKind: "persistentvolumeclaims.storage.k8s.io", + ResourceNameRegex: ".*", + Namespaces: []string{"bar", "foo"}, + }, + Patches: []JsonPatch{ + { + Operation: "replace", + Path: "/spec/storageClassName", + NewValue: "premium", + }, + }, + }, + }, + }, + wantErr: true, + }, + { + name: "correct version, 0 length ResourceModifierRules", + fields: fields{ + Version: "v1", + ResourceModifierRules: []ResourceModifierRule{}, + }, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + p := &ResourceModifiers{ + Version: tt.fields.Version, + ResourceModifierRules: tt.fields.ResourceModifierRules, + } + if err := p.Validate(); (err != nil) != tt.wantErr { + t.Errorf("ResourceModifiers.Validate() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestJsonPatch_Validate(t *testing.T) { + type fields struct { + Operation string + Path string + NewValue string + } + tests := []struct { + name string + fields fields + wantErr bool + }{ + { + name: "not empty operation, path, and new value, valid scenario", + fields: fields{ + Operation: "replace", + Path: "/spec/storageClassName", + NewValue: "premium", + }, + wantErr: false, + }, + { + name: "empty operation throws error", + fields: fields{ + Operation: "", + Path: "/spec/storageClassName", + NewValue: "premium", + }, + wantErr: true, + }, + { + name: "empty path throws error", + fields: fields{ + Operation: "replace", + Path: "", + NewValue: "premium", + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + p := &JsonPatch{ + Operation: tt.fields.Operation, + Path: tt.fields.Path, + NewValue: tt.fields.NewValue, + } + if err := p.Validate(); (err != nil) != tt.wantErr { + t.Errorf("JsonPatch.Validate() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestConditions_Validate(t *testing.T) { + type fields struct { + Namespaces []string + GroupKind string + ResourceNameRegex string + } + tests := []struct { + name string + fields fields + wantErr bool + }{ + { + name: "non 0 length namespaces, non empty group kind, non empty resource name regex", + fields: fields{ + Namespaces: []string{"bar", "foo"}, + GroupKind: "persistentvolumeclaims.storage.k8s.io", + ResourceNameRegex: ".*", + }, + wantErr: false, + }, + { + name: "empty group kind throws error", + fields: fields{ + Namespaces: []string{"bar", "foo"}, + GroupKind: "", + ResourceNameRegex: ".*", + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := &Conditions{ + Namespaces: tt.fields.Namespaces, + GroupKind: tt.fields.GroupKind, + ResourceNameRegex: tt.fields.ResourceNameRegex, + } + if err := c.Validate(); (err != nil) != tt.wantErr { + t.Errorf("Conditions.Validate() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} diff --git a/pkg/controller/restore_controller.go b/pkg/controller/restore_controller.go index c63de4fa5..7e9ccfd78 100644 --- a/pkg/controller/restore_controller.go +++ b/pkg/controller/restore_controller.go @@ -29,6 +29,7 @@ import ( "github.com/pkg/errors" "github.com/sirupsen/logrus" + corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" @@ -37,8 +38,10 @@ import ( "k8s.io/utils/clock" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + kbclient "sigs.k8s.io/controller-runtime/pkg/client" "github.com/vmware-tanzu/velero/internal/hook" + "github.com/vmware-tanzu/velero/internal/resourcemodifiers" api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/itemoperation" "github.com/vmware-tanzu/velero/pkg/label" @@ -163,7 +166,7 @@ func (r *restoreReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct original := restore.DeepCopy() // Validate the restore and fetch the backup - info := r.validateAndComplete(restore) + info, resourceModifiers := r.validateAndComplete(restore) // Register attempts after validation so we don't have to fetch the backup multiple times backupScheduleName := restore.Spec.ScheduleName @@ -197,7 +200,7 @@ func (r *restoreReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct return ctrl.Result{}, nil } - if err := r.runValidatedRestore(restore, info); err != nil { + if err := r.runValidatedRestore(restore, info, resourceModifiers); err != nil { log.WithError(err).Debug("Restore failed") restore.Status.Phase = api.RestorePhaseFailed restore.Status.FailureReason = err.Error() @@ -241,7 +244,7 @@ func (r *restoreReconciler) SetupWithManager(mgr ctrl.Manager) error { Complete(r) } -func (r *restoreReconciler) validateAndComplete(restore *api.Restore) backupInfo { +func (r *restoreReconciler) validateAndComplete(restore *api.Restore) (backupInfo, *resourcemodifiers.ResourceModifiers) { // add non-restorable resources to restore's excluded resources excludedResources := sets.NewString(restore.Spec.ExcludedResources...) for _, nonrestorable := range nonRestorableResources { @@ -276,7 +279,7 @@ func (r *restoreReconciler) validateAndComplete(restore *api.Restore) backupInfo // validate that exactly one of BackupName and ScheduleName have been specified if !backupXorScheduleProvided(restore) { restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, "Either a backup or schedule must be specified as a source for the restore, but not both") - return backupInfo{} + return backupInfo{}, nil } // validate Restore Init Hook's InitContainers @@ -307,7 +310,7 @@ func (r *restoreReconciler) validateAndComplete(restore *api.Restore) backupInfo backupList := &api.BackupList{} if err := r.kbClient.List(context.Background(), backupList, &client.ListOptions{LabelSelector: selector}); err != nil { restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, "Unable to list backups for schedule") - return backupInfo{} + return backupInfo{}, nil } if len(backupList.Items) == 0 { restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, "No backups found for schedule") @@ -317,14 +320,14 @@ func (r *restoreReconciler) validateAndComplete(restore *api.Restore) backupInfo restore.Spec.BackupName = backup.Name } else { restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, "No completed backups found for schedule") - return backupInfo{} + return backupInfo{}, nil } } info, err := r.fetchBackupInfo(restore.Spec.BackupName) if err != nil { restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, fmt.Sprintf("Error retrieving backup: %v", err)) - return backupInfo{} + return backupInfo{}, nil } // Fill in the ScheduleName so it's easier to consume for metrics. @@ -332,7 +335,25 @@ func (r *restoreReconciler) validateAndComplete(restore *api.Restore) backupInfo restore.Spec.ScheduleName = info.backup.GetLabels()[api.ScheduleNameLabel] } - return info + var resourceModifiers *resourcemodifiers.ResourceModifiers = nil + if restore.Spec.ResourceModifier != nil && restore.Spec.ResourceModifier.Kind == resourcemodifiers.ConfigmapRefType { + ResourceModifierConfigMap := &corev1api.ConfigMap{} + err := r.kbClient.Get(context.Background(), kbclient.ObjectKey{Namespace: restore.Namespace, Name: restore.Spec.ResourceModifier.Name}, ResourceModifierConfigMap) + if err != nil { + restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, fmt.Sprintf("failed to get resource modifiers configmap %s/%s", restore.Namespace, restore.Spec.ResourceModifier.Name)) + return backupInfo{}, nil + } + resourceModifiers, err = resourcemodifiers.GetResourceModifiersFromConfig(ResourceModifierConfigMap) + if err != nil { + restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, errors.Wrapf(err, fmt.Sprintf("Error in parsing resource modifiers provided in configmap %s/%s", restore.Namespace, restore.Spec.ResourceModifier.Name)).Error()) + return backupInfo{}, nil + } else if err = resourceModifiers.Validate(); err != nil { + restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, errors.Wrapf(err, fmt.Sprintf("Validation error in resource modifiers provided in configmap %s/%s", restore.Namespace, restore.Spec.ResourceModifier.Name)).Error()) + return backupInfo{}, nil + } + } + + return info, resourceModifiers } // backupXorScheduleProvided returns true if exactly one of BackupName and @@ -405,7 +426,7 @@ func fetchBackupInfoInternal(kbClient client.Client, namespace, backupName strin // The log and results files are uploaded to backup storage. Any error returned from this function // means that the restore failed. This function updates the restore API object with warning and error // counts, but *does not* update its phase or patch it via the API. -func (r *restoreReconciler) runValidatedRestore(restore *api.Restore, info backupInfo) error { +func (r *restoreReconciler) runValidatedRestore(restore *api.Restore, info backupInfo, resourceModifiers *resourcemodifiers.ResourceModifiers) error { // instantiate the per-restore logger that will output both to a temp file // (for upload to object storage) and to stdout. restoreLog, err := logging.NewTempFileLogger(r.restoreLogLevel, r.logFormat, nil, logrus.Fields{"restore": kubeutil.NamespaceAndName(restore)}) @@ -458,13 +479,15 @@ func (r *restoreReconciler) runValidatedRestore(restore *api.Restore, info backu for i := range podVolumeBackupList.Items { podVolumeBackups = append(podVolumeBackups, &podVolumeBackupList.Items[i]) } + restoreReq := &pkgrestore.Request{ - Log: restoreLog, - Restore: restore, - Backup: info.backup, - PodVolumeBackups: podVolumeBackups, - VolumeSnapshots: volumeSnapshots, - BackupReader: backupFile, + Log: restoreLog, + Restore: restore, + Backup: info.backup, + PodVolumeBackups: podVolumeBackups, + VolumeSnapshots: volumeSnapshots, + BackupReader: backupFile, + ResourceModifiers: resourceModifiers, } restoreWarnings, restoreErrors := r.restorer.RestoreWithResolvers(restoreReq, actionsResolver, pluginManager) diff --git a/pkg/restore/restore.go b/pkg/restore/restore.go index 22bf860ae..5ae90dec9 100644 --- a/pkg/restore/restore.go +++ b/pkg/restore/restore.go @@ -1344,8 +1344,10 @@ func (ctx *restoreContext) restoreItem(obj *unstructured.Unstructured, groupReso addRestoreLabels(obj, ctx.restore.Name, ctx.restore.Spec.BackupName) if ctx.resourceModifiers != nil { - if err := ctx.resourceModifiers.ApplyResourceModifierRules(obj, ctx.log); err != nil { - errs.Add(namespace, err) + if errList := ctx.resourceModifiers.ApplyResourceModifierRules(obj, ctx.log); errList != nil { + for _, err := range errList { + errs.Add(namespace, err) + } } }