initial draft

Signed-off-by: Anshul Ahuja <anshulahuja@microsoft.com>
pull/6452/head
Anshul Ahuja 2023-07-05 13:51:32 +05:30
parent bb20d0d2f2
commit ed4437ad22
6 changed files with 529 additions and 17 deletions

View File

@ -0,0 +1,148 @@
package resourcemodifiers
import (
"fmt"
"io"
"regexp"
"strings"
jsonpatch "github.com/evanphx/json-patch"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"github.com/vmware-tanzu/velero/pkg/util/collections"
"gopkg.in/yaml.v3"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
)
const (
ConfigmapRefType = "configmap"
ResourceModifierSupportedVersionV1 = "v1"
)
type JsonPatch struct {
Operation string `yaml:"operation"`
Path string `yaml:"path"`
NewValue string `yaml:"newValue,omitempty"`
}
type Conditions struct {
Namespaces []string `yaml:"namespaces,omitempty"`
GroupKind string `yaml:"groupKind"`
ResourceNameRegex string `yaml:"resourceNameRegex"`
}
type ResourceModifierRule struct {
Conditions Conditions `yaml:"conditions"`
Patches []JsonPatch `yaml:"patches"`
}
type ResourceModifiers struct {
Version string `yaml:"version"`
ResourceModifierRules []ResourceModifierRule `yaml:"resourceModifierRules"`
}
func GetResourceModifiersFromConfig(cm *v1.ConfigMap) (*ResourceModifiers, error) {
if cm == nil {
return nil, fmt.Errorf("could not parse config from nil configmap")
}
if len(cm.Data) != 1 {
return nil, fmt.Errorf("illegal resource modifiers %s/%s configmap", cm.Name, cm.Namespace)
}
var yamlData string
for _, v := range cm.Data {
yamlData = v
}
resModifiers, err := unmarshalResourceModifiers(&yamlData)
if err != nil {
return nil, errors.WithStack(err)
}
return resModifiers, nil
}
func (p *ResourceModifiers) ApplyResourceModifierRules(obj *unstructured.Unstructured, log logrus.FieldLogger) []error {
var errs []error
for _, rule := range p.ResourceModifierRules {
errs = append(errs, rule.Apply(obj, log))
}
return errs
}
func (r *ResourceModifierRule) Apply(obj *unstructured.Unstructured, log logrus.FieldLogger) error {
namespaceInclusion := collections.NewIncludesExcludes().Includes(r.Conditions.Namespaces...)
if !namespaceInclusion.ShouldInclude(obj.GetNamespace()) {
return nil
}
if !strings.EqualFold(obj.GroupVersionKind().GroupKind().String(), r.Conditions.GroupKind) {
return nil
}
if r.Conditions.ResourceNameRegex != "" {
match, _ := regexp.MatchString(r.Conditions.ResourceNameRegex, obj.GetName())
if !match {
return nil
}
}
patches, err := r.PatchArrayToByteArray()
if err != nil {
return err
}
err = ApplyPatch(patches, obj, log)
if err != nil {
return err
}
return nil
}
// convert all JsonPatch to string array with the format of jsonpatch.Patch and then convert it to byte array
func (r *ResourceModifierRule) PatchArrayToByteArray() ([]byte, error) {
var patches []string
for _, patch := range r.Patches {
patches = append(patches, patch.ToString())
}
patchesStr := strings.Join(patches, ",\n\t")
return []byte(fmt.Sprintf(`[%s]`, patchesStr)), nil
}
func (p *JsonPatch) ToString() string {
return fmt.Sprintf(`{"op": "%s", "path": "%s", "value": "%s"}`, p.Operation, p.Path, p.NewValue)
}
func ApplyPatch(patch []byte, obj *unstructured.Unstructured, log logrus.FieldLogger) error {
jsonPatch, err := jsonpatch.DecodePatch(patch)
if err != nil {
return fmt.Errorf("error in decoding json patch %s", err.Error())
}
objBytes, err := obj.MarshalJSON()
if err != nil {
return fmt.Errorf("error in marshalling object %s", err.Error())
}
modifiedObjBytes, err := jsonPatch.Apply(objBytes)
if err != nil {
return fmt.Errorf("error in applying JSON Patch, could be due to test operator failing %s", err.Error())
}
err = obj.UnmarshalJSON(modifiedObjBytes)
if err != nil {
return fmt.Errorf("error in unmarshalling modified object %s", err.Error())
}
return nil
}
func unmarshalResourceModifiers(yamlData *string) (*ResourceModifiers, error) {
resModifiers := &ResourceModifiers{}
err := decodeStruct(strings.NewReader(*yamlData), resModifiers)
if err != nil {
return nil, fmt.Errorf("failed to decode yaml data into resource modifiers %v", err)
}
return resModifiers, nil
}
// decodeStruct restrict validate the keys in decoded mappings to exist as fields in the struct being decoded into
func decodeStruct(r io.Reader, s interface{}) error {
dec := yaml.NewDecoder(r)
dec.KnownFields(true)
return dec.Decode(s)
}

View File

@ -0,0 +1,103 @@
package resourcemodifiers
import (
"testing"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
)
func TestGetResourceModifiersFromConfig(t *testing.T) {
// Create a test ConfigMap
cm := &v1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: "test-configmap",
Namespace: "test-namespace",
},
Data: map[string]string{
"sub.yml": "# kubectl delete cm jsonsub-configmap -n velero\n# kubectl create cm jsonsub-configmap --from-file /home/anshulahuja/workspace/fork-velero/tilt-resources/examples/sub.yml -nvelero\nversion: v1\nresourceModifierRules:\n- conditions:\n groupKind: persistentvolumeclaims.storage.k8s.io\n resourceNameRegex: \".*\"\n namespaces:\n - bar\n - foo\n patches:\n - operation: replace\n path: \"/spec/storageClassName\"\n newValue: \"premium\"\n - operation: remove\n path: \"/metadata/labels/test\"\n\n\n",
},
}
// Call the function and check for errors
original, err := GetResourceModifiersFromConfig(cm)
assert.Nil(t, err)
// Check that the returned resourceModifiers object contains the expected data
assert.Equal(t, "v1", original.Version)
assert.Len(t, original.ResourceModifierRules, 1)
assert.Len(t, original.ResourceModifierRules[0].Patches, 2)
expected := &ResourceModifiers{
Version: "v1",
ResourceModifierRules: []ResourceModifierRule{
{
Conditions: Conditions{
GroupKind: "persistentvolumeclaims.storage.k8s.io",
ResourceNameRegex: ".*",
Namespaces: []string{"bar", "foo"},
},
Patches: []JsonPatch{
{
Operation: "replace",
Path: "/spec/storageClassName",
NewValue: "premium",
},
{
Operation: "remove",
Path: "/metadata/labels/test",
},
},
},
},
}
if err != nil {
t.Fatalf("failed to build policy with error %v", err)
}
assert.Equal(t, original, expected)
}
// pvc1 := &unstructured.Unstructured{
// Object: map[string]interface{}{
// "kind": "PersistentVolumeClaim",
// "metadata": map[string]interface{}{
// "name": "test-pvc",
// "namespace": "foo",
// },
// },
// }
func TestResourceModifiers_ApplyResourceModifierRules(t *testing.T) {
type fields struct {
Version string
ResourceModifierRules []ResourceModifierRule
}
type args struct {
obj *unstructured.Unstructured
log logrus.FieldLogger
}
tests := []struct {
name string
fields fields
args args
want []error
wantObj *unstructured.Unstructured
}{
// TODO: Add test cases.
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
p := &ResourceModifiers{
Version: tt.fields.Version,
ResourceModifierRules: tt.fields.ResourceModifierRules,
}
// comapre lenght of got and want {
got := p.ApplyResourceModifierRules(tt.args.obj, tt.args.log)
assert.Equal(t, len(got), len(tt.want))
assert.Equal(t, *tt.args.obj, *tt.wantObj)
})
}
}

View File

@ -0,0 +1,55 @@
package resourcemodifiers
import (
"strings"
"fmt"
"github.com/pkg/errors"
)
func (r *ResourceModifierRule) Validate() error {
if err := r.Conditions.Validate(); err != nil {
return errors.WithStack(err)
}
for _, patch := range r.Patches {
if err := patch.Validate(); err != nil {
return errors.WithStack(err)
}
}
return nil
}
func (p *ResourceModifiers) Validate() error {
if !strings.EqualFold(p.Version, ResourceModifierSupportedVersionV1) {
return fmt.Errorf("unsupported resource modifier version %s", p.Version)
}
if len(p.ResourceModifierRules) == 0 {
return fmt.Errorf("resource modifier rules cannot be empty")
}
for _, rule := range p.ResourceModifierRules {
if err := rule.Validate(); err != nil {
return errors.WithStack(err)
}
}
return nil
}
func (p *JsonPatch) Validate() error {
// TODO validate allowed operation
if p.Operation == "" {
return fmt.Errorf("operation cannot be empty")
}
if p.Path == "" {
return fmt.Errorf("path cannot be empty")
}
return nil
}
func (c *Conditions) Validate() error {
if c.GroupKind == "" {
return fmt.Errorf("groupkind cannot be empty")
}
return nil
}

View File

@ -0,0 +1,181 @@
package resourcemodifiers
import (
"testing"
)
func TestResourceModifiers_Validate(t *testing.T) {
type fields struct {
Version string
ResourceModifierRules []ResourceModifierRule
}
tests := []struct {
name string
fields fields
wantErr bool
}{
{
name: "correct version, non 0 length ResourceModifierRules",
fields: fields{
Version: "v1",
ResourceModifierRules: []ResourceModifierRule{
{
Conditions: Conditions{
GroupKind: "persistentvolumeclaims.storage.k8s.io",
ResourceNameRegex: ".*",
Namespaces: []string{"bar", "foo"},
},
Patches: []JsonPatch{
{
Operation: "replace",
Path: "/spec/storageClassName",
NewValue: "premium",
},
},
},
},
},
wantErr: false,
},
{
name: "incorrect version, non 0 length ResourceModifierRules",
fields: fields{
Version: "v2",
ResourceModifierRules: []ResourceModifierRule{
{
Conditions: Conditions{
GroupKind: "persistentvolumeclaims.storage.k8s.io",
ResourceNameRegex: ".*",
Namespaces: []string{"bar", "foo"},
},
Patches: []JsonPatch{
{
Operation: "replace",
Path: "/spec/storageClassName",
NewValue: "premium",
},
},
},
},
},
wantErr: true,
},
{
name: "correct version, 0 length ResourceModifierRules",
fields: fields{
Version: "v1",
ResourceModifierRules: []ResourceModifierRule{},
},
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
p := &ResourceModifiers{
Version: tt.fields.Version,
ResourceModifierRules: tt.fields.ResourceModifierRules,
}
if err := p.Validate(); (err != nil) != tt.wantErr {
t.Errorf("ResourceModifiers.Validate() error = %v, wantErr %v", err, tt.wantErr)
}
})
}
}
func TestJsonPatch_Validate(t *testing.T) {
type fields struct {
Operation string
Path string
NewValue string
}
tests := []struct {
name string
fields fields
wantErr bool
}{
{
name: "not empty operation, path, and new value, valid scenario",
fields: fields{
Operation: "replace",
Path: "/spec/storageClassName",
NewValue: "premium",
},
wantErr: false,
},
{
name: "empty operation throws error",
fields: fields{
Operation: "",
Path: "/spec/storageClassName",
NewValue: "premium",
},
wantErr: true,
},
{
name: "empty path throws error",
fields: fields{
Operation: "replace",
Path: "",
NewValue: "premium",
},
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
p := &JsonPatch{
Operation: tt.fields.Operation,
Path: tt.fields.Path,
NewValue: tt.fields.NewValue,
}
if err := p.Validate(); (err != nil) != tt.wantErr {
t.Errorf("JsonPatch.Validate() error = %v, wantErr %v", err, tt.wantErr)
}
})
}
}
func TestConditions_Validate(t *testing.T) {
type fields struct {
Namespaces []string
GroupKind string
ResourceNameRegex string
}
tests := []struct {
name string
fields fields
wantErr bool
}{
{
name: "non 0 length namespaces, non empty group kind, non empty resource name regex",
fields: fields{
Namespaces: []string{"bar", "foo"},
GroupKind: "persistentvolumeclaims.storage.k8s.io",
ResourceNameRegex: ".*",
},
wantErr: false,
},
{
name: "empty group kind throws error",
fields: fields{
Namespaces: []string{"bar", "foo"},
GroupKind: "",
ResourceNameRegex: ".*",
},
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
c := &Conditions{
Namespaces: tt.fields.Namespaces,
GroupKind: tt.fields.GroupKind,
ResourceNameRegex: tt.fields.ResourceNameRegex,
}
if err := c.Validate(); (err != nil) != tt.wantErr {
t.Errorf("Conditions.Validate() error = %v, wantErr %v", err, tt.wantErr)
}
})
}
}

View File

@ -29,6 +29,7 @@ import (
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
corev1api "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/types"
@ -37,8 +38,10 @@ import (
"k8s.io/utils/clock"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
kbclient "sigs.k8s.io/controller-runtime/pkg/client"
"github.com/vmware-tanzu/velero/internal/hook"
"github.com/vmware-tanzu/velero/internal/resourcemodifiers"
api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
"github.com/vmware-tanzu/velero/pkg/itemoperation"
"github.com/vmware-tanzu/velero/pkg/label"
@ -163,7 +166,7 @@ func (r *restoreReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct
original := restore.DeepCopy()
// Validate the restore and fetch the backup
info := r.validateAndComplete(restore)
info, resourceModifiers := r.validateAndComplete(restore)
// Register attempts after validation so we don't have to fetch the backup multiple times
backupScheduleName := restore.Spec.ScheduleName
@ -197,7 +200,7 @@ func (r *restoreReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct
return ctrl.Result{}, nil
}
if err := r.runValidatedRestore(restore, info); err != nil {
if err := r.runValidatedRestore(restore, info, resourceModifiers); err != nil {
log.WithError(err).Debug("Restore failed")
restore.Status.Phase = api.RestorePhaseFailed
restore.Status.FailureReason = err.Error()
@ -241,7 +244,7 @@ func (r *restoreReconciler) SetupWithManager(mgr ctrl.Manager) error {
Complete(r)
}
func (r *restoreReconciler) validateAndComplete(restore *api.Restore) backupInfo {
func (r *restoreReconciler) validateAndComplete(restore *api.Restore) (backupInfo, *resourcemodifiers.ResourceModifiers) {
// add non-restorable resources to restore's excluded resources
excludedResources := sets.NewString(restore.Spec.ExcludedResources...)
for _, nonrestorable := range nonRestorableResources {
@ -276,7 +279,7 @@ func (r *restoreReconciler) validateAndComplete(restore *api.Restore) backupInfo
// validate that exactly one of BackupName and ScheduleName have been specified
if !backupXorScheduleProvided(restore) {
restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, "Either a backup or schedule must be specified as a source for the restore, but not both")
return backupInfo{}
return backupInfo{}, nil
}
// validate Restore Init Hook's InitContainers
@ -307,7 +310,7 @@ func (r *restoreReconciler) validateAndComplete(restore *api.Restore) backupInfo
backupList := &api.BackupList{}
if err := r.kbClient.List(context.Background(), backupList, &client.ListOptions{LabelSelector: selector}); err != nil {
restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, "Unable to list backups for schedule")
return backupInfo{}
return backupInfo{}, nil
}
if len(backupList.Items) == 0 {
restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, "No backups found for schedule")
@ -317,14 +320,14 @@ func (r *restoreReconciler) validateAndComplete(restore *api.Restore) backupInfo
restore.Spec.BackupName = backup.Name
} else {
restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, "No completed backups found for schedule")
return backupInfo{}
return backupInfo{}, nil
}
}
info, err := r.fetchBackupInfo(restore.Spec.BackupName)
if err != nil {
restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, fmt.Sprintf("Error retrieving backup: %v", err))
return backupInfo{}
return backupInfo{}, nil
}
// Fill in the ScheduleName so it's easier to consume for metrics.
@ -332,7 +335,25 @@ func (r *restoreReconciler) validateAndComplete(restore *api.Restore) backupInfo
restore.Spec.ScheduleName = info.backup.GetLabels()[api.ScheduleNameLabel]
}
return info
var resourceModifiers *resourcemodifiers.ResourceModifiers = nil
if restore.Spec.ResourceModifier != nil && restore.Spec.ResourceModifier.Kind == resourcemodifiers.ConfigmapRefType {
ResourceModifierConfigMap := &corev1api.ConfigMap{}
err := r.kbClient.Get(context.Background(), kbclient.ObjectKey{Namespace: restore.Namespace, Name: restore.Spec.ResourceModifier.Name}, ResourceModifierConfigMap)
if err != nil {
restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, fmt.Sprintf("failed to get resource modifiers configmap %s/%s", restore.Namespace, restore.Spec.ResourceModifier.Name))
return backupInfo{}, nil
}
resourceModifiers, err = resourcemodifiers.GetResourceModifiersFromConfig(ResourceModifierConfigMap)
if err != nil {
restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, errors.Wrapf(err, fmt.Sprintf("Error in parsing resource modifiers provided in configmap %s/%s", restore.Namespace, restore.Spec.ResourceModifier.Name)).Error())
return backupInfo{}, nil
} else if err = resourceModifiers.Validate(); err != nil {
restore.Status.ValidationErrors = append(restore.Status.ValidationErrors, errors.Wrapf(err, fmt.Sprintf("Validation error in resource modifiers provided in configmap %s/%s", restore.Namespace, restore.Spec.ResourceModifier.Name)).Error())
return backupInfo{}, nil
}
}
return info, resourceModifiers
}
// backupXorScheduleProvided returns true if exactly one of BackupName and
@ -405,7 +426,7 @@ func fetchBackupInfoInternal(kbClient client.Client, namespace, backupName strin
// The log and results files are uploaded to backup storage. Any error returned from this function
// means that the restore failed. This function updates the restore API object with warning and error
// counts, but *does not* update its phase or patch it via the API.
func (r *restoreReconciler) runValidatedRestore(restore *api.Restore, info backupInfo) error {
func (r *restoreReconciler) runValidatedRestore(restore *api.Restore, info backupInfo, resourceModifiers *resourcemodifiers.ResourceModifiers) error {
// instantiate the per-restore logger that will output both to a temp file
// (for upload to object storage) and to stdout.
restoreLog, err := logging.NewTempFileLogger(r.restoreLogLevel, r.logFormat, nil, logrus.Fields{"restore": kubeutil.NamespaceAndName(restore)})
@ -458,13 +479,15 @@ func (r *restoreReconciler) runValidatedRestore(restore *api.Restore, info backu
for i := range podVolumeBackupList.Items {
podVolumeBackups = append(podVolumeBackups, &podVolumeBackupList.Items[i])
}
restoreReq := &pkgrestore.Request{
Log: restoreLog,
Restore: restore,
Backup: info.backup,
PodVolumeBackups: podVolumeBackups,
VolumeSnapshots: volumeSnapshots,
BackupReader: backupFile,
Log: restoreLog,
Restore: restore,
Backup: info.backup,
PodVolumeBackups: podVolumeBackups,
VolumeSnapshots: volumeSnapshots,
BackupReader: backupFile,
ResourceModifiers: resourceModifiers,
}
restoreWarnings, restoreErrors := r.restorer.RestoreWithResolvers(restoreReq, actionsResolver, pluginManager)

View File

@ -1344,8 +1344,10 @@ func (ctx *restoreContext) restoreItem(obj *unstructured.Unstructured, groupReso
addRestoreLabels(obj, ctx.restore.Name, ctx.restore.Spec.BackupName)
if ctx.resourceModifiers != nil {
if err := ctx.resourceModifiers.ApplyResourceModifierRules(obj, ctx.log); err != nil {
errs.Add(namespace, err)
if errList := ctx.resourceModifiers.ApplyResourceModifierRules(obj, ctx.log); errList != nil {
for _, err := range errList {
errs.Add(namespace, err)
}
}
}