Add secret restore item action to handle service account token secret

Add secret restore item action to handle service account token secret:
1. Skip the restoration for the auto-created service account token secret
2. Remove several fields for non-auto-created service account token secret to make sure the secret can be restored

Signed-off-by: Wenkai Yin(尹文开) <yinw@vmware.com>

changelogs/unreleased/5843-ywk253100

@@ -0,0 +1 @@
+Add secret restore item action to handle service account token secret

pkg/cmd/server/plugin/plugin.go

@@ -58,7 +58,8 @@ func NewCommand(f client.Factory) *cobra.Command {
 				RegisterRestoreItemAction("velero.io/crd-preserve-fields", newCRDV1PreserveUnknownFieldsItemAction).
 				RegisterRestoreItemAction("velero.io/change-pvc-node-selector", newChangePVCNodeSelectorItemAction(f)).
 				RegisterRestoreItemAction("velero.io/apiservice", newAPIServiceRestoreItemAction).
-				RegisterRestoreItemAction("velero.io/admission-webhook-configuration", newAdmissionWebhookConfigurationAction)
+				RegisterRestoreItemAction("velero.io/admission-webhook-configuration", newAdmissionWebhookConfigurationAction).
+				RegisterRestoreItemAction("velero.io/secret", newSecretRestoreItemAction(f))
 			if !features.IsEnabled(velerov1api.APIGroupVersionsFeatureFlag) {
 				// Do not register crd-remap-version BIA if the API Group feature flag is enabled, so that the v1 CRD can be backed up
 				pluginServer = pluginServer.RegisterBackupItemAction("velero.io/crd-remap-version", newRemapCRDVersionAction(f))
@@ -234,3 +235,13 @@ func newAPIServiceRestoreItemAction(logger logrus.FieldLogger) (interface{}, err
 func newAdmissionWebhookConfigurationAction(logger logrus.FieldLogger) (interface{}, error) {
 	return restore.NewAdmissionWebhookConfigurationAction(logger), nil
 }
+
+func newSecretRestoreItemAction(f client.Factory) plugincommon.HandlerInitializer {
+	return func(logger logrus.FieldLogger) (interface{}, error) {
+		client, err := f.KubebuilderClient()
+		if err != nil {
+			return nil, err
+		}
+		return restore.NewSecretAction(logger, client), nil
+	}
+}

pkg/cmd/server/server.go

@@ -499,9 +499,9 @@ func (s *server) veleroResourcesExist() error {
 // - VolumeSnapshots are needed to create PVCs using the VolumeSnapshot as their data source.
 // - PVs go before PVCs because PVCs depend on them.
 // - PVCs go before pods or controllers so they can be mounted as volumes.
+// - Service accounts go before secrets so service account token secrets can be filled automatically.
 // - Secrets and config maps go before pods or controllers so they can be mounted
 // 	 as volumes.
-// - Service accounts go before pods or controllers so pods can use them.
 // - Limit ranges go before pods or controllers so pods can use them.
 // - Pods go before controllers so they can be explicitly restored and potentially
 //	 have pod volume restores run before controllers adopt the pods.
@@ -525,9 +525,9 @@ var defaultRestorePriorities = restore.Priorities{
 		"volumesnapshots.snapshot.storage.k8s.io",
 		"persistentvolumes",
 		"persistentvolumeclaims",
+		"serviceaccounts",
 		"secrets",
 		"configmaps",
-		"serviceaccounts",
 		"limitranges",
 		"pods",
 		// we fully qualify replicasets.apps because prior to Kubernetes 1.16, replicasets also

pkg/restore/secret_action.go

@@ -0,0 +1,107 @@
+/*
+Copyright The Velero Contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package restore
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	"github.com/vmware-tanzu/velero/pkg/plugin/velero"
+	"github.com/vmware-tanzu/velero/pkg/util/kube"
+)
+
+// SecretAction is a restore item action for secrets
+type SecretAction struct {
+	logger logrus.FieldLogger
+	client client.Client
+}
+
+// NewSecretAction creates a new SecretAction instance
+func NewSecretAction(logger logrus.FieldLogger, client client.Client) *SecretAction {
+	return &SecretAction{
+		logger: logger,
+		client: client,
+	}
+}
+
+// AppliesTo indicates which resources this action applies
+func (s *SecretAction) AppliesTo() (velero.ResourceSelector, error) {
+	return velero.ResourceSelector{
+		IncludedResources: []string{"secrets"},
+	}, nil
+}
+
+// Execute the action
+func (s *SecretAction) Execute(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) {
+	s.logger.Info("Executing SecretAction")
+	defer s.logger.Info("Done executing SecretAction")
+
+	var secret corev1.Secret
+	if err := runtime.DefaultUnstructuredConverter.FromUnstructured(input.Item.UnstructuredContent(), &secret); err != nil {
+		return nil, errors.Wrap(err, "unable to convert secret from runtime.Unstructured")
+	}
+
+	log := s.logger.WithField("secret", kube.NamespaceAndName(&secret))
+	if secret.Type != corev1.SecretTypeServiceAccountToken {
+		log.Debug("No match found - including this secret")
+		return &velero.RestoreItemActionExecuteOutput{
+			UpdatedItem: input.Item,
+		}, nil
+	}
+
+	// The auto created service account token secret will be created by kube controller automatically again(before Kubernetes v1.22), no need to restore.
+	// This will cause the patch operation of managedFields failed if we restore it as the secret is removed immediately
+	// after restoration and the patch operation reports not found error.
+	list := &corev1.ServiceAccountList{}
+	if err := s.client.List(context.Background(), list, &client.ListOptions{Namespace: secret.Namespace}); err != nil {
+		return nil, errors.Wrap(err, "unable to list the service accounts")
+	}
+	for _, sa := range list.Items {
+		if strings.HasPrefix(secret.Name, fmt.Sprintf("%s-token-", sa.Name)) {
+			log.Debug("auto created service account token secret found - excluding this secret")
+			return &velero.RestoreItemActionExecuteOutput{
+				UpdatedItem: input.Item,
+				SkipRestore: true,
+			}, nil
+		}
+	}
+
+	log.Debug("service account token secret(not auto created) found - remove some fields from this secret")
+	// If the annotation and data are not removed, the secret cannot be restored successfully.
+	// The kube controller will fill the annotation and data with new value automatically:
+	// https://kubernetes.io/docs/concepts/configuration/secret/#service-account-token-secrets
+	delete(secret.Annotations, "kubernetes.io/service-account.uid")
+	delete(secret.Data, "token")
+	delete(secret.Data, "ca.crt")
+
+	res, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&secret)
+	if err != nil {
+		return nil, errors.Wrap(err, "unable to convert secret to runtime.Unstructured")
+	}
+
+	return &velero.RestoreItemActionExecuteOutput{
+		UpdatedItem: &unstructured.Unstructured{Object: res},
+	}, nil
+}

pkg/restore/secret_action_test.go

@@ -0,0 +1,142 @@
+/*
+Copyright The Velero Contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package restore
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+
+	"github.com/vmware-tanzu/velero/pkg/plugin/velero"
+	"github.com/vmware-tanzu/velero/pkg/test"
+)
+
+func TestSecretActionAppliesTo(t *testing.T) {
+	action := NewSecretAction(test.NewLogger(), nil)
+	actual, err := action.AppliesTo()
+	require.NoError(t, err)
+	assert.Equal(t, velero.ResourceSelector{IncludedResources: []string{"secrets"}}, actual)
+}
+
+func TestSecretActionExecute(t *testing.T) {
+	tests := []struct {
+		name           string
+		input          *corev1.Secret
+		serviceAccount *corev1.ServiceAccount
+		skipped        bool
+		output         *corev1.Secret
+	}{
+		{
+			name: "not service account token secret",
+			input: &corev1.Secret{
+				ObjectMeta: metav1.ObjectMeta{
+					Namespace: "foo",
+					Name:      "default-token-sfafa",
+				},
+				Type: corev1.SecretTypeOpaque,
+			},
+			skipped: false,
+			output: &corev1.Secret{
+				ObjectMeta: metav1.ObjectMeta{
+					Namespace: "foo",
+					Name:      "default-token-sfafa",
+				},
+				Type: corev1.SecretTypeOpaque,
+			},
+		},
+		{
+			name: "auto created service account token",
+			input: &corev1.Secret{
+				ObjectMeta: metav1.ObjectMeta{
+					Namespace: "foo",
+					Name:      "default-token-sfafa",
+				},
+				Type: corev1.SecretTypeServiceAccountToken,
+			},
+			serviceAccount: &corev1.ServiceAccount{
+				ObjectMeta: metav1.ObjectMeta{
+					Namespace: "foo",
+					Name:      "default",
+				},
+			},
+			skipped: true,
+		},
+		{
+			name: "not auto created service account token",
+			input: &corev1.Secret{
+				ObjectMeta: metav1.ObjectMeta{
+					Namespace: "foo",
+					Name:      "my-token",
+					Annotations: map[string]string{
+						"kubernetes.io/service-account.uid": "uid",
+						"key":                               "value",
+					},
+				},
+				Type: corev1.SecretTypeServiceAccountToken,
+				Data: map[string][]byte{
+					"token":  []byte("token"),
+					"ca.crt": []byte("ca"),
+					"key":    []byte("value"),
+				},
+			},
+			skipped: false,
+			output: &corev1.Secret{
+				ObjectMeta: metav1.ObjectMeta{
+					Namespace: "foo",
+					Name:      "my-token",
+					Annotations: map[string]string{
+						"key": "value",
+					},
+				},
+				Type: corev1.SecretTypeServiceAccountToken,
+				Data: map[string][]byte{
+					"key": []byte("value"),
+				},
+			},
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			secretUnstructured, err := runtime.DefaultUnstructuredConverter.ToUnstructured(tc.input)
+			require.NoError(t, err)
+			var serviceAccounts []client.Object
+			if tc.serviceAccount != nil {
+				serviceAccounts = append(serviceAccounts, tc.serviceAccount)
+			}
+			client := fake.NewClientBuilder().WithObjects(serviceAccounts...).Build()
+			action := NewSecretAction(test.NewLogger(), client)
+			res, err := action.Execute(&velero.RestoreItemActionExecuteInput{
+				Item: &unstructured.Unstructured{Object: secretUnstructured},
+			})
+			require.NoError(t, err)
+			assert.Equal(t, tc.skipped, res.SkipRestore)
+			if !tc.skipped {
+				r, err := runtime.DefaultUnstructuredConverter.ToUnstructured(tc.output)
+				require.NoError(t, err)
+				assert.EqualValues(t, &unstructured.Unstructured{Object: r}, res.UpdatedItem)
+			}
+		})
+	}
+}