backup repo config

Signed-off-by: Lyndon-Li <lyonghui@vmware.com>

changelogs/unreleased/8093-Lyndon-Li

@@ -0,0 +1 @@
+Fix issue #7620, add backup repository configuration implementation and support cacheLimit configuration for Kopia repo

design/backup-repo-config.md

@@ -86,18 +86,6 @@ For any reason, if the configMap doesn't effect, nothing is specified to the bac
 The BackupRepository configMap supports backup repository type specific configurations, even though users can only specify one configMap.  
 So in the configMap struct, multiple entries are supported, indexed by the backup repository type. During the backup repository creation, the configMap is searched by the repository type.  
 
-Below are the struct for the configMap:  
-``` golang
-type RepoConfig struct {
-	CacheLimitMB int `json:"cacheLimitMB,omitempty"`
-	EnableCompression int `json:"enableCompression,omitempty"`
-}
-
-type RepoConfigs struct {
-	Configs map[string]RepoConfig `json:"configs"`
-}
-```
-
 ### Configurations
 
 With the above mechanisms, any kind of configuration could be added. Here list the configurations defined at present:  
@@ -105,23 +93,7 @@ With the above mechanisms, any kind of configuration could be added. Here list t
 ```enableCompression```: specifies to enable/disable compression for a backup repsotiory. Most of the backup repositories support the data compression feature, if it is not supported by a backup repository, this parameter is ignored. Most of the backup repositories support to dynamically enable/disable compression, so this parameter is defined to be used whenever creating a write connection to the backup repository, if the dynamically changing is not supported, this parameter will be hornored only when initializing the backup repository. For Kopia repository, this parameter is supported and can be dynamically modified.  
 
 ### Sample
-Below is an example of the BackupRepository configMap with the configurations:  
-json format:  
-```json
-{
-  "configs": {
-    "repo-type-1": {
-      "cacheLimitMB": 2048,
-      "enableCompression": true    
-    },
-    "repo-type-2": {
-      "cacheLimitMB": 1024,
-      "enableCompression": false    
-    }
-  }
-}
-```  
-yaml format:  
+Below is an example of the BackupRepository configMap with the configurations:     
 ```yaml
 apiVersion: v1
 kind: ConfigMap
@@ -129,25 +101,20 @@ metadata:
   name: <config-name>
   namespace: velero
 data:
-  configs: |
+  <repository-type-1>: |
     {
-      "repo-type-1": {
-        "cacheLimitMB": 2048,
-        "enableCompression": true    
-      },
-      "repo-type-2": {
-        "cacheLimitMB": 1024,
-        "enableCompression": false    
-      }
-    }    
+      "cacheLimitMB": 2048,
+      "enableCompression": true    
+    }
+  <repository-type-2>: |
+    {
+      "cacheLimitMB": 1,
+      "enableCompression": false    
+    }        
 ```
 
 To create the configMap, users need to save something like the above sample to a file and then run below commands:  
 ```
-kubectl create cm <config-name> -n velero --from-file=<json file name>
-```  
-Or
-```
 kubectl apply -f <yaml file name>
 ```  
 

pkg/controller/backup_repository_controller.go

@@ -246,7 +246,7 @@ func (r *BackupRepoReconciler) initializeRepo(ctx context.Context, req *velerov1
 
 	config, err := getBackupRepositoryConfig(ctx, r, r.backukpRepoConfig, r.namespace, req.Name, req.Spec.RepositoryType, log)
 	if err != nil {
-		log.WithError(err).Warnf("Failed to get repo config from %s for repo %s, repo config is ignored", r.backukpRepoConfig, req.Name)
+		log.WithError(err).Warn("Failed to get repo config, repo config is ignored")
 	} else if config != nil {
 		log.Infof("Init repo with config %v", config)
 	}

pkg/repository/provider/unified_repo_test.go

@@ -452,11 +452,11 @@ func TestGetStorageVariables(t *testing.T) {
 				udmrepo.StoreOptionCacheLimit: "1000",
 			},
 			expected: map[string]string{
-				"fspath":     "fake-path",
-				"bucket":     "",
-				"prefix":     "fake-prefix/fake-repo-type/",
-				"region":     "",
-				"cacheLimit": "1000",
+				"fspath":       "fake-path",
+				"bucket":       "",
+				"prefix":       "fake-prefix/fake-repo-type/",
+				"region":       "",
+				"cacheLimitMB": "1000",
 			},
 		},
 	}

pkg/repository/udmrepo/kopialib/backend/common.go

@@ -33,7 +33,7 @@ import (
 )
 
 const (
-	defaultCacheLimitMB    = 2000
+	defaultCacheLimitMB    = 5000
 	maxCacheDurationSecond = 30
 )
 
@@ -68,7 +68,7 @@ func SetupNewRepositoryOptions(ctx context.Context, flags map[string]string) rep
 func SetupConnectOptions(ctx context.Context, repoOptions udmrepo.RepoOptions) repo.ConnectOptions {
 	cacheLimit := optionalHaveIntWithDefault(ctx, udmrepo.StoreOptionCacheLimit, repoOptions.StorageOptions, defaultCacheLimitMB) << 20
 
-	// 80% for data cache and 20% for metadata cahce and align to KB
+	// 80% for data cache and 20% for metadata cache and align to KB
 	dataCacheLimit := (cacheLimit / 5 * 4) >> 10
 	metadataCacheLimit := (cacheLimit / 5) >> 10
 

pkg/repository/udmrepo/kopialib/backend/common_test.go

@@ -111,9 +111,11 @@ func TestSetupNewRepositoryOptions(t *testing.T) {
 
 func TestSetupConnectOptions(t *testing.T) {
 	defaultCacheOption := content.CachingOptions{
-		ContentCacheSizeBytes:  2000 << 20,
-		MetadataCacheSizeBytes: 2000 << 20,
-		MaxListCacheDuration:   content.DurationSeconds(time.Duration(30) * time.Second),
+		ContentCacheSizeBytes:       3200 << 20,
+		MetadataCacheSizeBytes:      800 << 20,
+		ContentCacheSizeLimitBytes:  4000 << 20,
+		MetadataCacheSizeLimitBytes: 1000 << 20,
+		MaxListCacheDuration:        content.DurationSeconds(time.Duration(30) * time.Second),
 	}
 
 	testCases := []struct {

pkg/repository/udmrepo/kopialib/backend/utils_test.go

@@ -90,3 +90,68 @@ func TestOptionalHaveBool(t *testing.T) {
 		})
 	}
 }
+
+func TestOptionalHaveIntWithDefault(t *testing.T) {
+	var expectMsg string
+	testCases := []struct {
+		name         string
+		key          string
+		flags        map[string]string
+		defaultValue int64
+		logger       *storagemocks.Core
+		retFuncCheck func(mock.Arguments)
+		expectMsg    string
+		retValue     int64
+	}{
+		{
+			name:         "key not exist",
+			key:          "fake-key",
+			flags:        map[string]string{},
+			defaultValue: 2000,
+			retValue:     2000,
+		},
+		{
+			name: "value valid",
+			key:  "fake-key",
+			flags: map[string]string{
+				"fake-key": "1000",
+			},
+			retValue: 1000,
+		},
+		{
+			name: "value invalid",
+			key:  "fake-key",
+			flags: map[string]string{
+				"fake-key": "fake-value",
+			},
+			logger: new(storagemocks.Core),
+			retFuncCheck: func(args mock.Arguments) {
+				ent := args[0].(zapcore.Entry)
+				if ent.Level == zapcore.ErrorLevel {
+					expectMsg = ent.Message
+				}
+			},
+			expectMsg:    "Ignore fake-key, value [fake-value] is invalid, err strconv.ParseInt: parsing \"fake-value\": invalid syntax",
+			defaultValue: 2000,
+			retValue:     2000,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			if tc.logger != nil {
+				tc.logger.On("Enabled", mock.Anything).Return(true)
+				tc.logger.On("Check", mock.Anything, mock.Anything).Run(tc.retFuncCheck).Return(&zapcore.CheckedEntry{})
+			}
+
+			ctx := logging.WithLogger(context.Background(), func(module string) logging.Logger {
+				return zap.New(tc.logger).Sugar()
+			})
+
+			retValue := optionalHaveIntWithDefault(ctx, tc.key, tc.flags, tc.defaultValue)
+
+			require.Equal(t, retValue, tc.retValue)
+			require.Equal(t, tc.expectMsg, expectMsg)
+		})
+	}
+}