Merge pull request #5478 from Lyndon-Li/s3-compatible-provider

Issue fix 5477

changelogs/unreleased/5478-lyndon

@@ -0,0 +1 @@
+Issue fix 5477: create the common way to support S3 compatible object storages that work for both Restic and Kopia; Keep the resticRepoPrefix parameter for compatibility

pkg/repository/config/config.go

@@ -56,7 +56,7 @@ func getRepoPrefix(location *velerov1api.BackupStorageLocation) (string, error)
 		prefix = layout.GetResticDir()
 	}
 
-	backendType := GetBackendType(location.Spec.Provider)
+	backendType := GetBackendType(location.Spec.Provider, location.Spec.Config)
 
 	if repoPrefix := location.Spec.Config["resticRepoPrefix"]; repoPrefix != "" {
 		return repoPrefix, nil
@@ -87,15 +87,25 @@ func getRepoPrefix(location *velerov1api.BackupStorageLocation) (string, error)
 		return fmt.Sprintf("gs:%s:/%s", bucket, prefix), nil
 	}
 
-	return "", errors.New("restic repository prefix (resticRepoPrefix) not specified in backup storage location's config")
+	return "", errors.Errorf("invalid backend type %s, provider %s", backendType, location.Spec.Provider)
 }
 
-func GetBackendType(provider string) BackendType {
+// GetBackendType returns a backend type that is known by Velero.
+// If the provider doesn't indicate a known backend type, but the endpoint is
+// specified, Velero regards it as a S3 compatible object store and return AWSBackend as the type.
+func GetBackendType(provider string, config map[string]string) BackendType {
 	if !strings.Contains(provider, "/") {
 		provider = "velero.io/" + provider
 	}
 
-	return BackendType(provider)
+	bt := BackendType(provider)
+	if IsBackendTypeValid(bt) {
+		return bt
+	} else if config != nil && config["s3Url"] != "" {
+		return AWSBackend
+	} else {
+		return bt
+	}
 }
 
 func IsBackendTypeValid(backendType BackendType) bool {

pkg/repository/config/config_test.go

@@ -48,7 +48,7 @@ func TestGetRepoIdentifier(t *testing.T) {
 				},
 			},
 			repoName:    "repo-1",
-			expectedErr: "restic repository prefix (resticRepoPrefix) not specified in backup storage location's config",
+			expectedErr: "invalid backend type velero.io/unsupported-provider, provider unsupported-provider",
 		},
 		{
 			name: "resticRepoPrefix in BSL config is used if set",
@@ -69,6 +69,25 @@ func TestGetRepoIdentifier(t *testing.T) {
 			repoName: "repo-1",
 			expected: "custom:prefix:/restic/repo-1",
 		},
+		{
+			name: "s3Url in BSL config is used",
+			bsl: &velerov1api.BackupStorageLocation{
+				Spec: velerov1api.BackupStorageLocationSpec{
+					Provider: "custom-repo-identifier",
+					Config: map[string]string{
+						"s3Url": "s3Url",
+					},
+					StorageType: velerov1api.StorageType{
+						ObjectStorage: &velerov1api.ObjectStorageLocation{
+							Bucket: "bucket",
+							Prefix: "prefix",
+						},
+					},
+				},
+			},
+			repoName: "repo-1",
+			expected: "s3:s3Url/bucket/prefix/restic/repo-1",
+		},
 		{
 			name: "s3.amazonaws.com URL format is used if region cannot be determined for AWS BSL",
 			bsl: &velerov1api.BackupStorageLocation{

pkg/repository/provider/unified_repo.go

@@ -344,7 +344,7 @@ func getRepoPassword(secretStore credentials.SecretStore) (string, error) {
 }
 
 func getStorageType(backupLocation *velerov1api.BackupStorageLocation) string {
-	backendType := repoconfig.GetBackendType(backupLocation.Spec.Provider)
+	backendType := repoconfig.GetBackendType(backupLocation.Spec.Provider, backupLocation.Spec.Config)
 
 	switch backendType {
 	case repoconfig.AWSBackend:
@@ -368,7 +368,7 @@ func getStorageCredentials(backupLocation *velerov1api.BackupStorageLocation, cr
 		return map[string]string{}, errors.New("invalid credentials interface")
 	}
 
-	backendType := repoconfig.GetBackendType(backupLocation.Spec.Provider)
+	backendType := repoconfig.GetBackendType(backupLocation.Spec.Provider, backupLocation.Spec.Config)
 	if !repoconfig.IsBackendTypeValid(backendType) {
 		return map[string]string{}, errors.New("invalid storage provider")
 	}
@@ -414,7 +414,7 @@ func getStorageCredentials(backupLocation *velerov1api.BackupStorageLocation, cr
 func getStorageVariables(backupLocation *velerov1api.BackupStorageLocation, repoBackend string, repoName string) (map[string]string, error) {
 	result := make(map[string]string)
 
-	backendType := repoconfig.GetBackendType(backupLocation.Spec.Provider)
+	backendType := repoconfig.GetBackendType(backupLocation.Spec.Provider, backupLocation.Spec.Config)
 	if !repoconfig.IsBackendTypeValid(backendType) {
 		return map[string]string{}, errors.New("invalid storage provider")
 	}

pkg/restic/common.go

@@ -92,7 +92,7 @@ func CmdEnv(backupLocation *velerov1api.BackupStorageLocation, credentialFileSto
 		config[repoconfig.CredentialsFileKey] = credsFile
 	}
 
-	backendType := repoconfig.GetBackendType(backupLocation.Spec.Provider)
+	backendType := repoconfig.GetBackendType(backupLocation.Spec.Provider, backupLocation.Spec.Config)
 
 	switch backendType {
 	case repoconfig.AWSBackend: