AWS Paging Support

- Adding in paging support for the S3 and Snapshot
AWS integration.

As a testing note, you can add in a a MaxKeys to the S3
request as an easy way to ensure that paging is working
properly without having to creation over 1k backups.

Signed-off-by: Justin Nauman <justin.r.nauman@gmail.com>

pkg/cloudprovider/aws/block_storage_adapter.go

@@ -166,17 +166,18 @@ func (op *blockStorageAdapter) ListSnapshots(tagFilters map[string]string) ([]st
 		req.Filters = append(req.Filters, filter)
 	}
 
-	res, err := op.ec2.DescribeSnapshots(req)
-	if err != nil {
-		return nil, err
-	}
-
 	var ret []string
-
+	err := op.ec2.DescribeSnapshotsPages(req, func (res *ec2.DescribeSnapshotsOutput, lastPage bool) bool {
 		for _, snapshot := range res.Snapshots {
 			ret = append(ret, *snapshot.SnapshotId)
 		}
 
+		return !lastPage
+	})
+	if err != nil {
+		return nil, err
+	}
+
 	return ret, nil
 }
 

pkg/cloudprovider/aws/object_storage_adapter.go

@@ -106,16 +106,17 @@ func (op *objectStorageAdapter) ListCommonPrefixes(bucket string, delimiter stri
 		Delimiter: &delimiter,
 	}
 
-	res, err := op.s3.ListObjectsV2(req)
+	var ret []string
-	if err != nil {
+	err := op.s3.ListObjectsV2Pages(req, func(res *s3.ListObjectsV2Output, lastPage bool) bool {
-		return nil, err
-	}
-
-	ret := make([]string, 0, len(res.CommonPrefixes))
-
 		for _, prefix := range res.CommonPrefixes {
 			ret = append(ret, *prefix.Prefix)
 		}
+		return !lastPage
+	})
+
+	if err != nil {
+		return nil, err
+	}
 
 	return ret, nil
 }