AWS Paging Support

- Adding in paging support for the S3 and Snapshot
AWS integration.

As a testing note, you can add in a a MaxKeys to the S3
request as an easy way to ensure that paging is working
properly without having to creation over 1k backups.

Signed-off-by: Justin Nauman <justin.r.nauman@gmail.com>

pkg/cloudprovider/aws/block_storage_adapter.go

@@ -166,17 +166,18 @@ func (op *blockStorageAdapter) ListSnapshots(tagFilters map[string]string) ([]st
 		req.Filters = append(req.Filters, filter)
 	}
 
-	res, err := op.ec2.DescribeSnapshots(req)
+	var ret []string
+	err := op.ec2.DescribeSnapshotsPages(req, func (res *ec2.DescribeSnapshotsOutput, lastPage bool) bool {
+		for _, snapshot := range res.Snapshots {
+			ret = append(ret, *snapshot.SnapshotId)
+		}
+
+		return !lastPage
+	})
 	if err != nil {
 		return nil, err
 	}
 
-	var ret []string
-
-	for _, snapshot := range res.Snapshots {
-		ret = append(ret, *snapshot.SnapshotId)
-	}
-
 	return ret, nil
 }
 

pkg/cloudprovider/aws/object_storage_adapter.go

@@ -106,17 +106,18 @@ func (op *objectStorageAdapter) ListCommonPrefixes(bucket string, delimiter stri
 		Delimiter: &delimiter,
 	}
 
-	res, err := op.s3.ListObjectsV2(req)
+	var ret []string
+	err := op.s3.ListObjectsV2Pages(req, func(res *s3.ListObjectsV2Output, lastPage bool) bool {
+		for _, prefix := range res.CommonPrefixes {
+			ret = append(ret, *prefix.Prefix)
+		}
+		return !lastPage
+	})
+
 	if err != nil {
 		return nil, err
 	}
 
-	ret := make([]string, 0, len(res.CommonPrefixes))
-
-	for _, prefix := range res.CommonPrefixes {
-		ret = append(ret, *prefix.Prefix)
-	}
-
 	return ret, nil
 }