Adding in support for backup download

- Adding in support for a new `download` subcommand of backup
- Adjusted signing to allow for multiple types
- Adding in git sha version during build more granular version
  debugging

Signed-off-by: Justin Nauman <justin.r.nauman@gmail.com>

.gitignore

@@ -27,3 +27,4 @@ _testmain.go
 debug
 
 /ark
+.idea/

Makefile

@@ -19,6 +19,11 @@ VERSION ?= v0.3.3
 GOTARGET = github.com/heptio/$(PROJECT)
 OUTPUT_DIR = $(ROOT_DIR)/_output
 BIN_DIR = $(OUTPUT_DIR)/bin
+GIT_SHA=$(shell git rev-parse --short HEAD)
+GIT_DIRTY=$(shell git status --porcelain $(ROOT_DIR) 2> /dev/null)
+ifneq ($(GIT_DIRTY),)
+	GIT_SHA := $(GIT_SHA)-dirty
+endif
 
 # docker related vars
 DOCKER ?= docker
@@ -26,7 +31,7 @@ REGISTRY ?= gcr.io/heptio-images
 BUILD_IMAGE ?= gcr.io/heptio-images/golang:1.8-alpine3.6
 # go build -i installs compiled packages so they can be reused later.
 # This speeds up recompiles.
-BUILDCMD = go build -i -v -ldflags "-X $(GOTARGET)/pkg/buildinfo.Version=$(VERSION) -X $(GOTARGET)/pkg/buildinfo.DockerImage=$(REGISTRY)/$(PROJECT)"
+BUILDCMD = go build -i -v -ldflags "-X $(GOTARGET)/pkg/buildinfo.Version=$(VERSION) -X $(GOTARGET)/pkg/buildinfo.DockerImage=$(REGISTRY)/$(PROJECT) -X $(GOTARGET)/pkg/buildinfo.GitSHA=$(GIT_SHA)"
 BUILDMNT = /go/src/$(GOTARGET)
 EXTRA_MNTS ?=
 
@@ -44,6 +49,10 @@ $(BINARIES):
 	mkdir -p $(BIN_DIR)
 	$(BUILDCMD) -o $(BIN_DIR)/$@ $(GOTARGET)/cmd/$@
 
+fmt:
+	gofmt -w=true $$(find . -type f -name '*.go' -not -path "./vendor/*" -not -path "./pkg/generated/*")
+	goimports -w=true -d $$(find . -type f -name '*.go' -not -path "./vendor/*" -not -path "./pkg/generated/*")
+
 test:
 ifneq ($(SKIP_TESTS), 1)
 # go test -i installs compiled packages so they can be reused later
@@ -60,7 +69,7 @@ ifneq ($(SKIP_TESTS), 1)
 	${ROOT_DIR}/hack/verify-generated-informers.sh
 endif
 
-update:
+update: fmt
 	${ROOT_DIR}/hack/update-generated-clientsets.sh
 	${ROOT_DIR}/hack/update-generated-listers.sh
 	${ROOT_DIR}/hack/update-generated-informers.sh
@@ -80,7 +89,7 @@ container-local: $(BINARIES)
 push:
 	docker -- push $(REGISTRY)/$(PROJECT):$(VERSION)
 
-.PHONY: all local container cbuild push test verify update $(BINARIES)
+.PHONY: all local container cbuild push test verify update fmt $(BINARIES)
 
 clean:
 	rm -rf $(OUTPUT_DIR)

docs/cli-reference/ark_backup.md

@@ -29,6 +29,7 @@ Work with backups
 ### SEE ALSO
 * [ark](ark.md)	 - Back up and restore Kubernetes cluster resources.
 * [ark backup create](ark_backup_create.md)	 - Create a backup
+* [ark backup download](ark_backup_download.md)	 - Download a backup
 * [ark backup get](ark_backup_get.md)	 - Get backups
 * [ark backup logs](ark_backup_logs.md)	 - Get backup logs
 

docs/cli-reference/ark_backup_download.md

@@ -0,0 +1,38 @@
+## ark backup download
+
+Download a backup
+
+### Synopsis
+
+
+Download a backup
+
+```
+ark backup download NAME [flags]
+```
+
+### Options
+
+```
+      --force               forces the download and will overwrite file if it exists already
+  -h, --help                help for download
+      --output-dir string   directory to download backup to. (Default cwd)
+      --timeout duration    maximum time to wait to process download request (default 1m0s)
+```
+
+### Options inherited from parent commands
+
+```
+      --alsologtostderr                  log to standard error as well as files
+      --kubeconfig string                Path to the kubeconfig file to use to talk to the Kubernetes apiserver. If unset, try the environment variable KUBECONFIG, as well as in-cluster configuration
+      --log_backtrace_at traceLocation   when logging hits line file:N, emit a stack trace (default :0)
+      --log_dir string                   If non-empty, write log files in this directory
+      --logtostderr                      log to standard error instead of files
+      --stderrthreshold severity         logs at or above this threshold go to stderr (default 2)
+  -v, --v Level                          log level for V logs
+      --vmodule moduleSpec               comma-separated list of pattern=N settings for file-filtered logging
+```
+
+### SEE ALSO
+* [ark backup](ark_backup.md)	 - Work with backups
+

docs/generate/ark.go

@@ -18,17 +18,17 @@ package main
 
 import (
 	"log"
-  "os"
+	"os"
 
-	"github.com/spf13/cobra/doc"
 	"github.com/heptio/ark/pkg/cmd/ark"
+	"github.com/spf13/cobra/doc"
 )
 
 func main() {
-  cmdName := os.Args[1]
+	cmdName := os.Args[1]
 	outputDir := os.Args[2]
 
-  cmd := ark.NewCommand(cmdName)
+	cmd := ark.NewCommand(cmdName)
 	// Remove auto-generated timestamps
 	cmd.DisableAutoGenTag = true
 

pkg/apis/ark/v1/download_request.go

@@ -28,7 +28,8 @@ type DownloadRequestSpec struct {
 type DownloadTargetKind string
 
 const (
-	DownloadTargetKindBackupLog DownloadTargetKind = "BackupLog"
+	DownloadTargetKindBackupLog      DownloadTargetKind = "BackupLog"
+	DownloadTargetKindBackupContents DownloadTargetKind = "BackupContents"
 )
 
 // DownloadTarget is the specification for what kind of file to download, and the name of the

pkg/buildinfo/version.go

@@ -25,3 +25,6 @@ var Version string
 // DockerImage is the full path to the docker image for this build, for example
 // gcr.io/heptio-images/ark.
 var DockerImage string
+
+// GitSHA is the actual commit that is being built, set by the go linker's -X flag at build time.
+var GitSHA string

pkg/cloudprovider/backup_service.go

@@ -53,7 +53,7 @@ type BackupService interface {
 
 	// CreateBackupLogSignedURL creates a pre-signed URL that can be used to download a backup's log
 	// file from object storage. The URL expires after ttl.
-	CreateBackupLogSignedURL(bucket, backupName string, ttl time.Duration) (string, error)
+	CreateBackupSignedURL(backupType api.DownloadTargetKind, bucket, backupName string, ttl time.Duration) (string, error)
 }
 
 // BackupGetter knows how to list backups in object storage.
@@ -72,7 +72,7 @@ func getMetadataKey(backup string) string {
 	return fmt.Sprintf(metadataFileFormatString, backup)
 }
 
-func getBackupKey(backup string) string {
+func getBackupContentsKey(backup string) string {
 	return fmt.Sprintf(backupFileFormatString, backup, backup)
 }
 
@@ -105,7 +105,7 @@ func (br *backupService) UploadBackup(bucket, backupName string, metadata, backu
 	}
 
 	// upload tar file
-	if err := br.objectStorage.PutObject(bucket, getBackupKey(backupName), backup); err != nil {
+	if err := br.objectStorage.PutObject(bucket, getBackupContentsKey(backupName), backup); err != nil {
 		// try to delete the metadata file since the data upload failed
 		deleteErr := br.objectStorage.DeleteObject(bucket, metadataKey)
 
@@ -123,7 +123,7 @@ func (br *backupService) UploadBackup(bucket, backupName string, metadata, backu
 }
 
 func (br *backupService) DownloadBackup(bucket, backupName string) (io.ReadCloser, error) {
-	return br.objectStorage.GetObject(bucket, getBackupKey(backupName))
+	return br.objectStorage.GetObject(bucket, getBackupContentsKey(backupName))
 }
 
 func (br *backupService) GetAllBackups(bucket string) ([]*api.Backup, error) {
@@ -194,8 +194,15 @@ func (br *backupService) DeleteBackupDir(bucket, backupName string) error {
 	return errors.NewAggregate(errs)
 }
 
-func (br *backupService) CreateBackupLogSignedURL(bucket, backupName string, ttl time.Duration) (string, error) {
-	return br.objectStorage.CreateSignedURL(bucket, getBackupLogKey(backupName), ttl)
+func (br *backupService) CreateBackupSignedURL(backupType api.DownloadTargetKind, bucket, backupName string, ttl time.Duration) (string, error) {
+	switch backupType {
+	case api.DownloadTargetKindBackupContents:
+		return br.objectStorage.CreateSignedURL(bucket, getBackupContentsKey(backupName), ttl)
+	case api.DownloadTargetKindBackupLog:
+		return br.objectStorage.CreateSignedURL(bucket, getBackupLogKey(backupName), ttl)
+	default:
+		return "", fmt.Errorf("unsupported download target kind %q", backupType)
+	}
 }
 
 // cachedBackupService wraps a real backup service with a cache for getting cloud backups.

pkg/cmd/cli/backup/backup.go

@@ -33,6 +33,7 @@ func NewCommand(f client.Factory) *cobra.Command {
 		NewCreateCommand(f),
 		NewGetCommand(f),
 		NewLogsCommand(f),
+		NewDownloadCommand(f),
 
 		// Will implement describe later
 		// NewDescribeCommand(f),

pkg/cmd/cli/backup/download.go

@@ -0,0 +1,113 @@
+/*
+Copyright 2017 Heptio Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package backup
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path"
+	"time"
+
+	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
+
+	"github.com/heptio/ark/pkg/apis/ark/v1"
+	"github.com/heptio/ark/pkg/client"
+	"github.com/heptio/ark/pkg/cmd"
+	"github.com/heptio/ark/pkg/cmd/util/downloadrequest"
+)
+
+func NewDownloadCommand(f client.Factory) *cobra.Command {
+	o := NewDownloadOptions()
+	c := &cobra.Command{
+		Use:   "download NAME",
+		Short: "Download a backup",
+		Run: func(c *cobra.Command, args []string) {
+			cmd.CheckError(o.Validate(c, args))
+			cmd.CheckError(o.Complete(args))
+			cmd.CheckError(o.Run(c, f))
+		},
+	}
+
+	o.BindFlags(c.Flags())
+
+	return c
+}
+
+type DownloadOptions struct {
+	Name         string
+	Path         string
+	Force        bool
+	Timeout      time.Duration
+	writeOptions int
+}
+
+func NewDownloadOptions() *DownloadOptions {
+	return &DownloadOptions{
+		Timeout: time.Minute,
+	}
+}
+
+func (o *DownloadOptions) BindFlags(flags *pflag.FlagSet) {
+	flags.StringVar(&o.Path, "output-dir", "", "directory to download backup to. (Default cwd)")
+	flags.BoolVar(&o.Force, "force", o.Force, "forces the download and will overwrite file if it exists already")
+	flags.DurationVar(&o.Timeout, "timeout", o.Timeout, "maximum time to wait to process download request")
+}
+
+func (o *DownloadOptions) Validate(c *cobra.Command, args []string) error {
+	if len(args) != 1 {
+		return errors.New("backup name is required")
+	}
+
+	o.writeOptions = os.O_RDWR | os.O_CREATE | os.O_EXCL
+	if o.Force {
+		o.writeOptions = os.O_RDWR | os.O_CREATE | os.O_TRUNC
+	}
+
+	if o.Path == "" {
+		path, err := os.Getwd()
+		if err != nil {
+			return errors.New("an issue occurred attempting to determine the current working directory.")
+		}
+		o.Path = path
+	}
+
+	return nil
+}
+
+func (o *DownloadOptions) Complete(args []string) error {
+	o.Name = args[0]
+	return nil
+}
+
+func (o *DownloadOptions) Run(c *cobra.Command, f client.Factory) error {
+	arkClient, err := f.Client()
+	cmd.CheckError(err)
+
+	backupDest, err := os.OpenFile(path.Join(o.Path, fmt.Sprintf("%s-data.tar.gz", o.Name)), o.writeOptions, 0600)
+	if err != nil {
+		return err
+	}
+	defer backupDest.Close()
+
+	err = downloadrequest.Stream(arkClient.ArkV1(), o.Name, v1.DownloadTargetKindBackupContents, backupDest, o.Timeout)
+	cmd.CheckError(err)
+
+	fmt.Printf("Backup %s has been successfully downloaded to %s\n", o.Name, backupDest.Name())
+	return nil
+}

pkg/cmd/version/version.go

@@ -29,7 +29,7 @@ func NewCommand() *cobra.Command {
 		Use:   "version",
 		Short: "Print the ark version and associated image",
 		Run: func(cmd *cobra.Command, args []string) {
-			fmt.Println(buildinfo.Version)
+			fmt.Printf("Version: [%s] - [%s]\n", buildinfo.Version, buildinfo.GitSHA)
 			fmt.Println("Configured docker image:", buildinfo.DockerImage)
 		},
 	}

pkg/controller/download_request_controller.go

@@ -216,13 +216,13 @@ const signedURLTTL = 10 * time.Minute
 // Processed, and persists the changes to storage.
 func (c *downloadRequestController) generatePreSignedURL(downloadRequest *v1.DownloadRequest) error {
 	switch downloadRequest.Spec.Target.Kind {
-	case v1.DownloadTargetKindBackupLog:
+	case v1.DownloadTargetKindBackupLog, v1.DownloadTargetKindBackupContents:
 		update, err := cloneDownloadRequest(downloadRequest)
 		if err != nil {
 			return err
 		}
 
-		update.Status.DownloadURL, err = c.backupService.CreateBackupLogSignedURL(c.bucket, update.Spec.Target.Name, signedURLTTL)
+		update.Status.DownloadURL, err = c.backupService.CreateBackupSignedURL(downloadRequest.Spec.Target.Kind, c.bucket, update.Spec.Target.Name, signedURLTTL)
 		if err != nil {
 			return err
 		}
@@ -233,8 +233,8 @@ func (c *downloadRequestController) generatePreSignedURL(downloadRequest *v1.Dow
 		_, err = c.downloadRequestClient.DownloadRequests(update.Namespace).Update(update)
 		return err
 	}
-
 	return fmt.Errorf("unsupported download target kind %q", downloadRequest.Spec.Target.Kind)
+
 }
 
 // deleteIfExpired deletes downloadRequest if it has expired.

pkg/util/test/backup_service.go

@@ -28,7 +28,7 @@ type BackupService struct {
 }
 
 // CreateBackupLogSignedURL provides a mock function with given fields: bucket, backupName, ttl
-func (_m *BackupService) CreateBackupLogSignedURL(bucket string, backupName string, ttl time.Duration) (string, error) {
+func (_m *BackupService) CreateBackupSignedURL(backupType v1.DownloadTargetKind, bucket string, backupName string, ttl time.Duration) (string, error) {
 	ret := _m.Called(bucket, backupName, ttl)
 
 	var r0 string