Merge pull request #313 from keel-hq/feature/306_private_poll

Feature/306 private poll

registry/registry_test.go

@@ -30,7 +30,7 @@ func TestGet(t *testing.T) {
 	client := New()
 	repo, err := client.Get(Opts{
 		Registry: constants.DefaultDockerRegistry,
-		Name:     "karolisr/keel",
+		Name:     "keelhq/keel",
 	})
 
 	if err != nil {

secrets/secrets.go

@@ -139,6 +139,7 @@ func getPodImagePullSecrets(pod *v1.Pod) []string {
 func (g *DefaultGetter) getCredentialsFromSecret(image *types.TrackedImage) (*types.Credentials, error) {
 
 	credentials := &types.Credentials{}
+	secretFound := false
 
 	for _, secretRef := range image.Secrets {
 		secret, err := g.kubernetesImplementer.Secret(image.Namespace, secretRef)
@@ -178,6 +179,7 @@ func (g *DefaultGetter) getCredentialsFromSecret(image *types.TrackedImage) (*ty
 				}).Error("secrets.defaultGetter: failed to decode secret")
 				continue
 			}
+			secretFound = true
 		case v1.SecretTypeDockerConfigJson:
 			secretDataBts, ok := secret.Data[dockerConfigJSONKey]
 			if !ok {
@@ -190,6 +192,7 @@ func (g *DefaultGetter) getCredentialsFromSecret(image *types.TrackedImage) (*ty
 				}).Warn("secrets.defaultGetter: secret is missing key '.dockerconfigjson', ensure that key exists")
 				continue
 			}
+			secretFound = true
 
 			dockerCfg, err = DecodeDockerCfgJson(secretDataBts)
 			if err != nil {
@@ -202,6 +205,7 @@ func (g *DefaultGetter) getCredentialsFromSecret(image *types.TrackedImage) (*ty
 				}).Error("secrets.defaultGetter: failed to decode secret")
 				continue
 			}
+			secretFound = true
 
 		default:
 			log.WithFields(log.Fields{
@@ -219,7 +223,15 @@ func (g *DefaultGetter) getCredentialsFromSecret(image *types.TrackedImage) (*ty
 		}
 	}
 
-	if len(image.Secrets) > 0 {
+	if secretFound {
+		log.WithFields(log.Fields{
+			"namespace": image.Namespace,
+			"provider":  image.Provider,
+			"registry":  image.Image.Registry(),
+			"image":     image.Image.Repository(),
+			"secrets":   image.Secrets,
+		}).Warn("secrets.defaultGetter.lookupSecrets: secret found but couldn't detect authentication for the desired registry")
+	} else if len(image.Secrets) > 0 {
 		log.WithFields(log.Fields{
 			"namespace": image.Namespace,
 			"provider":  image.Provider,
@@ -235,7 +247,7 @@ func (g *DefaultGetter) getCredentialsFromSecret(image *types.TrackedImage) (*ty
 func credentialsFromConfig(image *types.TrackedImage, cfg DockerCfg) (*types.Credentials, bool) {
 	credentials := &types.Credentials{}
 	found := false
-	
+
 	imageRegistry, err := domainOnly(image.Image.Registry())
 	if err != nil {
 		log.WithFields(log.Fields{
@@ -244,11 +256,12 @@ func credentialsFromConfig(image *types.TrackedImage, cfg DockerCfg) (*types.Cre
 			"error":     err,
 		}).Error("secrets.credentialsFromConfig: failed to parse registry hostname")
 		return credentials, false
-    }
+	}
+
 	// looking for our registry
 	for registry, auth := range cfg {
 		h, err := hostname(registry)
-		
+
 		if err != nil {
 			log.WithFields(log.Fields{
 				"image":     image.Image.Repository(),
@@ -306,7 +319,7 @@ func decodeBase64Secret(authSecret string) (username, password string, err error
 		return
 	}
 
-	parts := strings.Split(string(decoded), ":")
+	parts := strings.SplitN(string(decoded), ":", 2)
 
 	if len(parts) != 2 {
 		return "", "", fmt.Errorf("unexpected auth secret format")
@@ -315,6 +328,10 @@ func decodeBase64Secret(authSecret string) (username, password string, err error
 	return parts[0], parts[1], nil
 }
 
+func EncodeBase64Secret(username, password string) string {
+	return base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", username, password)))
+}
+
 func hostname(registry string) (string, error) {
 	if strings.HasPrefix(registry, "http://") || strings.HasPrefix(registry, "https://") {
 		u, err := url.Parse(registry)
@@ -345,6 +362,7 @@ func decodeSecret(data []byte) (DockerCfg, error) {
 }
 
 func DecodeDockerCfgJson(data []byte) (DockerCfg, error) {
+	// var cfg DockerCfg
 	var cfg DockerCfgJSON
 	err := json.Unmarshal(data, &cfg)
 	if err != nil {
@@ -353,6 +371,12 @@ func DecodeDockerCfgJson(data []byte) (DockerCfg, error) {
 	return cfg.Auths, nil
 }
 
+func EncodeDockerCfgJson(cfg *DockerCfg) ([]byte, error) {
+	return json.Marshal(&DockerCfgJSON{
+		Auths: *cfg,
+	})
+}
+
 // DockerCfgJSON - secret structure when dockerconfigjson is used
 type DockerCfgJSON struct {
 	Auths DockerCfg `json:"auths"`

secrets/secrets_test.go

@@ -452,12 +452,12 @@ func Test_hostname(t *testing.T) {
 			want:    "quay.io",
 			wantErr: false,
 		},
-                {
-                        name:    "withport",
-                        args:    args{registry: "https://example.com:3456"},
-                        want:    "example.com",
-                        wantErr: false,
-                },
+		{
+			name:    "withport",
+			args:    args{registry: "https://example.com:3456"},
+			want:    "example.com",
+			wantErr: false,
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {

tests/acceptance_polling_test.go

@@ -0,0 +1,307 @@
+package tests
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/keel-hq/keel/secrets"
+	"github.com/keel-hq/keel/types"
+	log "github.com/sirupsen/logrus"
+	apps_v1 "k8s.io/api/apps/v1"
+	"k8s.io/api/core/v1"
+	meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+func TestPollingSemverUpdate(t *testing.T) {
+
+	// stop := make(chan struct{})
+	ctx, cancel := context.WithCancel(context.Background())
+	// defer close(ctx)
+	defer cancel()
+
+	// go startKeel(ctx)
+	keel := &KeelCmd{}
+	go func() {
+		err := keel.Start(ctx)
+		if err != nil {
+			log.WithFields(log.Fields{
+				"error": err,
+			}).Error("failed to start Keel process")
+		}
+	}()
+
+	defer func() {
+		err := keel.Stop()
+		if err != nil {
+			log.WithFields(log.Fields{
+				"error": err,
+			}).Error("failed to stop Keel process")
+		}
+	}()
+
+	_, kcs := getKubernetesClient()
+
+	t.Run("UpdateThroughDockerHubPollingA", func(t *testing.T) {
+		// UpdateThroughDockerHubPollingA tests a polling trigger when we have a higher version
+		// but without a pre-release tag and a lower version with pre-release. The version of the deployment
+		// is with pre-prerealse so we should upgrade to that one.
+
+		testNamespace := createNamespaceForTest()
+		defer deleteTestNamespace(testNamespace)
+
+		dep := &apps_v1.Deployment{
+			meta_v1.TypeMeta{},
+			meta_v1.ObjectMeta{
+				Name:      "deployment-1",
+				Namespace: testNamespace,
+				Labels: map[string]string{
+					types.KeelPolicyLabel:  "major",
+					types.KeelTriggerLabel: "poll",
+				},
+				Annotations: map[string]string{
+					types.KeelPollScheduleAnnotation: "@every 2s",
+				},
+			},
+			apps_v1.DeploymentSpec{
+				Selector: &meta_v1.LabelSelector{
+					MatchLabels: map[string]string{
+						"app": "wd-1",
+					},
+				},
+				Template: v1.PodTemplateSpec{
+					ObjectMeta: meta_v1.ObjectMeta{
+						Labels: map[string]string{
+							"app":     "wd-1",
+							"release": "1",
+						},
+					},
+					Spec: v1.PodSpec{
+						Containers: []v1.Container{
+							v1.Container{
+								Name:  "wd-1",
+								Image: "keelhq/push-workflow-example:0.1.0-dev",
+							},
+						},
+					},
+				},
+			},
+			apps_v1.DeploymentStatus{},
+		}
+
+		_, err := kcs.AppsV1().Deployments(testNamespace).Create(dep)
+		if err != nil {
+			t.Fatalf("failed to create deployment: %s", err)
+		}
+		ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
+		defer cancel()
+
+		err = waitFor(ctx, kcs, testNamespace, dep.ObjectMeta.Name, "keelhq/push-workflow-example:0.5.0-dev")
+		if err != nil {
+			t.Errorf("update failed: %s", err)
+		}
+	})
+
+	t.Run("UpdateThroughDockerHubPollingB", func(t *testing.T) {
+		// UpdateThroughDockerHubPollingA tests a polling trigger when we have a higher version
+		// but without a pre-release tag and a lower version with pre-release. The version of the deployment
+		// is without pre-prerealse
+
+		testNamespace := createNamespaceForTest()
+		defer deleteTestNamespace(testNamespace)
+
+		dep := &apps_v1.Deployment{
+			meta_v1.TypeMeta{},
+			meta_v1.ObjectMeta{
+				Name:      "deployment-2",
+				Namespace: testNamespace,
+				Labels: map[string]string{
+					types.KeelPolicyLabel:  "major",
+					types.KeelTriggerLabel: "poll",
+				},
+				Annotations: map[string]string{
+					types.KeelPollScheduleAnnotation: "@every 2s",
+				},
+			},
+			apps_v1.DeploymentSpec{
+				Selector: &meta_v1.LabelSelector{
+					MatchLabels: map[string]string{
+						"app": "wd-1",
+					},
+				},
+				Template: v1.PodTemplateSpec{
+					ObjectMeta: meta_v1.ObjectMeta{
+						Labels: map[string]string{
+							"app":     "wd-1",
+							"release": "1",
+						},
+					},
+					Spec: v1.PodSpec{
+						Containers: []v1.Container{
+							v1.Container{
+								Name:  "wd-1",
+								Image: "keelhq/push-workflow-example:0.1.0",
+							},
+						},
+					},
+				},
+			},
+			apps_v1.DeploymentStatus{},
+		}
+
+		_, err := kcs.AppsV1().Deployments(testNamespace).Create(dep)
+		if err != nil {
+			t.Fatalf("failed to create deployment: %s", err)
+		}
+		ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
+		defer cancel()
+
+		err = waitFor(ctx, kcs, testNamespace, dep.ObjectMeta.Name, "keelhq/push-workflow-example:0.10.0")
+		if err != nil {
+			t.Errorf("update failed: %s", err)
+		}
+	})
+}
+
+func TestPollingPrivateRegistry(t *testing.T) {
+
+	// stop := make(chan struct{})
+	ctx, cancel := context.WithCancel(context.Background())
+	// defer close(ctx)
+	defer cancel()
+
+	// go startKeel(ctx)
+	keel := &KeelCmd{}
+	go func() {
+		err := keel.Start(ctx)
+		if err != nil {
+			log.WithFields(log.Fields{
+				"error": err,
+			}).Error("failed to start Keel process")
+		}
+	}()
+
+	defer func() {
+		err := keel.Stop()
+		if err != nil {
+			log.WithFields(log.Fields{
+				"error": err,
+			}).Error("failed to stop Keel process")
+		}
+	}()
+
+	_, kcs := getKubernetesClient()
+
+	t.Run("UpdateThroughPrivateQuayPollingA", func(t *testing.T) {
+
+		user := os.Getenv("DOCKERHUB_USERNAME")
+		password := os.Getenv("DOCKERHUB_PASSWORD")
+
+		if user == "" || password == "" {
+			fmt.Println("[X] Skipping UpdateThroughPrivateQuayPollingA test since DOCKERHUB_USERNAME and/or DOCKERHUB_PASSWORD env vars not set")
+			t.Skip()
+		}
+
+		// UpdateThroughDockerHubPollingA tests a polling trigger when we have a higher version
+		// but without a pre-release tag and a lower version with pre-release. The version of the deployment
+		// is with pre-prerealse so we should upgrade to that one.
+
+		testNamespace := createNamespaceForTest()
+		defer deleteTestNamespace(testNamespace)
+
+		payload, err := secrets.EncodeDockerCfgJson(&secrets.DockerCfg{
+			"https://index.docker.io/v1/": &secrets.Auth{
+				Auth: secrets.EncodeBase64Secret(user, password),
+			},
+		})
+		if err != nil {
+			t.Fatalf("failed to encode docker cfg secret payload: %s", err)
+		}
+
+		secretName := "verysecret"
+
+		fmt.Println(string(payload))
+
+		secret := &v1.Secret{
+			TypeMeta: meta_v1.TypeMeta{},
+			ObjectMeta: meta_v1.ObjectMeta{
+				Name:        secretName,
+				Namespace:   testNamespace,
+				Labels:      map[string]string{},
+				Annotations: map[string]string{},
+			},
+			Type: v1.SecretTypeDockerConfigJson,
+			Data: map[string][]byte{
+				// ".dockerconfigjson": []byte(base64.StdEncoding.EncodeToString([]byte(payload))),
+				".dockerconfigjson": []byte(payload),
+			},
+		}
+
+		_, err = kcs.CoreV1().Secrets(testNamespace).Create(secret)
+		if err != nil {
+			t.Fatalf("failed to create secret: %s", err)
+		}
+
+		dep := &apps_v1.Deployment{
+			meta_v1.TypeMeta{},
+			meta_v1.ObjectMeta{
+				Name:      "deployment-1",
+				Namespace: testNamespace,
+				Labels: map[string]string{
+					types.KeelPolicyLabel:  "major",
+					types.KeelTriggerLabel: "poll",
+				},
+				Annotations: map[string]string{
+					types.KeelPollScheduleAnnotation: "@every 2s",
+				},
+			},
+			apps_v1.DeploymentSpec{
+
+				Selector: &meta_v1.LabelSelector{
+					MatchLabels: map[string]string{
+						"app": "wd-1",
+					},
+				},
+				Template: v1.PodTemplateSpec{
+
+					ObjectMeta: meta_v1.ObjectMeta{
+						Labels: map[string]string{
+							"app":     "wd-1",
+							"release": "1",
+						},
+					},
+					Spec: v1.PodSpec{
+						ImagePullSecrets: []v1.LocalObjectReference{
+							{
+								Name: secretName,
+							},
+						},
+						Containers: []v1.Container{
+							v1.Container{
+								ImagePullPolicy: v1.PullAlways,
+								Name:            "wd-1",
+								Image:           "karolisr/demo-webhook:0.0.1",
+							},
+						},
+					},
+				},
+			},
+			apps_v1.DeploymentStatus{},
+		}
+
+		_, err = kcs.AppsV1().Deployments(testNamespace).Create(dep)
+		if err != nil {
+			t.Fatalf("failed to create deployment: %s", err)
+		}
+		ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
+		defer cancel()
+
+		err = waitFor(ctx, kcs, testNamespace, dep.ObjectMeta.Name, "karolisr/demo-webhook:0.0.2")
+		if err != nil {
+			t.Errorf("update failed: %s", err)
+		}
+	})
+
+}

tests/acceptance_test.go

@@ -143,156 +143,6 @@ func TestWebhooksSemverUpdate(t *testing.T) {
 	})
 }
 
-func TestPollingSemverUpdate(t *testing.T) {
-
-	// stop := make(chan struct{})
-	ctx, cancel := context.WithCancel(context.Background())
-	// defer close(ctx)
-	defer cancel()
-
-	// go startKeel(ctx)
-	keel := &KeelCmd{}
-	go func() {
-		err := keel.Start(ctx)
-		if err != nil {
-			log.WithFields(log.Fields{
-				"error": err,
-			}).Error("failed to start Keel process")
-		}
-	}()
-
-	defer func() {
-		err := keel.Stop()
-		if err != nil {
-			log.WithFields(log.Fields{
-				"error": err,
-			}).Error("failed to stop Keel process")
-		}
-	}()
-
-	_, kcs := getKubernetesClient()
-
-	t.Run("UpdateThroughDockerHubPollingA", func(t *testing.T) {
-		// UpdateThroughDockerHubPollingA tests a polling trigger when we have a higher version
-		// but without a pre-release tag and a lower version with pre-release. The version of the deployment
-		// is with pre-prerealse so we should upgrade to that one.
-
-		testNamespace := createNamespaceForTest()
-		defer deleteTestNamespace(testNamespace)
-
-		dep := &apps_v1.Deployment{
-			meta_v1.TypeMeta{},
-			meta_v1.ObjectMeta{
-				Name:      "deployment-1",
-				Namespace: testNamespace,
-				Labels: map[string]string{
-					types.KeelPolicyLabel:  "major",
-					types.KeelTriggerLabel: "poll",
-				},
-				Annotations: map[string]string{
-					types.KeelPollScheduleAnnotation: "@every 2s",
-				},
-			},
-			apps_v1.DeploymentSpec{
-				Selector: &meta_v1.LabelSelector{
-					MatchLabels: map[string]string{
-						"app": "wd-1",
-					},
-				},
-				Template: v1.PodTemplateSpec{
-					ObjectMeta: meta_v1.ObjectMeta{
-						Labels: map[string]string{
-							"app":     "wd-1",
-							"release": "1",
-						},
-					},
-					Spec: v1.PodSpec{
-						Containers: []v1.Container{
-							v1.Container{
-								Name:  "wd-1",
-								Image: "keelhq/push-workflow-example:0.1.0-dev",
-							},
-						},
-					},
-				},
-			},
-			apps_v1.DeploymentStatus{},
-		}
-
-		_, err := kcs.AppsV1().Deployments(testNamespace).Create(dep)
-		if err != nil {
-			t.Fatalf("failed to create deployment: %s", err)
-		}
-		ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
-		defer cancel()
-
-		err = waitFor(ctx, kcs, testNamespace, dep.ObjectMeta.Name, "keelhq/push-workflow-example:0.5.0-dev")
-		if err != nil {
-			t.Errorf("update failed: %s", err)
-		}
-	})
-
-	t.Run("UpdateThroughDockerHubPollingB", func(t *testing.T) {
-		// UpdateThroughDockerHubPollingA tests a polling trigger when we have a higher version
-		// but without a pre-release tag and a lower version with pre-release. The version of the deployment
-		// is without pre-prerealse
-
-		testNamespace := createNamespaceForTest()
-		defer deleteTestNamespace(testNamespace)
-
-		dep := &apps_v1.Deployment{
-			meta_v1.TypeMeta{},
-			meta_v1.ObjectMeta{
-				Name:      "deployment-2",
-				Namespace: testNamespace,
-				Labels: map[string]string{
-					types.KeelPolicyLabel:  "major",
-					types.KeelTriggerLabel: "poll",
-				},
-				Annotations: map[string]string{
-					types.KeelPollScheduleAnnotation: "@every 2s",
-				},
-			},
-			apps_v1.DeploymentSpec{
-				Selector: &meta_v1.LabelSelector{
-					MatchLabels: map[string]string{
-						"app": "wd-1",
-					},
-				},
-				Template: v1.PodTemplateSpec{
-					ObjectMeta: meta_v1.ObjectMeta{
-						Labels: map[string]string{
-							"app":     "wd-1",
-							"release": "1",
-						},
-					},
-					Spec: v1.PodSpec{
-						Containers: []v1.Container{
-							v1.Container{
-								Name:  "wd-1",
-								Image: "keelhq/push-workflow-example:0.1.0",
-							},
-						},
-					},
-				},
-			},
-			apps_v1.DeploymentStatus{},
-		}
-
-		_, err := kcs.AppsV1().Deployments(testNamespace).Create(dep)
-		if err != nil {
-			t.Fatalf("failed to create deployment: %s", err)
-		}
-		ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
-		defer cancel()
-
-		err = waitFor(ctx, kcs, testNamespace, dep.ObjectMeta.Name, "keelhq/push-workflow-example:0.10.0")
-		if err != nil {
-			t.Errorf("update failed: %s", err)
-		}
-	})
-}
-
 func TestApprovals(t *testing.T) {
 
 	ctx, cancel := context.WithCancel(context.Background())