diff --git a/go.mod b/go.mod index 6dfaa1eb..dbf6a0b1 100644 --- a/go.mod +++ b/go.mod @@ -42,6 +42,7 @@ require ( github.com/jinzhu/gorm v1.9.16 github.com/nlopes/slack v0.6.0 github.com/opencontainers/go-digest v1.0.0 + github.com/opencontainers/image-spec v1.1.0-rc2 github.com/prometheus/client_golang v1.14.0 github.com/rusenask/cron v1.1.0 github.com/rusenask/docker-registry-client v0.0.0-20200210164146-049272422097 @@ -89,7 +90,7 @@ require ( github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-metrics v0.0.1 // indirect github.com/docker/go-units v0.4.0 // indirect - github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1 // indirect + github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 // indirect github.com/emicklei/go-restful/v3 v3.9.0 // indirect github.com/evanphx/json-patch v5.6.0+incompatible // indirect github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d // indirect @@ -146,7 +147,6 @@ require ( github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect github.com/morikuni/aec v1.0.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/opencontainers/image-spec v1.1.0-rc2 // indirect github.com/peterbourgon/diskv v2.0.1+incompatible // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect diff --git a/go.sum b/go.sum index 74c69da8..01852dd0 100644 --- a/go.sum +++ b/go.sum @@ -167,8 +167,8 @@ github.com/docker/go-metrics v0.0.1 h1:AgB/0SvBxihN0X8OR4SjsblXkbMvalQ8cjmtKQ2rQ github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw= github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1 h1:ZClxb8laGDf5arXfYcAtECDFgAgHklGI8CxgjHnXKJ4= -github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE= +github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 h1:UhxFibDNY/bfvqU5CAUmr9zpesgbU6SWc8/B4mflAE4= +github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc= github.com/emicklei/go-restful/v3 v3.9.0 h1:XwGDlfxEnQZzuopoqxwSEllNcCOM9DhhFyhFIIGKwxE= diff --git a/registry/docker/json.go b/registry/docker/json.go new file mode 100644 index 00000000..d25ee394 --- /dev/null +++ b/registry/docker/json.go @@ -0,0 +1,48 @@ +package docker + +import ( + "encoding/json" + "errors" + "net/http" + "regexp" +) + +var ErrNoMorePages = errors.New("no more pages") + +// Matches an RFC 5988 (https://tools.ietf.org/html/rfc5988#section-5) +// Link header. For example, +// +// ; type="application/json"; rel="next" +// +// The URL is _supposed_ to be wrapped by angle brackets `< ... >`, +// but e.g., quay.io does not include them. Similarly, params like +// `rel="next"` may not have quoted values in the wild. +var nextLinkRE = regexp.MustCompile(`^ *]+)>? *(?:;[^;]*)*; *rel="?next"?(?:;.*)?`) + +func getNextLink(resp *http.Response) (string, error) { + for _, link := range resp.Header[http.CanonicalHeaderKey("Link")] { + parts := nextLinkRE.FindStringSubmatch(link) + if parts != nil { + return parts[1], nil + } + } + return "", ErrNoMorePages +} + +// getPaginatedJSON accepts a string and a pointer, and returns the +// next page URL while updating pointed-to variable with a parsed JSON +// value. When there are no more pages it returns `ErrNoMorePages`. +func (registry *Registry) getPaginatedJSON(url string, response interface{}) (string, error) { + resp, err := registry.Client.Get(registry.url(url)) + if err != nil { + return "", err + } + defer resp.Body.Close() + + decoder := json.NewDecoder(resp.Body) + err = decoder.Decode(response) + if err != nil { + return "", err + } + return getNextLink(resp) +} diff --git a/registry/docker/manifest.go b/registry/docker/manifest.go new file mode 100644 index 00000000..99ebe1b6 --- /dev/null +++ b/registry/docker/manifest.go @@ -0,0 +1,48 @@ +package docker + +import ( + "io/ioutil" + "net/http" + "strings" + + manifestv2 "github.com/docker/distribution/manifest/schema2" + "github.com/opencontainers/go-digest" + oci "github.com/opencontainers/image-spec/specs-go/v1" +) + +// ManifestDigest - get manifest digest +func (registry *Registry) ManifestDigest(repository, reference string) (digest.Digest, error) { + url := registry.url("/v2/%s/manifests/%s", repository, reference) + registry.Logf("registry.manifest.head url=%s repository=%s reference=%s", url, repository, reference) + + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return "", err + } + + req.Header.Set("Accept", manifestv2.MediaTypeManifest) + resp, err := registry.Client.Do(req) + if err != nil { + // Try OCI headers if error relates to OCI + if strings.Contains(err.Error(), "OCI index found, but accept header does not support OCI indexes") { + req.Header.Set("Accept", oci.MediaTypeImageIndex) + resp, err = registry.Client.Do(req) + } + if err != nil { + return "", err + } + } + defer resp.Body.Close() + + if hdr := resp.Header.Get("Docker-Content-Digest"); hdr != "" { + return digest.Parse(hdr) + } + + // Try to get digest from body instead, should be equal to what would be presented + // in Docker-Content-Digest + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return "", err + } + return digest.FromBytes(body), nil +} diff --git a/registry/docker/manifest_test.go b/registry/docker/manifest_test.go new file mode 100644 index 00000000..d8101e87 --- /dev/null +++ b/registry/docker/manifest_test.go @@ -0,0 +1,43 @@ +package docker + +import ( + "io" + "net/http" + "net/http/httptest" + "testing" + + manifestV2 "github.com/docker/distribution/manifest/schema2" +) + +func TestGetDigest(t *testing.T) { + + req, err := http.NewRequest("GET", "https://registry.opensource.zalan.do/v2/teapot/external-dns/manifests/v0.4.8", nil) + if err != nil { + t.Fatalf("failed to create request: %s", err) + } + req.Header.Set("Accept", manifestV2.MediaTypeManifest) + + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + t.Fatalf("failed to request: %s", err) + } + defer resp.Body.Close() + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Add("content-type", "application/vnd.docker.distribution.manifest.v2+json; charset=ISO-8859-1") + io.Copy(w, resp.Body) + })) + defer ts.Close() + + reg := New(ts.URL, "", "") + + digest, err := reg.ManifestDigest(ts.URL, "notimportant") + if err != nil { + t.Errorf("failed to get digest") + } + + if digest.String() != "sha256:7aa5175f39a7e8a4172972524302c9a8196f681e40d6ee5d2f6bf0ab7d600fee" { + t.Errorf("unexpected digest: %s", digest.String()) + } +} diff --git a/registry/docker/registry.go b/registry/docker/registry.go new file mode 100644 index 00000000..e3f2a2a5 --- /dev/null +++ b/registry/docker/registry.go @@ -0,0 +1,115 @@ +package docker + +import ( + "crypto/tls" + "fmt" + "log" + "net" + "net/http" + "strings" + "time" + + drc "github.com/rusenask/docker-registry-client/registry" +) + +type LogfCallback func(format string, args ...interface{}) + +type Registry struct { + URL string + Client *http.Client + Logf LogfCallback +} + +/* + * Pass log messages along to Go's "log" module. + */ +func Log(format string, args ...interface{}) { + log.Printf(format, args...) +} + +/* + * Create a new Registry with the given URL and credentials, then Ping()s it + * before returning it to verify that the registry is available. + * + * You can, alternately, construct a Registry manually by populating the fields. + * This passes http.DefaultTransport to WrapTransport when creating the + * http.Client. + */ +func New(registryURL, username, password string) *Registry { + transport := &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + Timeout: 10 * time.Second, + KeepAlive: 30 * time.Second, + DualStack: true, + }).DialContext, + MaxIdleConns: 10, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + } + + return newFromTransport(registryURL, username, password, transport, Log) +} + +/* + * Create a new Registry, as with New, using an http.Transport that disables + * SSL certificate verification. + */ +func NewInsecure(registryURL, username, password string) *Registry { + transport := &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + Timeout: 10 * time.Second, + KeepAlive: 30 * time.Second, + DualStack: true, + }).DialContext, + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: true, + }, + MaxIdleConns: 10, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + } + + return newFromTransport(registryURL, username, password, transport, Log) +} + +func newFromTransport(registryURL, username, password string, transport *http.Transport, logf LogfCallback) *Registry { + url := strings.TrimSuffix(registryURL, "/") + registry := &Registry{ + URL: url, + Client: &http.Client{ + Transport: drc.WrapTransport(transport, url, username, password), + }, + Logf: logf, + } + + return registry +} + +func (r *Registry) Ping() error { + url := r.url("/v2/") + r.Logf("registry.ping url=%s", url) + resp, err := r.Client.Get(url) + if resp != nil { + defer resp.Body.Close() + } + return err +} + +type tagsResponse struct { + Tags []string `json:"tags"` +} + +func (r *Registry) url(pathTemplate string, args ...interface{}) string { + pathSuffix := fmt.Sprintf(pathTemplate, args...) + if strings.HasPrefix(pathSuffix, r.URL) { + return pathSuffix + } + + url := fmt.Sprintf("%s%s", r.URL, pathSuffix) + + return url +} diff --git a/registry/docker/tags.go b/registry/docker/tags.go new file mode 100644 index 00000000..1c928192 --- /dev/null +++ b/registry/docker/tags.go @@ -0,0 +1,21 @@ +package docker + +func (registry *Registry) Tags(repository string) (tags []string, err error) { + url := registry.url("/v2/%s/tags/list", repository) + + var response tagsResponse + for { + registry.Logf("registry.tags url=%s repository=%s", url, repository) + url, err = registry.getPaginatedJSON(url, &response) + switch err { + case ErrNoMorePages: + tags = append(tags, response.Tags...) + return tags, nil + case nil: + tags = append(tags, response.Tags...) + continue + default: + return nil, err + } + } +} diff --git a/registry/docker/tags_test.go b/registry/docker/tags_test.go new file mode 100644 index 00000000..47045466 --- /dev/null +++ b/registry/docker/tags_test.go @@ -0,0 +1,18 @@ +package docker + +import ( + "testing" +) + +func TestGetDigestDockerHub(t *testing.T) { + client := New("https://index.docker.io", "", "") + + tags, err := client.Tags("karolisr/keel") + if err != nil { + t.Errorf("failed to get tags, error: %s", err) + } + + if len(tags) == 0 { + t.Errorf("no tags?") + } +} diff --git a/registry/registry.go b/registry/registry.go index c50c07ac..333fa26b 100644 --- a/registry/registry.go +++ b/registry/registry.go @@ -7,7 +7,7 @@ import ( "strings" "sync" - "github.com/rusenask/docker-registry-client/registry" + "github.com/keel-hq/keel/registry/docker" log "github.com/sirupsen/logrus" ) @@ -40,7 +40,7 @@ func New() *DefaultClient { } return &DefaultClient{ mu: &sync.Mutex{}, - registries: make(map[uint32]*registry.Registry), + registries: make(map[uint32]*docker.Registry), insecure: insecure, } } @@ -49,7 +49,7 @@ func New() *DefaultClient { type DefaultClient struct { // a map of registries to reuse for polling mu *sync.Mutex - registries map[uint32]*registry.Registry + registries map[uint32]*docker.Registry insecure bool } @@ -71,11 +71,11 @@ func hash(s string) uint32 { return h.Sum32() } -func (c *DefaultClient) getRegistryClient(registryAddress, username, password string) (*registry.Registry, error) { +func (c *DefaultClient) getRegistryClient(registryAddress, username, password string) (*docker.Registry, error) { c.mu.Lock() defer c.mu.Unlock() - var r *registry.Registry + var r *docker.Registry h := hash(registryAddress + username + password) r, ok := c.registries[h] @@ -85,9 +85,9 @@ func (c *DefaultClient) getRegistryClient(registryAddress, username, password st url := strings.TrimSuffix(registryAddress, "/") if os.Getenv(EnvInsecure) == "true" { - r = registry.NewInsecure(url, username, password) + r = docker.NewInsecure(url, username, password) } else { - r = registry.New(url, username, password) + r = docker.New(url, username, password) } r.Logf = LogFormatter diff --git a/registry/registry_test.go b/registry/registry_test.go index db108199..f2a0fde3 100644 --- a/registry/registry_test.go +++ b/registry/registry_test.go @@ -1,17 +1,15 @@ package registry import ( + "fmt" "net/http" "net/http/httptest" + "os" "strings" + "testing" "github.com/keel-hq/keel/constants" - - "github.com/rusenask/docker-registry-client/registry" - - "fmt" - "os" - "testing" + "github.com/keel-hq/keel/registry/docker" ) func TestDigest(t *testing.T) { @@ -32,6 +30,24 @@ func TestDigest(t *testing.T) { } } +func TestOCIDigest(t *testing.T) { + + client := New() + digest, err := client.Digest(Opts{ + Registry: "https://index.docker.io", + Name: "vaultwarden/server", + Tag: "1.25.1", + }) + + if err != nil { + t.Errorf("error while getting digest: %s", err) + } + + if digest != "sha256:dd8cf61d1997c098cc5686ef3116ca5cfef36f12192c01caa1de79a968397d4c" { + t.Errorf("unexpected digest: %s", digest) + } +} + func TestGet(t *testing.T) { client := New() repo, err := client.Get(Opts{ @@ -306,7 +322,7 @@ var tagsResp = `{ }` func TestGetDockerHubManyTags(t *testing.T) { - client := registry.New("https://quay.io", "", "") + client := docker.New("https://quay.io", "", "") tags, err := client.Tags("coreos/prometheus-operator") if err != nil { t.Errorf("error while getting repo: %s", err) diff --git a/tests/acceptance_polling_test.go b/tests/acceptance_polling_test.go index d364dc54..ef3db0c8 100644 --- a/tests/acceptance_polling_test.go +++ b/tests/acceptance_polling_test.go @@ -223,6 +223,64 @@ func TestPollingSemverUpdate(t *testing.T) { t.Errorf("update failed: %s", err) } }) + + t.Run("UpdateThroughDockerHubPollingD", func(t *testing.T) { + + testNamespace := createNamespaceForTest() + defer deleteTestNamespace(testNamespace) + + dep := &apps_v1.Deployment{ + meta_v1.TypeMeta{}, + meta_v1.ObjectMeta{ + Name: "deployment-3", + Namespace: testNamespace, + Labels: map[string]string{ + types.KeelPolicyLabel: "patch", + types.KeelTriggerLabel: "poll", + }, + Annotations: map[string]string{ + types.KeelPollScheduleAnnotation: "@every 2s", + }, + }, + apps_v1.DeploymentSpec{ + Selector: &meta_v1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "wd-1", + }, + }, + Template: v1.PodTemplateSpec{ + ObjectMeta: meta_v1.ObjectMeta{ + Labels: map[string]string{ + "app": "wd-1", + "release": "1", + }, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "wd-1", + Image: "vaultwarden/server:1.25.1", + }, + }, + }, + }, + }, + apps_v1.DeploymentStatus{}, + } + createOptions := meta_v1.CreateOptions{} + + _, err := kcs.AppsV1().Deployments(testNamespace).Create(context.Background(), dep, createOptions) + if err != nil { + t.Fatalf("failed to create deployment: %s", err) + } + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + + err = waitFor(ctx, kcs, testNamespace, dep.ObjectMeta.Name, "vaultwarden/server:1.25.2") + if err != nil { + t.Errorf("update failed: %s", err) + } + }) } func TestPollingPrivateRegistry(t *testing.T) {