keel-hq
/
keel
mirror of https://github.com/keel-hq/keel/tree/master/chart/keel
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Add support for OCI images

pull/717/head
Scott Dawson 2023-04-11 15:53:31 +10:00
parent a58d5a638d
commit 2315aced1b
No known key found for this signature in database
GPG Key ID: D27F63617C60D508
11 changed files with 385 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
go.mod
View File

@ -42,6 +42,7 @@ require (
github.com/jinzhu/gorm v1.9.16
github.com/nlopes/slack v0.6.0
github.com/opencontainers/go-digest v1.0.0
github.com/opencontainers/image-spec v1.1.0-rc2
github.com/prometheus/client_golang v1.14.0
github.com/rusenask/cron v1.1.0
github.com/rusenask/docker-registry-client v0.0.0-20200210164146-049272422097
@ -89,7 +90,7 @@ require (
github.com/docker/go-connections v0.4.0 // indirect
github.com/docker/go-metrics v0.0.1 // indirect
github.com/docker/go-units v0.4.0 // indirect
github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1 // indirect
github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 // indirect
github.com/emicklei/go-restful/v3 v3.9.0 // indirect
github.com/evanphx/json-patch v5.6.0+incompatible // indirect
github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d // indirect
@ -146,7 +147,6 @@ require (
github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect
github.com/morikuni/aec v1.0.0 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/opencontainers/image-spec v1.1.0-rc2 // indirect
github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect

4
go.sum
View File

@ -167,8 +167,8 @@ github.com/docker/go-metrics v0.0.1 h1:AgB/0SvBxihN0X8OR4SjsblXkbMvalQ8cjmtKQ2rQ
github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw=
github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw=
github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1 h1:ZClxb8laGDf5arXfYcAtECDFgAgHklGI8CxgjHnXKJ4=
github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE=
github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 h1:UhxFibDNY/bfvqU5CAUmr9zpesgbU6SWc8/B4mflAE4=
github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE=
github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc=
github.com/emicklei/go-restful/v3 v3.9.0 h1:XwGDlfxEnQZzuopoqxwSEllNcCOM9DhhFyhFIIGKwxE=

48
registry/docker/json.go Normal file
View File

@ -0,0 +1,48 @@
package docker
import (
"encoding/json"
"errors"
"net/http"
"regexp"
)
var ErrNoMorePages = errors.New("no more pages")
// Matches an RFC 5988 (https://tools.ietf.org/html/rfc5988#section-5)
// Link header. For example,
//
// <http://registry.example.com/v2/_catalog?n=5&last=tag5>; type="application/json"; rel="next"
//
// The URL is _supposed_ to be wrapped by angle brackets `< ... >`,
// but e.g., quay.io does not include them. Similarly, params like
// `rel="next"` may not have quoted values in the wild.
var nextLinkRE = regexp.MustCompile(`^ *<?([^;>]+)>? *(?:;[^;]*)*; *rel="?next"?(?:;.*)?`)
func getNextLink(resp *http.Response) (string, error) {
for _, link := range resp.Header[http.CanonicalHeaderKey("Link")] {
parts := nextLinkRE.FindStringSubmatch(link)
if parts != nil {
return parts[1], nil
}
}
return "", ErrNoMorePages
}
// getPaginatedJSON accepts a string and a pointer, and returns the
// next page URL while updating pointed-to variable with a parsed JSON
// value. When there are no more pages it returns `ErrNoMorePages`.
func (registry *Registry) getPaginatedJSON(url string, response interface{}) (string, error) {
resp, err := registry.Client.Get(registry.url(url))
if err != nil {
return "", err
}
defer resp.Body.Close()
decoder := json.NewDecoder(resp.Body)
err = decoder.Decode(response)
if err != nil {
return "", err
}
return getNextLink(resp)
}

48
registry/docker/manifest.go Normal file
View File

@ -0,0 +1,48 @@
package docker
import (
"io/ioutil"
"net/http"
"strings"
manifestv2 "github.com/docker/distribution/manifest/schema2"
"github.com/opencontainers/go-digest"
oci "github.com/opencontainers/image-spec/specs-go/v1"
)
// ManifestDigest - get manifest digest
func (registry *Registry) ManifestDigest(repository, reference string) (digest.Digest, error) {
url := registry.url("/v2/%s/manifests/%s", repository, reference)
registry.Logf("registry.manifest.head url=%s repository=%s reference=%s", url, repository, reference)
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return "", err
}
req.Header.Set("Accept", manifestv2.MediaTypeManifest)
resp, err := registry.Client.Do(req)
if err != nil {
// Try OCI headers if error relates to OCI
if strings.Contains(err.Error(), "OCI index found, but accept header does not support OCI indexes") {
req.Header.Set("Accept", oci.MediaTypeImageIndex)
resp, err = registry.Client.Do(req)
}
if err != nil {
return "", err
}
}
defer resp.Body.Close()
if hdr := resp.Header.Get("Docker-Content-Digest"); hdr != "" {
return digest.Parse(hdr)
}
// Try to get digest from body instead, should be equal to what would be presented
// in Docker-Content-Digest
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return "", err
}
return digest.FromBytes(body), nil
}

43
registry/docker/manifest_test.go Normal file
View File

@ -0,0 +1,43 @@
package docker
import (
"io"
"net/http"
"net/http/httptest"
"testing"
manifestV2 "github.com/docker/distribution/manifest/schema2"
)
func TestGetDigest(t *testing.T) {
req, err := http.NewRequest("GET", "https://registry.opensource.zalan.do/v2/teapot/external-dns/manifests/v0.4.8", nil)
if err != nil {
t.Fatalf("failed to create request: %s", err)
}
req.Header.Set("Accept", manifestV2.MediaTypeManifest)
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
t.Fatalf("failed to request: %s", err)
}
defer resp.Body.Close()
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Add("content-type", "application/vnd.docker.distribution.manifest.v2+json; charset=ISO-8859-1")
io.Copy(w, resp.Body)
}))
defer ts.Close()
reg := New(ts.URL, "", "")
digest, err := reg.ManifestDigest(ts.URL, "notimportant")
if err != nil {
t.Errorf("failed to get digest")
}
if digest.String() != "sha256:7aa5175f39a7e8a4172972524302c9a8196f681e40d6ee5d2f6bf0ab7d600fee" {
t.Errorf("unexpected digest: %s", digest.String())
}
}

115
registry/docker/registry.go Normal file
View File

@ -0,0 +1,115 @@
package docker
import (
"crypto/tls"
"fmt"
"log"
"net"
"net/http"
"strings"
"time"
drc "github.com/rusenask/docker-registry-client/registry"
)
type LogfCallback func(format string, args ...interface{})
type Registry struct {
URL string
Client *http.Client
Logf LogfCallback
}
/*
* Pass log messages along to Go's "log" module.
*/
func Log(format string, args ...interface{}) {
log.Printf(format, args...)
}
/*
* Create a new Registry with the given URL and credentials, then Ping()s it
* before returning it to verify that the registry is available.
*
* You can, alternately, construct a Registry manually by populating the fields.
* This passes http.DefaultTransport to WrapTransport when creating the
* http.Client.
*/
func New(registryURL, username, password string) *Registry {
transport := &http.Transport{
Proxy: http.ProxyFromEnvironment,
DialContext: (&net.Dialer{
Timeout: 10 * time.Second,
KeepAlive: 30 * time.Second,
DualStack: true,
}).DialContext,
MaxIdleConns: 10,
IdleConnTimeout: 90 * time.Second,
TLSHandshakeTimeout: 10 * time.Second,
ExpectContinueTimeout: 1 * time.Second,
}
return newFromTransport(registryURL, username, password, transport, Log)
}
/*
* Create a new Registry, as with New, using an http.Transport that disables
* SSL certificate verification.
*/
func NewInsecure(registryURL, username, password string) *Registry {
transport := &http.Transport{
Proxy: http.ProxyFromEnvironment,
DialContext: (&net.Dialer{
Timeout: 10 * time.Second,
KeepAlive: 30 * time.Second,
DualStack: true,
}).DialContext,
TLSClientConfig: &tls.Config{
InsecureSkipVerify: true,
},
MaxIdleConns: 10,
IdleConnTimeout: 90 * time.Second,
TLSHandshakeTimeout: 10 * time.Second,
ExpectContinueTimeout: 1 * time.Second,
}
return newFromTransport(registryURL, username, password, transport, Log)
}
func newFromTransport(registryURL, username, password string, transport *http.Transport, logf LogfCallback) *Registry {
url := strings.TrimSuffix(registryURL, "/")
registry := &Registry{
URL: url,
Client: &http.Client{
Transport: drc.WrapTransport(transport, url, username, password),
},
Logf: logf,
}
return registry
}
func (r *Registry) Ping() error {
url := r.url("/v2/")
r.Logf("registry.ping url=%s", url)
resp, err := r.Client.Get(url)
if resp != nil {
defer resp.Body.Close()
}
return err
}
type tagsResponse struct {
Tags []string `json:"tags"`
}
func (r *Registry) url(pathTemplate string, args ...interface{}) string {
pathSuffix := fmt.Sprintf(pathTemplate, args...)
if strings.HasPrefix(pathSuffix, r.URL) {
return pathSuffix
}
url := fmt.Sprintf("%s%s", r.URL, pathSuffix)
return url
}

21
registry/docker/tags.go Normal file
View File

@ -0,0 +1,21 @@
package docker
func (registry *Registry) Tags(repository string) (tags []string, err error) {
url := registry.url("/v2/%s/tags/list", repository)
var response tagsResponse
for {
registry.Logf("registry.tags url=%s repository=%s", url, repository)
url, err = registry.getPaginatedJSON(url, &response)
switch err {
case ErrNoMorePages:
tags = append(tags, response.Tags...)
return tags, nil
case nil:
tags = append(tags, response.Tags...)
continue
default:
return nil, err
}
}
}

18
registry/docker/tags_test.go Normal file
View File

@ -0,0 +1,18 @@
package docker
import (
"testing"
)
func TestGetDigestDockerHub(t *testing.T) {
client := New("https://index.docker.io", "", "")
tags, err := client.Tags("karolisr/keel")
if err != nil {
t.Errorf("failed to get tags, error: %s", err)
}
if len(tags) == 0 {
t.Errorf("no tags?")
}
}

14
registry/registry.go
View File

@ -7,7 +7,7 @@ import (
"strings"
"sync"
"github.com/rusenask/docker-registry-client/registry"
"github.com/keel-hq/keel/registry/docker"
log "github.com/sirupsen/logrus"
)
@ -40,7 +40,7 @@ func New() *DefaultClient {
}
return &DefaultClient{
mu: &sync.Mutex{},
registries: make(map[uint32]*registry.Registry),
registries: make(map[uint32]*docker.Registry),
insecure: insecure,
}
}
@ -49,7 +49,7 @@ func New() *DefaultClient {
type DefaultClient struct {
// a map of registries to reuse for polling
mu *sync.Mutex
registries map[uint32]*registry.Registry
registries map[uint32]*docker.Registry
insecure bool
}
@ -71,11 +71,11 @@ func hash(s string) uint32 {
return h.Sum32()
}
func (c *DefaultClient) getRegistryClient(registryAddress, username, password string) (*registry.Registry, error) {
func (c *DefaultClient) getRegistryClient(registryAddress, username, password string) (*docker.Registry, error) {
c.mu.Lock()
defer c.mu.Unlock()
var r *registry.Registry
var r *docker.Registry
h := hash(registryAddress + username + password)
r, ok := c.registries[h]
@ -85,9 +85,9 @@ func (c *DefaultClient) getRegistryClient(registryAddress, username, password st
url := strings.TrimSuffix(registryAddress, "/")
if os.Getenv(EnvInsecure) == "true" {
r = registry.NewInsecure(url, username, password)
r = docker.NewInsecure(url, username, password)
} else {
r = registry.New(url, username, password)
r = docker.New(url, username, password)
}
r.Logf = LogFormatter

30
registry/registry_test.go
View File

@ -1,17 +1,15 @@
package registry
import (
"fmt"
"net/http"
"net/http/httptest"
"os"
"strings"
"testing"
"github.com/keel-hq/keel/constants"
"github.com/rusenask/docker-registry-client/registry"
"fmt"
"os"
"testing"
"github.com/keel-hq/keel/registry/docker"
)
func TestDigest(t *testing.T) {
@ -32,6 +30,24 @@ func TestDigest(t *testing.T) {
}
}
func TestOCIDigest(t *testing.T) {
client := New()
digest, err := client.Digest(Opts{
Registry: "https://index.docker.io",
Name: "vaultwarden/server",
Tag: "1.25.1",
})
if err != nil {
t.Errorf("error while getting digest: %s", err)
}
if digest != "sha256:dd8cf61d1997c098cc5686ef3116ca5cfef36f12192c01caa1de79a968397d4c" {
t.Errorf("unexpected digest: %s", digest)
}
}
func TestGet(t *testing.T) {
client := New()
repo, err := client.Get(Opts{
@ -306,7 +322,7 @@ var tagsResp = `{
}`
func TestGetDockerHubManyTags(t *testing.T) {
client := registry.New("https://quay.io", "", "")
client := docker.New("https://quay.io", "", "")
tags, err := client.Tags("coreos/prometheus-operator")
if err != nil {
t.Errorf("error while getting repo: %s", err)

58
tests/acceptance_polling_test.go
View File

@ -223,6 +223,64 @@ func TestPollingSemverUpdate(t *testing.T) {
t.Errorf("update failed: %s", err)
}
})
t.Run("UpdateThroughDockerHubPollingD", func(t *testing.T) {
testNamespace := createNamespaceForTest()
defer deleteTestNamespace(testNamespace)
dep := &apps_v1.Deployment{
meta_v1.TypeMeta{},
meta_v1.ObjectMeta{
Name: "deployment-3",
Namespace: testNamespace,
Labels: map[string]string{
types.KeelPolicyLabel: "patch",
types.KeelTriggerLabel: "poll",
},
Annotations: map[string]string{
types.KeelPollScheduleAnnotation: "@every 2s",
},
},
apps_v1.DeploymentSpec{
Selector: &meta_v1.LabelSelector{
MatchLabels: map[string]string{
"app": "wd-1",
},
},
Template: v1.PodTemplateSpec{
ObjectMeta: meta_v1.ObjectMeta{
Labels: map[string]string{
"app": "wd-1",
"release": "1",
},
},
Spec: v1.PodSpec{
Containers: []v1.Container{
{
Name: "wd-1",
Image: "vaultwarden/server:1.25.1",
},
},
},
},
},
apps_v1.DeploymentStatus{},
}
createOptions := meta_v1.CreateOptions{}
_, err := kcs.AppsV1().Deployments(testNamespace).Create(context.Background(), dep, createOptions)
if err != nil {
t.Fatalf("failed to create deployment: %s", err)
}
ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
defer cancel()
err = waitFor(ctx, kcs, testNamespace, dep.ObjectMeta.Name, "vaultwarden/server:1.25.2")
if err != nil {
t.Errorf("update failed: %s", err)
}
})
}
func TestPollingPrivateRegistry(t *testing.T) {