Add webhook to inject env to redirect inner cluster requests to haproxy

in auto-pause
pull/10823/head
Yanshu Zhao 2021-03-15 05:56:44 +00:00
parent ff0e25ada8
commit 91f9bd6ef9
8 changed files with 502 additions and 0 deletions

View File

@ -819,6 +819,20 @@ out/mkcmp:
deploy/kicbase/auto-pause: $(SOURCE_GENERATED) $(SOURCE_FILES)
GOOS=linux GOARCH=$(GOARCH) go build -o $@ cmd/auto-pause/auto-pause.go
.PHONY: out/auto-pause-hook
out/auto-pause-hook: $(SOURCE_GENERATED) ## Build auto-pause hook addon
$(if $(quiet),@echo " GO $@")
$(Q)GOOS=linux CGO_ENABLED=0 go build -a --ldflags '-extldflags "-static"' -tags netgo -installsuffix netgo -o $@ cmd/auto-pause/auto-pause-hook/main.go cmd/auto-pause/auto-pause-hook/config.go cmd/auto-pause/auto-pause-hook/certs.go
.PHONY: auto-pause-hook-image
auto-pause-hook-image: out/auto-pause-hook ## Build docker image for auto-pause hook
docker build -t docker.io/azhao155/auto-pause-hook:1.3 -f deploy/addons/auto-pause/Dockerfile .
.PHONY: push-auto-pause-hook-image
push-auto-pause-hook-image: auto-pause-hook-image
docker login docker.io/azhao155
$(MAKE) push-docker IMAGE=docker.io/azhao155/auto-pause-hook:1.3
.PHONY: out/performance-bot
out/performance-bot:
GOOS=$(GOOS) GOARCH=$(GOARCH) go build -o $@ cmd/performance/pr-bot/bot.go

View File

@ -0,0 +1,95 @@
package main
import (
"bytes"
cryptorand "crypto/rand"
"crypto/rsa"
"crypto/x509"
"crypto/x509/pkix"
"encoding/pem"
"fmt"
"math/big"
"time"
)
func gencerts() (caCert []byte, serverCert []byte, serverKey []byte) {
var caPEM, serverCertPEM, serverPrivKeyPEM *bytes.Buffer
// CA config
ca := &x509.Certificate{
SerialNumber: big.NewInt(2020),
Subject: pkix.Name{
Organization: []string{"velotio.com"},
},
NotBefore: time.Now(),
NotAfter: time.Now().AddDate(1, 0, 0),
IsCA: true,
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth},
KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
BasicConstraintsValid: true,
}
// CA private key
caPrivKey, err := rsa.GenerateKey(cryptorand.Reader, 4096)
if err != nil {
fmt.Println(err)
}
// Self signed CA certificate
caBytes, err := x509.CreateCertificate(cryptorand.Reader, ca, ca, &caPrivKey.PublicKey, caPrivKey)
if err != nil {
fmt.Println(err)
}
// PEM encode CA cert
caPEM = new(bytes.Buffer)
_ = pem.Encode(caPEM, &pem.Block{
Type: "CERTIFICATE",
Bytes: caBytes,
})
dnsNames := []string{"webhook",
"webhook.auto-pause", "webhook.auto-pause.svc"}
commonName := "webhook.auto-pause.svc"
// server cert config
cert := &x509.Certificate{
DNSNames: dnsNames,
SerialNumber: big.NewInt(1658),
Subject: pkix.Name{
CommonName: commonName,
Organization: []string{"velotio.com"},
},
NotBefore: time.Now(),
NotAfter: time.Now().AddDate(1, 0, 0),
SubjectKeyId: []byte{1, 2, 3, 4, 6},
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth},
KeyUsage: x509.KeyUsageDigitalSignature,
}
// server private key
serverPrivKey, err := rsa.GenerateKey(cryptorand.Reader, 4096)
if err != nil {
fmt.Println(err)
}
// sign the server cert
serverCertBytes, err := x509.CreateCertificate(cryptorand.Reader, cert, ca, &serverPrivKey.PublicKey, caPrivKey)
if err != nil {
fmt.Println(err)
}
// PEM encode the server cert and key
serverCertPEM = new(bytes.Buffer)
_ = pem.Encode(serverCertPEM, &pem.Block{
Type: "CERTIFICATE",
Bytes: serverCertBytes,
})
serverPrivKeyPEM = new(bytes.Buffer)
_ = pem.Encode(serverPrivKeyPEM, &pem.Block{
Type: "RSA PRIVATE KEY",
Bytes: x509.MarshalPKCS1PrivateKey(serverPrivKey),
})
return caPEM.Bytes(), serverCertPEM.Bytes(), serverPrivKeyPEM.Bytes()
}

View File

@ -0,0 +1,140 @@
/*
This file is a modified version of
https://github.com/caesarxuchao/example-webhook-admission-controller/blob/master/config.go
*/
/*
Copyright 2017 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package main
import (
"crypto/tls"
"crypto/x509"
"fmt"
"log"
"time"
v1 "k8s.io/api/admissionregistration/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
"github.com/golang/glog"
)
// get a clientset with in-cluster config.
func getClient() *kubernetes.Clientset {
config, err := rest.InClusterConfig()
if err != nil {
glog.Fatal(err)
}
clientset, err := kubernetes.NewForConfig(config)
if err != nil {
glog.Fatal(err)
}
return clientset
}
// retrieve the CA cert that will signed the cert used by the
// "GenericAdmissionWebhook" plugin admission controller.
func getAPIServerCert(clientset *kubernetes.Clientset) []byte {
c, err := clientset.CoreV1().ConfigMaps("kube-system").Get("extension-apiserver-authentication", metav1.GetOptions{})
if err != nil {
glog.Fatal(err)
}
pem, ok := c.Data["requestheader-client-ca-file"]
if !ok {
glog.Fatalf(fmt.Sprintf("cannot find the ca.crt in the configmap, configMap.Data is %#v", c.Data))
}
glog.Info("client-ca-file=", pem)
return []byte(pem)
}
func configTLS(clientset *kubernetes.Clientset, serverCert []byte, serverKey []byte) *tls.Config {
cert := getAPIServerCert(clientset)
apiserverCA := x509.NewCertPool()
apiserverCA.AppendCertsFromPEM(cert)
sCert, err := tls.X509KeyPair(serverCert, serverKey)
if err != nil {
glog.Fatal(err)
}
return &tls.Config{
Certificates: []tls.Certificate{sCert},
ClientCAs: apiserverCA,
ClientAuth: tls.VerifyClientCertIfGiven, // TODO: actually require client cert
}
}
// register this example webhook admission controller with the kube-apiserver
// by creating externalAdmissionHookConfigurations.
func selfRegistration(clientset *kubernetes.Clientset, caCert []byte) {
time.Sleep(10 * time.Second)
client := clientset.AdmissionregistrationV1().MutatingWebhookConfigurations()
_, err := client.Get("env-inject-webhook", metav1.GetOptions{})
if err == nil {
if err2 := client.Delete("env-inject-webhook", &metav1.DeleteOptions{}); err2 != nil {
glog.Fatal(err2)
}
}
var failurePolicy v1.FailurePolicyType = v1.Fail
var sideEffects v1.SideEffectClass = v1.SideEffectClassNone
webhookConfig := &v1.MutatingWebhookConfiguration{
ObjectMeta: metav1.ObjectMeta{
Name: "env-inject-webhook",
},
Webhooks: []v1.MutatingWebhook{
{
Name: "env-inject.zyanshu.io",
Rules: []v1.RuleWithOperations{
{
Operations: []v1.OperationType{v1.Create, v1.Update},
Rule: v1.Rule{
APIGroups: []string{""},
APIVersions: []string{"v1"},
Resources: []string{"pods"},
},
},
{
Operations: []v1.OperationType{v1.Create, v1.Update},
Rule: v1.Rule{
APIGroups: []string{"extensions"},
APIVersions: []string{"v1"},
Resources: []string{"deployments"},
},
},
},
FailurePolicy: &failurePolicy,
ClientConfig: v1.WebhookClientConfig{
Service: &v1.ServiceReference{
Namespace: "auto-pause",
Name: "webhook",
},
CABundle: caCert,
},
AdmissionReviewVersions: []string{"v1"},
SideEffects: &sideEffects,
},
},
}
if _, err := client.Create(webhookConfig); err != nil {
glog.Fatalf("Client creation failed with %s", err)
}
log.Println("CLIENT CREATED")
}

View File

@ -0,0 +1,190 @@
package main
import (
"encoding/json"
"flag"
"fmt"
"io/ioutil"
"log"
"net/http"
"strings"
"github.com/mattbaird/jsonpatch"
v1 "k8s.io/api/admission/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/serializer"
)
var (
runtimeScheme = runtime.NewScheme()
codecs = serializer.NewCodecFactory(runtimeScheme)
deserializer = codecs.UniversalDeserializer()
// TODO(https://github.com/kubernetes/kubernetes/issues/57982)
defaulter = runtime.ObjectDefaulter(runtimeScheme)
)
// the Path of the JSON patch is a JSON pointer value
// so we need to escape any "/"s in the key we add to the annotation
// https://tools.ietf.org/html/rfc6901
func escapeJSONPointer(s string) string {
esc := strings.Replace(s, "~", "~0", -1)
esc = strings.Replace(esc, "/", "~1", -1)
return esc
}
var minikubeSystemNamespaces = []string{
metav1.NamespaceSystem,
metav1.NamespacePublic,
"auto-pause",
}
func handler(w http.ResponseWriter, r *http.Request) {
log.Println("Handling a request")
body, err := ioutil.ReadAll(r.Body)
if err != nil {
log.Printf("error: %v", err)
return
}
contentType := r.Header.Get("Content-Type")
if contentType != "application/json" {
log.Printf("Wrong content type. Got: %s", contentType)
return
}
admReq := v1.AdmissionReview{}
admResp := v1.AdmissionReview{}
if _, _, err := deserializer.Decode(body, nil, &admReq); err != nil {
log.Printf("Could not decode body: %v", err)
admResp.Response = admissionError(err)
} else {
admResp.Response = getAdmissionDecision(&admReq)
}
admResp.APIVersion = "admission.k8s.io/v1"
admResp.Kind = "AdmissionReview"
resp, err := json.Marshal(admResp)
if err != nil {
log.Printf("error marshalling decision: %v", err)
}
log.Printf(string(resp))
if _, err := w.Write(resp); err != nil {
log.Printf("error writing response %v", err)
}
}
func admissionError(err error) *v1.AdmissionResponse {
return &v1.AdmissionResponse{
Result: &metav1.Status{Message: err.Error()},
}
}
func getAdmissionDecision(admReq *v1.AdmissionReview) *v1.AdmissionResponse {
req := admReq.Request
var pod corev1.Pod
err := json.Unmarshal(req.Object.Raw, &pod)
if err != nil {
log.Printf("Could not unmarshal raw object: %v", err)
return admissionError(err)
}
log.Printf("AdmissionReview for Kind=%v Namespace=%v Name=%v UID=%v Operation=%v UserInfo=%v",
req.Kind, req.Namespace, req.Name, req.UID, req.Operation, req.UserInfo)
if !shouldInject(&pod.ObjectMeta) {
log.Printf("Skipping inject for %s %s", pod.Namespace, pod.Name)
return &v1.AdmissionResponse{
Allowed: true,
UID: req.UID,
}
}
patch, err := patchConfig(&pod)
if err != nil {
log.Printf("Error creating conduit patch: %v", err)
return admissionError(err)
}
jsonPatchType := v1.PatchTypeJSONPatch
return &v1.AdmissionResponse{
Allowed: true,
Patch: patch,
PatchType: &jsonPatchType,
UID: req.UID,
}
}
func patchConfig(pod *corev1.Pod) ([]byte, error) {
var patch []jsonpatch.JsonPatchOperation
configEnv := []corev1.EnvVar{
{Name: "KUBERNETES_SERVICE_HOST", Value: "192.168.49.2"},
{Name: "KUBERNETES_SERVICE_PORT", Value: "32443"}}
for idx, container := range pod.Spec.Containers {
patch = append(patch, addEnv(container.Env, configEnv, fmt.Sprintf("/spec/containers/%d/env", idx))...)
}
return json.Marshal(patch)
}
// addEnv performs the mutation(s) needed to add the extra environment variables to the target
// resource
func addEnv(target, envVars []corev1.EnvVar, basePath string) (patch []jsonpatch.JsonPatchOperation) {
first := len(target) == 0
var value interface{}
for _, envVar := range envVars {
value = envVar
path := basePath
if first {
first = false
value = []corev1.EnvVar{envVar}
} else {
path = path + "/-"
}
patch = append(patch, jsonpatch.JsonPatchOperation{
Operation: "add",
Path: path,
Value: value,
})
}
return patch
}
func shouldInject(metadata *metav1.ObjectMeta) bool {
shouldInject := true
// don't attempt to inject pods in the Kubernetes system namespaces
for _, ns := range minikubeSystemNamespaces {
if metadata.Namespace == ns {
shouldInject = false
}
}
return shouldInject
}
func main() {
addr := flag.String("addr", ":8080", "address to serve on")
http.HandleFunc("/", handler)
flag.CommandLine.Parse([]string{}) // hack fix for https://github.com/kubernetes/kubernetes/issues/17162
log.Printf("Starting HTTPS webhook server on %+v", *addr)
cacert, serverCert, serverKey := gencerts()
clientset := getClient()
server := &http.Server{
Addr: *addr,
TLSConfig: configTLS(clientset, serverCert, serverKey),
}
go selfRegistration(clientset, cacert)
server.ListenAndServeTLS("", "")
}

View File

@ -0,0 +1,4 @@
FROM golang:1.8
ADD out/auto-pause-hook /auto-pause-hook
ENTRYPOINT ["/auto-pause-hook"]

View File

@ -0,0 +1,55 @@
apiVersion: v1
kind: Namespace
metadata:
name: auto-pause
---
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
app: env-inject
name: env-inject
namespace: auto-pause
spec:
selector:
matchLabels:
app: env-inject
replicas: 1
template:
metadata:
labels:
app: env-inject
name: env-inject
spec:
containers:
- name: webhook
image: docker.io/azhao155/auto-pause-hook:1.3
ports:
- containerPort: 8080
---
apiVersion: v1
kind: Service
metadata:
labels:
role: webhook
name: webhook
namespace: auto-pause
spec:
ports:
- port: 443
targetPort: 8080
selector:
app: env-inject
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: webhook
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: cluster-admin
subjects:
- kind: ServiceAccount
name: default
namespace: auto-pause

1
go.mod
View File

@ -56,6 +56,7 @@ require (
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51
github.com/libvirt/libvirt-go v3.9.0+incompatible
github.com/machine-drivers/docker-machine-driver-vmware v0.1.1
github.com/mattbaird/jsonpatch v0.0.0-20200820163806-098863c1fc24 // indirect
github.com/mattn/go-isatty v0.0.12
github.com/mitchellh/go-ps v1.0.0
github.com/moby/hyperkit v0.0.0-20210108224842-2f061e447e14

3
go.sum
View File

@ -354,6 +354,7 @@ github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXP
github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3 h1:zN2lZNZRflqFyxVaTIU61KNKQ9C0055u9CAfpmqUvo4=
github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3/go.mod h1:nPpo7qLxd6XL3hWJG/O60sR8ZKfMCiIoNap5GvD12KU=
github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@ -632,6 +633,8 @@ github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN
github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs=
github.com/marten-seemann/qtls v0.2.3/go.mod h1:xzjG7avBwGGbdZ8dTGxlBnLArsVKLvwmjgmPuiQEcYk=
github.com/maruel/panicparse v1.5.0/go.mod h1:aOutY/MUjdj80R0AEVI9qE2zHqig+67t2ffUDDiLzAM=
github.com/mattbaird/jsonpatch v0.0.0-20200820163806-098863c1fc24 h1:uYuGXJBAi1umT+ZS4oQJUgKtfXCAYTR+n9zw1ViT0vA=
github.com/mattbaird/jsonpatch v0.0.0-20200820163806-098863c1fc24/go.mod h1:M1qoD/MqPgTZIk0EWKB38wE28ACRfVcn+cU08jyArI0=
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
github.com/mattn/go-colorable v0.1.6 h1:6Su7aK7lXmJ/U79bYtBjLNaha4Fs1Rg9plHpcH+vvnE=