Addon: add inspektor-gadget

Signed-off-by: yolossn <sannagaraj@microsoft.com>
pull/15869/head
yolossn 2023-02-16 22:41:43 +05:30
parent 8d12bf3d19
commit c6878c4733
14 changed files with 474 additions and 1 deletions

View File

@ -60,6 +60,10 @@ var (
//go:embed istio/istio-default-profile.yaml.tmpl
IstioAssets embed.FS
// InspektorGadgetAssets assets for inspektor-gadget addon
//go:embed inspektor-gadget/*.tmpl inspektor-gadget/*.yaml
InspektorGadgetAssets embed.FS
// KongAssets assets for kong addon
//go:embed kong/kong-ingress-controller.yaml.tmpl
KongAssets embed.FS

View File

@ -0,0 +1,32 @@
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: gadget-cluster-role
rules:
- apiGroups: [""]
resources: ["namespaces", "nodes", "pods"]
verbs: ["get", "watch", "list"]
- apiGroups: [""]
resources: ["services"]
# list services is needed by network-policy gadget.
verbs: ["list"]
- apiGroups: ["gadget.kinvolk.io"]
resources: ["traces", "traces/status"]
# For traces, we need all rights on them as we define this resource.
verbs: ["delete", "deletecollection", "get", "list", "patch", "create", "update", "watch"]
- apiGroups: ["*"]
resources: ["deployments", "replicasets", "statefulsets", "daemonsets", "jobs", "cronjobs", "replicationcontrollers"]
# Required to retrieve the owner references used by the seccomp gadget.
verbs: ["get"]
- apiGroups: ["security-profiles-operator.x-k8s.io"]
resources: ["seccompprofiles"]
# Required for integration with the Kubernetes Security Profiles Operator
verbs: ["list", "watch", "create"]
- apiGroups: ["security.openshift.io"]
# It is necessary to use the 'privileged' security context constraints to be
# able mount host directories as volumes, use the host networking, among others.
# This will be used only when running on OpenShift:
# https://docs.openshift.com/container-platform/4.9/authentication/managing-security-context-constraints.html#default-sccs_configuring-internal-oauth
resources: ["securitycontextconstraints"]
resourceNames: ["privileged"]
verbs: ["use"]

View File

@ -0,0 +1,12 @@
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: gadget-cluster-role-binding
subjects:
- kind: ServiceAccount
name: gadget
namespace: gadget
roleRef:
kind: ClusterRole
name: gadget-cluster-role
apiGroup: rbac.authorization.k8s.io

View File

@ -0,0 +1,126 @@
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
annotations:
controller-gen.kubebuilder.io/version: v0.4.1
name: traces.gadget.kinvolk.io
spec:
group: gadget.kinvolk.io
names:
kind: Trace
listKind: TraceList
plural: traces
singular: trace
scope: Namespaced
versions:
- name: v1alpha1
schema:
openAPIV3Schema:
description: Trace is the Schema for the traces API
properties:
apiVersion:
description: 'APIVersion defines the versioned schema of this representation
of an object. Servers should convert recognized schemas to the latest
internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
type: string
kind:
description: 'Kind is a string value representing the REST resource this
object represents. Servers may infer this from the endpoint the client
submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
type: string
metadata:
type: object
spec:
description: TraceSpec defines the desired state of Trace
properties:
filter:
description: Filter is to tell the gadget to filter events based on
namespace, pod name, labels or container name
properties:
containerName:
description: ContainerName selects events from containers with
this name
type: string
labels:
additionalProperties:
type: string
description: Labels selects events from pods with these labels
type: object
namespace:
description: Namespace selects events from this pod namespace
type: string
podname:
description: Podname selects events from this pod name
type: string
type: object
gadget:
description: Gadget is the name of the gadget such as "seccomp"
type: string
node:
description: Node is the name of the node on which this trace should
run
type: string
output:
description: Output allows a gadget to output the results in the specified
location. * With OutputMode=Status|Stream, Output is unused * With
OutputMode=File, Output specifies the file path * With OutputMode=ExternalResource,
Output specifies the external resource (such as seccompprofiles.security-profiles-operator.x-k8s.io
for the seccomp gadget)
type: string
outputMode:
description: OutputMode is "Status", "Stream", "File" or "ExternalResource"
enum:
- Status
- Stream
- File
- ExternalResource
type: string
parameters:
additionalProperties:
type: string
description: Parameters contains gadget specific configurations.
type: object
runMode:
description: RunMode is "Auto" to automatically start the trace as
soon as the resource is created, or "Manual" to be controlled by
the "gadget.kinvolk.io/operation" annotation
enum:
- Auto
- Manual
type: string
type: object
status:
description: TraceStatus defines the observed state of Trace
properties:
operationError:
description: OperationError is the error returned by the gadget when
applying the annotation gadget.kinvolk.io/operation=
type: string
operationWarning:
description: OperationWarning is returned by the gadget to notify
about a malfunction when applying the annotation gadget.kinvolk.io/operation=.
Unlike the OperationError that represents a fatal error, the OperationWarning
could be ignored according to the context.
type: string
output:
description: Output is the output of the gadget
type: string
state:
description: State is "Started", "Stopped" or "Completed"
enum:
- Started
- Stopped
- Completed
type: string
type: object
type: object
served: true
storage: true
subresources:
status: {}
status:
acceptedNames:
kind: ""
plural: ""
conditions: []
storedVersions: []

View File

@ -0,0 +1,199 @@
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: gadget
namespace: gadget
labels:
k8s-app: gadget
spec:
selector:
matchLabels:
k8s-app: gadget
template:
metadata:
labels:
k8s-app: gadget
annotations:
# We need to set gadget container as unconfined so it is able to write
# /sys/fs/bpf as well as /sys/kernel/debug/tracing.
# Otherwise, we can have error like:
# "failed to create server failed to create folder for pinning bpf maps: mkdir /sys/fs/bpf/gadget: permission denied"
# (For reference, see: https://github.com/inspektor-gadget/inspektor-gadget/runs/3966318270?check_suite_focus=true#step:20:221)
container.apparmor.security.beta.kubernetes.io/gadget: "unconfined"
inspektor-gadget.kinvolk.io/option-hook-mode: "auto"
spec:
serviceAccount: gadget
hostPID: true
hostNetwork: true
nodeSelector:
kubernetes.io/os: "linux"
containers:
- name: gadget
terminationMessagePolicy: FallbackToLogsOnError
image: {{.CustomRegistries.InspektorGadget | default .ImageRepository | default .Registries.InspektorGadget }}{{.Images.InspektorGadget}}
imagePullPolicy: "Always"
command: [ "/entrypoint.sh" ]
lifecycle:
preStop:
exec:
command:
- "/cleanup.sh"
readinessProbe:
periodSeconds: 5
timeoutSeconds: 2
exec:
command:
- /bin/gadgettracermanager
- -liveness
livenessProbe:
periodSeconds: 5
timeoutSeconds: 2
exec:
command:
- /bin/gadgettracermanager
- -liveness
env:
- name: NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
- name: GADGET_POD_UID
valueFrom:
fieldRef:
fieldPath: metadata.uid
- name: TRACELOOP_NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
- name: TRACELOOP_POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: TRACELOOP_POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: GADGET_IMAGE
value: "ghcr.io/inspektor-gadget/inspektor-gadget"
- name: INSPEKTOR_GADGET_VERSION
value: "v0.16.1"
- name: INSPEKTOR_GADGET_OPTION_HOOK_MODE
value: "auto"
- name: INSPEKTOR_GADGET_OPTION_FALLBACK_POD_INFORMER
value: "true"
# Make sure to keep these settings in sync with pkg/container-utils/runtime-client/interface.go
- name: INSPEKTOR_GADGET_CONTAINERD_SOCKETPATH
value: "/run/containerd/containerd.sock"
- name: INSPEKTOR_GADGET_CRIO_SOCKETPATH
value: "/run/crio/crio.sock"
- name: INSPEKTOR_GADGET_DOCKER_SOCKETPATH
value: "/run/docker.sock"
- name: HOST_ROOT
value: "/host"
securityContext:
capabilities:
add:
# We need CAP_NET_ADMIN to be able to create BPF link.
# Indeed, link_create is called with prog->type which equals
# BPF_PROG_TYPE_CGROUP_SKB.
# This value is then checked in
# bpf_prog_attach_check_attach_type() which also checks if we have
# CAP_NET_ADMIN:
# https://elixir.bootlin.com/linux/v5.14.14/source/kernel/bpf/syscall.c#L4099
# https://elixir.bootlin.com/linux/v5.14.14/source/kernel/bpf/syscall.c#L2967
- NET_ADMIN
# We need CAP_SYS_ADMIN to use Python-BCC gadgets because bcc
# internally calls bpf_get_map_fd_by_id() which contains the
# following snippet:
# if (!capable(CAP_SYS_ADMIN))
# return -EPERM;
# (https://elixir.bootlin.com/linux/v5.10.73/source/kernel/bpf/syscall.c#L3254)
#
# Details about this are given in:
# > The important design decision is to allow ID->FD transition for
# CAP_SYS_ADMIN only. What it means that user processes can run
# with CAP_BPF and CAP_NET_ADMIN and they will not be able to affect each
# other unless they pass FDs via scm_rights or via pinning in bpffs.
# ID->FD is a mechanism for human override and introspection.
# An admin can do 'sudo bpftool prog ...'. It's possible to enforce via LSM that
# only bpftool binary does bpf syscall with CAP_SYS_ADMIN and the rest of user
# space processes do bpf syscall with CAP_BPF isolating bpf objects (progs, maps,
# links) that are owned by such processes from each other.
# (https://lwn.net/Articles/820560/)
#
# Note that even with a kernel providing CAP_BPF, the above
# statement is still true.
- SYS_ADMIN
# We need this capability to get addresses from /proc/kallsyms.
# Without it, addresses displayed when reading this file will be
# 0.
# Thus, bcc_procutils_each_ksym will never call callback, so KSyms
# syms_ vector will be empty and it will return false.
# As a consequence, no prefix will be found in
# get_syscall_prefix(), so a default prefix (_sys) will be
# returned.
# Sadly, this default prefix is not used by the running kernel,
# which instead uses: __x64_sys_
- SYSLOG
# traceloop gadget uses strace which in turns use ptrace()
# syscall.
# Within kernel code, ptrace() calls ptrace_attach() which in
# turns calls __ptrace_may_access() which calls ptrace_has_cap()
# where CAP_SYS_PTRACE is finally checked:
# https://elixir.bootlin.com/linux/v5.14.14/source/kernel/ptrace.c#L284
- SYS_PTRACE
# Needed by setrlimit in gadgettracermanager and by the traceloop
# gadget.
- SYS_RESOURCE
# Needed for gadgets that don't dumb the memory rlimit.
# (Currently only applies to BCC python-based gadgets)
- IPC_LOCK
# Needed by BCC python-based gadgets to load the kheaders module:
# https://github.com/iovisor/bcc/blob/v0.24.0/src/cc/frontends/clang/kbuild_helper.cc#L158
- SYS_MODULE
# Needed by gadgets that open a raw sock like dns and snisnoop
- NET_RAW
volumeMounts:
- name: host
mountPath: /host
- name: run
mountPath: /run
- name: modules
mountPath: /lib/modules
- name: debugfs
mountPath: /sys/kernel/debug
- name: cgroup
mountPath: /sys/fs/cgroup
- name: bpffs
mountPath: /sys/fs/bpf
tolerations:
- effect: NoSchedule
operator: Exists
- effect: NoExecute
operator: Exists
volumes:
- name: host
hostPath:
path: /
- name: run
hostPath:
path: /run
- name: cgroup
hostPath:
path: /sys/fs/cgroup
- name: modules
hostPath:
path: /lib/modules
- name: bpffs
hostPath:
path: /sys/fs/bpf
- name: debugfs
hostPath:
path: /sys/kernel/debug

View File

@ -0,0 +1,4 @@
apiVersion: v1
kind: Namespace
metadata:
name: gadget

View File

@ -0,0 +1,10 @@
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
namespace: gadget
name: gadget-role
rules:
- apiGroups: [""]
resources: ["pods"]
# update is needed by traceloop gadget.
verbs: ["update"]

View File

@ -0,0 +1,12 @@
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: gadget-role-binding
namespace: gadget
subjects:
- kind: ServiceAccount
name: gadget
roleRef:
kind: Role
name: gadget-role
apiGroup: rbac.authorization.k8s.io

View File

@ -0,0 +1,5 @@
apiVersion: v1
kind: ServiceAccount
metadata:
name: gadget
namespace: gadget

View File

@ -99,6 +99,11 @@ var Addons = []*Addon{
set: SetBool,
callbacks: []setFn{EnableOrDisableAddon},
},
{
name: "inspektor-gadget",
set: SetBool,
callbacks: []setFn{EnableOrDisableAddon},
},
{
name: "kong",
set: SetBool,

View File

@ -277,6 +277,21 @@ var Addons = map[string]*Addon{
"istio-default-profile.yaml",
"0640"),
}, false, "istio", "3rd party (Istio)", "", "https://istio.io/latest/docs/setup/platform-setup/minikube/", nil, nil),
"inspektor-gadget": NewAddon([]*BinAsset{
MustBinAsset(addons.InspektorGadgetAssets, "inspektor-gadget/ig-namespace.yaml", vmpath.GuestAddonsDir, "ig-namespace.yaml", "0640"),
MustBinAsset(addons.InspektorGadgetAssets, "inspektor-gadget/ig-serviceaccount.yaml", vmpath.GuestAddonsDir, "ig-serviceaccount.yaml", "0640"),
MustBinAsset(addons.InspektorGadgetAssets, "inspektor-gadget/ig-role.yaml", vmpath.GuestAddonsDir, "ig-role.yaml", "0640"),
MustBinAsset(addons.InspektorGadgetAssets, "inspektor-gadget/ig-rolebinding.yaml", vmpath.GuestAddonsDir, "ig-rolebinding.yaml", "0640"),
MustBinAsset(addons.InspektorGadgetAssets, "inspektor-gadget/ig-clusterrole.yaml", vmpath.GuestAddonsDir, "ig-clusterrole.yaml", "0640"),
MustBinAsset(addons.InspektorGadgetAssets, "inspektor-gadget/ig-clusterrolebinding.yaml", vmpath.GuestAddonsDir, "ig-clusterrolebinding.yaml", "0640"),
MustBinAsset(addons.InspektorGadgetAssets, "inspektor-gadget/ig-crd.yaml", vmpath.GuestAddonsDir, "ig-crd.yaml", "0640"),
MustBinAsset(addons.InspektorGadgetAssets, "inspektor-gadget/ig-daemonset.yaml.tmpl", vmpath.GuestAddonsDir, "ig-daemonset.yaml", "0640"),
}, false, "inspektor-gadget", "3rd party (inspektor-gadget.io)", "https://github.com/orgs/inspektor-gadget/people", "https://minikube.sigs.k8s.io/docs/handbook/addons/inspektor-gadget/",
map[string]string{
"InspektorGadget": "inspektor-gadget/inspektor-gadget:v0.16.1@sha256:05515b802480613ae9f41e45e3b73449cf3a71fa781c697f182be0ac07319a2e",
}, map[string]string{
"InspektorGadget": "ghcr.io",
}),
"kong": NewAddon([]*BinAsset{
MustBinAsset(addons.KongAssets,
"kong/kong-ingress-controller.yaml.tmpl",

View File

@ -47,6 +47,9 @@ tests the GCP Auth addon with either phony or real credentials and makes sure th
#### validateHeadlampAddon
#### validateInspektorGadgetAddon
tests the inspektor-gadget addon by ensuring the pod has come up and addon disables
#### validateCloudSpannerAddon
tests the cloud-spanner addon by ensuring the deployment and pod come up and addon disables

View File

@ -0,0 +1,33 @@
---
title: "Using Inspektor Gadget Addon"
linkTitle: "Inspektor Gadget"
weight: 1
date: 2023-02-16
---
## Inspektor Gadget Addon
[Inspektor Gadget](https://github.com/inspektor-gadget/inspektor-gadget) is a collection of tools (or gadgets) to debug and inspect Kubernetes resources and applications. It manages the packaging, deployment and execution of [eBPF](https://ebpf.io/) programs in a Kubernetes cluster, including many based on [BCC](https://github.com/iovisor/bcc) tools, as well as some developed specifically for use in Inspektor Gadget. It automatically maps low-level kernel primitives to high-level Kubernetes resources, making it easier and quicker to find the relevant information.
### Enable Inspektor Gadget on minikube
To enable this addon, simply run:
```shell script
minikube addons enable inspektor-gadget
```
### Testing installation
```shell script
kubectl get pods -n gadget
```
If everything went well, there should be no errors about Inspektor Gadget's installation in your minikube cluster.
### Disable Inspektor Gadget
To disable this addon, simply run:
```shell script
minikube addons disable inspektor-gadget
```

View File

@ -78,7 +78,7 @@ func TestAddons(t *testing.T) {
// so we override that here to let minikube auto-detect appropriate cgroup driver
os.Setenv(constants.MinikubeForceSystemdEnv, "")
args := append([]string{"start", "-p", profile, "--wait=true", "--memory=4000", "--alsologtostderr", "--addons=registry", "--addons=metrics-server", "--addons=volumesnapshots", "--addons=csi-hostpath-driver", "--addons=gcp-auth", "--addons=cloud-spanner"}, StartArgs()...)
args := append([]string{"start", "-p", profile, "--wait=true", "--memory=4000", "--alsologtostderr", "--addons=registry", "--addons=metrics-server", "--addons=volumesnapshots", "--addons=csi-hostpath-driver", "--addons=gcp-auth", "--addons=cloud-spanner", "--addons=inspektor-gadget"}, StartArgs()...)
if !NoneDriver() { // none driver does not support ingress
args = append(args, "--addons=ingress", "--addons=ingress-dns")
}
@ -104,6 +104,7 @@ func TestAddons(t *testing.T) {
}{
{"Registry", validateRegistryAddon},
{"Ingress", validateIngressAddon},
{"InspektorGadget", validateInspektorGadgetAddon},
{"MetricsServer", validateMetricsServerAddon},
{"HelmTiller", validateHelmTillerAddon},
{"Olm", validateOlmAddon},
@ -796,6 +797,18 @@ func validateHeadlampAddon(ctx context.Context, t *testing.T, profile string) {
}
}
// validateInspektorGadgetAddon tests the inspektor-gadget addon by ensuring the pod has come up and addon disables
func validateInspektorGadgetAddon(ctx context.Context, t *testing.T, profile string) {
defer PostMortemLogs(t, profile)
if _, err := PodWait(ctx, t, profile, "gadget", "k8s-app=gadget", Minutes(8)); err != nil {
t.Fatalf("failed waiting for inspektor-gadget pod: %v", err)
}
if rr, err := Run(t, exec.CommandContext(ctx, Target(), "addons", "disable", "inspektor-gadget", "-p", profile)); err != nil {
t.Errorf("failed to disable inspektor-gadget addon: args %q : %v", rr.Command(), err)
}
}
// validateCloudSpannerAddon tests the cloud-spanner addon by ensuring the deployment and pod come up and addon disables
func validateCloudSpannerAddon(ctx context.Context, t *testing.T, profile string) {
defer PostMortemLogs(t, profile)