diff --git a/deploy/addons/assets.go b/deploy/addons/assets.go index 4ea4b5baf4..3c744bd40e 100644 --- a/deploy/addons/assets.go +++ b/deploy/addons/assets.go @@ -60,6 +60,10 @@ var ( //go:embed istio/istio-default-profile.yaml.tmpl IstioAssets embed.FS + // InspektorGadgetAssets assets for inspektor-gadget addon + //go:embed inspektor-gadget/*.tmpl inspektor-gadget/*.yaml + InspektorGadgetAssets embed.FS + // KongAssets assets for kong addon //go:embed kong/kong-ingress-controller.yaml.tmpl KongAssets embed.FS diff --git a/deploy/addons/inspektor-gadget/ig-clusterrole.yaml b/deploy/addons/inspektor-gadget/ig-clusterrole.yaml new file mode 100644 index 0000000000..afdabdd249 --- /dev/null +++ b/deploy/addons/inspektor-gadget/ig-clusterrole.yaml @@ -0,0 +1,32 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: gadget-cluster-role +rules: +- apiGroups: [""] + resources: ["namespaces", "nodes", "pods"] + verbs: ["get", "watch", "list"] +- apiGroups: [""] + resources: ["services"] + # list services is needed by network-policy gadget. + verbs: ["list"] +- apiGroups: ["gadget.kinvolk.io"] + resources: ["traces", "traces/status"] + # For traces, we need all rights on them as we define this resource. + verbs: ["delete", "deletecollection", "get", "list", "patch", "create", "update", "watch"] +- apiGroups: ["*"] + resources: ["deployments", "replicasets", "statefulsets", "daemonsets", "jobs", "cronjobs", "replicationcontrollers"] + # Required to retrieve the owner references used by the seccomp gadget. + verbs: ["get"] +- apiGroups: ["security-profiles-operator.x-k8s.io"] + resources: ["seccompprofiles"] + # Required for integration with the Kubernetes Security Profiles Operator + verbs: ["list", "watch", "create"] +- apiGroups: ["security.openshift.io"] + # It is necessary to use the 'privileged' security context constraints to be + # able mount host directories as volumes, use the host networking, among others. + # This will be used only when running on OpenShift: + # https://docs.openshift.com/container-platform/4.9/authentication/managing-security-context-constraints.html#default-sccs_configuring-internal-oauth + resources: ["securitycontextconstraints"] + resourceNames: ["privileged"] + verbs: ["use"] \ No newline at end of file diff --git a/deploy/addons/inspektor-gadget/ig-clusterrolebinding.yaml b/deploy/addons/inspektor-gadget/ig-clusterrolebinding.yaml new file mode 100644 index 0000000000..3f9d3bc080 --- /dev/null +++ b/deploy/addons/inspektor-gadget/ig-clusterrolebinding.yaml @@ -0,0 +1,12 @@ +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: gadget-cluster-role-binding +subjects: +- kind: ServiceAccount + name: gadget + namespace: gadget +roleRef: + kind: ClusterRole + name: gadget-cluster-role + apiGroup: rbac.authorization.k8s.io \ No newline at end of file diff --git a/deploy/addons/inspektor-gadget/ig-crd.yaml b/deploy/addons/inspektor-gadget/ig-crd.yaml new file mode 100644 index 0000000000..4243ed5c84 --- /dev/null +++ b/deploy/addons/inspektor-gadget/ig-crd.yaml @@ -0,0 +1,126 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.4.1 + name: traces.gadget.kinvolk.io +spec: + group: gadget.kinvolk.io + names: + kind: Trace + listKind: TraceList + plural: traces + singular: trace + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: Trace is the Schema for the traces API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: TraceSpec defines the desired state of Trace + properties: + filter: + description: Filter is to tell the gadget to filter events based on + namespace, pod name, labels or container name + properties: + containerName: + description: ContainerName selects events from containers with + this name + type: string + labels: + additionalProperties: + type: string + description: Labels selects events from pods with these labels + type: object + namespace: + description: Namespace selects events from this pod namespace + type: string + podname: + description: Podname selects events from this pod name + type: string + type: object + gadget: + description: Gadget is the name of the gadget such as "seccomp" + type: string + node: + description: Node is the name of the node on which this trace should + run + type: string + output: + description: Output allows a gadget to output the results in the specified + location. * With OutputMode=Status|Stream, Output is unused * With + OutputMode=File, Output specifies the file path * With OutputMode=ExternalResource, + Output specifies the external resource (such as seccompprofiles.security-profiles-operator.x-k8s.io + for the seccomp gadget) + type: string + outputMode: + description: OutputMode is "Status", "Stream", "File" or "ExternalResource" + enum: + - Status + - Stream + - File + - ExternalResource + type: string + parameters: + additionalProperties: + type: string + description: Parameters contains gadget specific configurations. + type: object + runMode: + description: RunMode is "Auto" to automatically start the trace as + soon as the resource is created, or "Manual" to be controlled by + the "gadget.kinvolk.io/operation" annotation + enum: + - Auto + - Manual + type: string + type: object + status: + description: TraceStatus defines the observed state of Trace + properties: + operationError: + description: OperationError is the error returned by the gadget when + applying the annotation gadget.kinvolk.io/operation= + type: string + operationWarning: + description: OperationWarning is returned by the gadget to notify + about a malfunction when applying the annotation gadget.kinvolk.io/operation=. + Unlike the OperationError that represents a fatal error, the OperationWarning + could be ignored according to the context. + type: string + output: + description: Output is the output of the gadget + type: string + state: + description: State is "Started", "Stopped" or "Completed" + enum: + - Started + - Stopped + - Completed + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] \ No newline at end of file diff --git a/deploy/addons/inspektor-gadget/ig-daemonset.yaml.tmpl b/deploy/addons/inspektor-gadget/ig-daemonset.yaml.tmpl new file mode 100644 index 0000000000..e358f9fe0c --- /dev/null +++ b/deploy/addons/inspektor-gadget/ig-daemonset.yaml.tmpl @@ -0,0 +1,199 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: gadget + namespace: gadget + labels: + k8s-app: gadget +spec: + selector: + matchLabels: + k8s-app: gadget + template: + metadata: + labels: + k8s-app: gadget + annotations: + # We need to set gadget container as unconfined so it is able to write + # /sys/fs/bpf as well as /sys/kernel/debug/tracing. + # Otherwise, we can have error like: + # "failed to create server failed to create folder for pinning bpf maps: mkdir /sys/fs/bpf/gadget: permission denied" + # (For reference, see: https://github.com/inspektor-gadget/inspektor-gadget/runs/3966318270?check_suite_focus=true#step:20:221) + container.apparmor.security.beta.kubernetes.io/gadget: "unconfined" + inspektor-gadget.kinvolk.io/option-hook-mode: "auto" + spec: + serviceAccount: gadget + hostPID: true + hostNetwork: true + nodeSelector: + kubernetes.io/os: "linux" + containers: + - name: gadget + terminationMessagePolicy: FallbackToLogsOnError + image: {{.CustomRegistries.InspektorGadget | default .ImageRepository | default .Registries.InspektorGadget }}{{.Images.InspektorGadget}} + imagePullPolicy: "Always" + command: [ "/entrypoint.sh" ] + lifecycle: + preStop: + exec: + command: + - "/cleanup.sh" + readinessProbe: + periodSeconds: 5 + timeoutSeconds: 2 + exec: + command: + - /bin/gadgettracermanager + - -liveness + livenessProbe: + periodSeconds: 5 + timeoutSeconds: 2 + exec: + command: + - /bin/gadgettracermanager + - -liveness + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: GADGET_POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + - name: TRACELOOP_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: TRACELOOP_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: TRACELOOP_POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: GADGET_IMAGE + value: "ghcr.io/inspektor-gadget/inspektor-gadget" + - name: INSPEKTOR_GADGET_VERSION + value: "v0.16.1" + - name: INSPEKTOR_GADGET_OPTION_HOOK_MODE + value: "auto" + - name: INSPEKTOR_GADGET_OPTION_FALLBACK_POD_INFORMER + value: "true" + # Make sure to keep these settings in sync with pkg/container-utils/runtime-client/interface.go + - name: INSPEKTOR_GADGET_CONTAINERD_SOCKETPATH + value: "/run/containerd/containerd.sock" + - name: INSPEKTOR_GADGET_CRIO_SOCKETPATH + value: "/run/crio/crio.sock" + - name: INSPEKTOR_GADGET_DOCKER_SOCKETPATH + value: "/run/docker.sock" + - name: HOST_ROOT + value: "/host" + securityContext: + capabilities: + add: + # We need CAP_NET_ADMIN to be able to create BPF link. + # Indeed, link_create is called with prog->type which equals + # BPF_PROG_TYPE_CGROUP_SKB. + # This value is then checked in + # bpf_prog_attach_check_attach_type() which also checks if we have + # CAP_NET_ADMIN: + # https://elixir.bootlin.com/linux/v5.14.14/source/kernel/bpf/syscall.c#L4099 + # https://elixir.bootlin.com/linux/v5.14.14/source/kernel/bpf/syscall.c#L2967 + - NET_ADMIN + + # We need CAP_SYS_ADMIN to use Python-BCC gadgets because bcc + # internally calls bpf_get_map_fd_by_id() which contains the + # following snippet: + # if (!capable(CAP_SYS_ADMIN)) + # return -EPERM; + # (https://elixir.bootlin.com/linux/v5.10.73/source/kernel/bpf/syscall.c#L3254) + # + # Details about this are given in: + # > The important design decision is to allow ID->FD transition for + # CAP_SYS_ADMIN only. What it means that user processes can run + # with CAP_BPF and CAP_NET_ADMIN and they will not be able to affect each + # other unless they pass FDs via scm_rights or via pinning in bpffs. + # ID->FD is a mechanism for human override and introspection. + # An admin can do 'sudo bpftool prog ...'. It's possible to enforce via LSM that + # only bpftool binary does bpf syscall with CAP_SYS_ADMIN and the rest of user + # space processes do bpf syscall with CAP_BPF isolating bpf objects (progs, maps, + # links) that are owned by such processes from each other. + # (https://lwn.net/Articles/820560/) + # + # Note that even with a kernel providing CAP_BPF, the above + # statement is still true. + - SYS_ADMIN + + # We need this capability to get addresses from /proc/kallsyms. + # Without it, addresses displayed when reading this file will be + # 0. + # Thus, bcc_procutils_each_ksym will never call callback, so KSyms + # syms_ vector will be empty and it will return false. + # As a consequence, no prefix will be found in + # get_syscall_prefix(), so a default prefix (_sys) will be + # returned. + # Sadly, this default prefix is not used by the running kernel, + # which instead uses: __x64_sys_ + - SYSLOG + + # traceloop gadget uses strace which in turns use ptrace() + # syscall. + # Within kernel code, ptrace() calls ptrace_attach() which in + # turns calls __ptrace_may_access() which calls ptrace_has_cap() + # where CAP_SYS_PTRACE is finally checked: + # https://elixir.bootlin.com/linux/v5.14.14/source/kernel/ptrace.c#L284 + - SYS_PTRACE + + # Needed by setrlimit in gadgettracermanager and by the traceloop + # gadget. + - SYS_RESOURCE + + # Needed for gadgets that don't dumb the memory rlimit. + # (Currently only applies to BCC python-based gadgets) + - IPC_LOCK + + # Needed by BCC python-based gadgets to load the kheaders module: + # https://github.com/iovisor/bcc/blob/v0.24.0/src/cc/frontends/clang/kbuild_helper.cc#L158 + - SYS_MODULE + + # Needed by gadgets that open a raw sock like dns and snisnoop + - NET_RAW + volumeMounts: + - name: host + mountPath: /host + - name: run + mountPath: /run + - name: modules + mountPath: /lib/modules + - name: debugfs + mountPath: /sys/kernel/debug + - name: cgroup + mountPath: /sys/fs/cgroup + - name: bpffs + mountPath: /sys/fs/bpf + tolerations: + - effect: NoSchedule + operator: Exists + - effect: NoExecute + operator: Exists + volumes: + - name: host + hostPath: + path: / + - name: run + hostPath: + path: /run + - name: cgroup + hostPath: + path: /sys/fs/cgroup + - name: modules + hostPath: + path: /lib/modules + - name: bpffs + hostPath: + path: /sys/fs/bpf + - name: debugfs + hostPath: + path: /sys/kernel/debug \ No newline at end of file diff --git a/deploy/addons/inspektor-gadget/ig-namespace.yaml b/deploy/addons/inspektor-gadget/ig-namespace.yaml new file mode 100644 index 0000000000..878088faae --- /dev/null +++ b/deploy/addons/inspektor-gadget/ig-namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: gadget \ No newline at end of file diff --git a/deploy/addons/inspektor-gadget/ig-role.yaml b/deploy/addons/inspektor-gadget/ig-role.yaml new file mode 100644 index 0000000000..7b73377180 --- /dev/null +++ b/deploy/addons/inspektor-gadget/ig-role.yaml @@ -0,0 +1,10 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + namespace: gadget + name: gadget-role +rules: +- apiGroups: [""] + resources: ["pods"] + # update is needed by traceloop gadget. + verbs: ["update"] \ No newline at end of file diff --git a/deploy/addons/inspektor-gadget/ig-rolebinding.yaml b/deploy/addons/inspektor-gadget/ig-rolebinding.yaml new file mode 100644 index 0000000000..35a7b905ce --- /dev/null +++ b/deploy/addons/inspektor-gadget/ig-rolebinding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: gadget-role-binding + namespace: gadget +subjects: +- kind: ServiceAccount + name: gadget +roleRef: + kind: Role + name: gadget-role + apiGroup: rbac.authorization.k8s.io \ No newline at end of file diff --git a/deploy/addons/inspektor-gadget/ig-serviceaccount.yaml b/deploy/addons/inspektor-gadget/ig-serviceaccount.yaml new file mode 100644 index 0000000000..c0102b538e --- /dev/null +++ b/deploy/addons/inspektor-gadget/ig-serviceaccount.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: gadget + namespace: gadget \ No newline at end of file diff --git a/pkg/addons/config.go b/pkg/addons/config.go index 5e74c34606..5ad465de4d 100644 --- a/pkg/addons/config.go +++ b/pkg/addons/config.go @@ -99,6 +99,11 @@ var Addons = []*Addon{ set: SetBool, callbacks: []setFn{EnableOrDisableAddon}, }, + { + name: "inspektor-gadget", + set: SetBool, + callbacks: []setFn{EnableOrDisableAddon}, + }, { name: "kong", set: SetBool, diff --git a/pkg/minikube/assets/addons.go b/pkg/minikube/assets/addons.go index 2da8ec94f8..c915f050cd 100644 --- a/pkg/minikube/assets/addons.go +++ b/pkg/minikube/assets/addons.go @@ -277,6 +277,21 @@ var Addons = map[string]*Addon{ "istio-default-profile.yaml", "0640"), }, false, "istio", "3rd party (Istio)", "", "https://istio.io/latest/docs/setup/platform-setup/minikube/", nil, nil), + "inspektor-gadget": NewAddon([]*BinAsset{ + MustBinAsset(addons.InspektorGadgetAssets, "inspektor-gadget/ig-namespace.yaml", vmpath.GuestAddonsDir, "ig-namespace.yaml", "0640"), + MustBinAsset(addons.InspektorGadgetAssets, "inspektor-gadget/ig-serviceaccount.yaml", vmpath.GuestAddonsDir, "ig-serviceaccount.yaml", "0640"), + MustBinAsset(addons.InspektorGadgetAssets, "inspektor-gadget/ig-role.yaml", vmpath.GuestAddonsDir, "ig-role.yaml", "0640"), + MustBinAsset(addons.InspektorGadgetAssets, "inspektor-gadget/ig-rolebinding.yaml", vmpath.GuestAddonsDir, "ig-rolebinding.yaml", "0640"), + MustBinAsset(addons.InspektorGadgetAssets, "inspektor-gadget/ig-clusterrole.yaml", vmpath.GuestAddonsDir, "ig-clusterrole.yaml", "0640"), + MustBinAsset(addons.InspektorGadgetAssets, "inspektor-gadget/ig-clusterrolebinding.yaml", vmpath.GuestAddonsDir, "ig-clusterrolebinding.yaml", "0640"), + MustBinAsset(addons.InspektorGadgetAssets, "inspektor-gadget/ig-crd.yaml", vmpath.GuestAddonsDir, "ig-crd.yaml", "0640"), + MustBinAsset(addons.InspektorGadgetAssets, "inspektor-gadget/ig-daemonset.yaml.tmpl", vmpath.GuestAddonsDir, "ig-daemonset.yaml", "0640"), + }, false, "inspektor-gadget", "3rd party (inspektor-gadget.io)", "https://github.com/orgs/inspektor-gadget/people", "https://minikube.sigs.k8s.io/docs/handbook/addons/inspektor-gadget/", + map[string]string{ + "InspektorGadget": "inspektor-gadget/inspektor-gadget:v0.16.1@sha256:05515b802480613ae9f41e45e3b73449cf3a71fa781c697f182be0ac07319a2e", + }, map[string]string{ + "InspektorGadget": "ghcr.io", + }), "kong": NewAddon([]*BinAsset{ MustBinAsset(addons.KongAssets, "kong/kong-ingress-controller.yaml.tmpl", diff --git a/site/content/en/docs/contrib/tests.en.md b/site/content/en/docs/contrib/tests.en.md index 4025bac8df..8300533e34 100644 --- a/site/content/en/docs/contrib/tests.en.md +++ b/site/content/en/docs/contrib/tests.en.md @@ -47,6 +47,9 @@ tests the GCP Auth addon with either phony or real credentials and makes sure th #### validateHeadlampAddon +#### validateInspektorGadgetAddon +tests the inspektor-gadget addon by ensuring the pod has come up and addon disables + #### validateCloudSpannerAddon tests the cloud-spanner addon by ensuring the deployment and pod come up and addon disables diff --git a/site/content/en/docs/handbook/addons/inspektor-gadget.md b/site/content/en/docs/handbook/addons/inspektor-gadget.md new file mode 100644 index 0000000000..45e8bd453c --- /dev/null +++ b/site/content/en/docs/handbook/addons/inspektor-gadget.md @@ -0,0 +1,33 @@ +--- +title: "Using Inspektor Gadget Addon" +linkTitle: "Inspektor Gadget" +weight: 1 +date: 2023-02-16 +--- + +## Inspektor Gadget Addon + +[Inspektor Gadget](https://github.com/inspektor-gadget/inspektor-gadget) is a collection of tools (or gadgets) to debug and inspect Kubernetes resources and applications. It manages the packaging, deployment and execution of [eBPF](https://ebpf.io/) programs in a Kubernetes cluster, including many based on [BCC](https://github.com/iovisor/bcc) tools, as well as some developed specifically for use in Inspektor Gadget. It automatically maps low-level kernel primitives to high-level Kubernetes resources, making it easier and quicker to find the relevant information. + +### Enable Inspektor Gadget on minikube + +To enable this addon, simply run: +```shell script +minikube addons enable inspektor-gadget +``` + +### Testing installation + +```shell script +kubectl get pods -n gadget +``` + +If everything went well, there should be no errors about Inspektor Gadget's installation in your minikube cluster. + +### Disable Inspektor Gadget + +To disable this addon, simply run: + +```shell script +minikube addons disable inspektor-gadget +``` \ No newline at end of file diff --git a/test/integration/addons_test.go b/test/integration/addons_test.go index f40a4f93cb..077c89c1e6 100644 --- a/test/integration/addons_test.go +++ b/test/integration/addons_test.go @@ -78,7 +78,7 @@ func TestAddons(t *testing.T) { // so we override that here to let minikube auto-detect appropriate cgroup driver os.Setenv(constants.MinikubeForceSystemdEnv, "") - args := append([]string{"start", "-p", profile, "--wait=true", "--memory=4000", "--alsologtostderr", "--addons=registry", "--addons=metrics-server", "--addons=volumesnapshots", "--addons=csi-hostpath-driver", "--addons=gcp-auth", "--addons=cloud-spanner"}, StartArgs()...) + args := append([]string{"start", "-p", profile, "--wait=true", "--memory=4000", "--alsologtostderr", "--addons=registry", "--addons=metrics-server", "--addons=volumesnapshots", "--addons=csi-hostpath-driver", "--addons=gcp-auth", "--addons=cloud-spanner", "--addons=inspektor-gadget"}, StartArgs()...) if !NoneDriver() { // none driver does not support ingress args = append(args, "--addons=ingress", "--addons=ingress-dns") } @@ -104,6 +104,7 @@ func TestAddons(t *testing.T) { }{ {"Registry", validateRegistryAddon}, {"Ingress", validateIngressAddon}, + {"InspektorGadget", validateInspektorGadgetAddon}, {"MetricsServer", validateMetricsServerAddon}, {"HelmTiller", validateHelmTillerAddon}, {"Olm", validateOlmAddon}, @@ -796,6 +797,18 @@ func validateHeadlampAddon(ctx context.Context, t *testing.T, profile string) { } } +// validateInspektorGadgetAddon tests the inspektor-gadget addon by ensuring the pod has come up and addon disables +func validateInspektorGadgetAddon(ctx context.Context, t *testing.T, profile string) { + defer PostMortemLogs(t, profile) + + if _, err := PodWait(ctx, t, profile, "gadget", "k8s-app=gadget", Minutes(8)); err != nil { + t.Fatalf("failed waiting for inspektor-gadget pod: %v", err) + } + if rr, err := Run(t, exec.CommandContext(ctx, Target(), "addons", "disable", "inspektor-gadget", "-p", profile)); err != nil { + t.Errorf("failed to disable inspektor-gadget addon: args %q : %v", rr.Command(), err) + } +} + // validateCloudSpannerAddon tests the cloud-spanner addon by ensuring the deployment and pod come up and addon disables func validateCloudSpannerAddon(ctx context.Context, t *testing.T, profile string) { defer PostMortemLogs(t, profile)