Addon: add inspektor-gadget

Signed-off-by: yolossn <sannagaraj@microsoft.com>

deploy/addons/assets.go

@@ -60,6 +60,10 @@ var (
 	//go:embed istio/istio-default-profile.yaml.tmpl
 	IstioAssets embed.FS
 
+	// InspektorGadgetAssets assets for inspektor-gadget addon
+	//go:embed inspektor-gadget/*.tmpl inspektor-gadget/*.yaml
+	InspektorGadgetAssets embed.FS
+
 	// KongAssets assets for kong addon
 	//go:embed kong/kong-ingress-controller.yaml.tmpl
 	KongAssets embed.FS

deploy/addons/inspektor-gadget/ig-clusterrole.yaml

@@ -0,0 +1,32 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: gadget-cluster-role
+rules:
+- apiGroups: [""]
+  resources: ["namespaces", "nodes", "pods"]
+  verbs: ["get", "watch", "list"]
+- apiGroups: [""]
+  resources: ["services"]
+  # list services is needed by network-policy gadget.
+  verbs: ["list"]
+- apiGroups: ["gadget.kinvolk.io"]
+  resources: ["traces", "traces/status"]
+  # For traces, we need all rights on them as we define this resource.
+  verbs: ["delete", "deletecollection", "get", "list", "patch", "create", "update", "watch"]
+- apiGroups: ["*"]
+  resources: ["deployments", "replicasets", "statefulsets", "daemonsets", "jobs", "cronjobs", "replicationcontrollers"]
+  # Required to retrieve the owner references used by the seccomp gadget.
+  verbs: ["get"]
+- apiGroups: ["security-profiles-operator.x-k8s.io"]
+  resources: ["seccompprofiles"]
+  # Required for integration with the Kubernetes Security Profiles Operator
+  verbs: ["list", "watch", "create"]
+- apiGroups: ["security.openshift.io"]
+  # It is necessary to use the 'privileged' security context constraints to be
+  # able mount host directories as volumes, use the host networking, among others.
+  # This will be used only when running on OpenShift:
+  # https://docs.openshift.com/container-platform/4.9/authentication/managing-security-context-constraints.html#default-sccs_configuring-internal-oauth
+  resources: ["securitycontextconstraints"]
+  resourceNames: ["privileged"]
+  verbs: ["use"]

deploy/addons/inspektor-gadget/ig-clusterrolebinding.yaml

@@ -0,0 +1,12 @@
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: gadget-cluster-role-binding
+subjects:
+- kind: ServiceAccount
+  name: gadget
+  namespace: gadget
+roleRef:
+  kind: ClusterRole
+  name: gadget-cluster-role
+  apiGroup: rbac.authorization.k8s.io

deploy/addons/inspektor-gadget/ig-crd.yaml

@@ -0,0 +1,126 @@
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.4.1
+  name: traces.gadget.kinvolk.io
+spec:
+  group: gadget.kinvolk.io
+  names:
+    kind: Trace
+    listKind: TraceList
+    plural: traces
+    singular: trace
+  scope: Namespaced
+  versions:
+  - name: v1alpha1
+    schema:
+      openAPIV3Schema:
+        description: Trace is the Schema for the traces API
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation
+              of an object. Servers should convert recognized schemas to the latest
+              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this
+              object represents. Servers may infer this from the endpoint the client
+              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: TraceSpec defines the desired state of Trace
+            properties:
+              filter:
+                description: Filter is to tell the gadget to filter events based on
+                  namespace, pod name, labels or container name
+                properties:
+                  containerName:
+                    description: ContainerName selects events from containers with
+                      this name
+                    type: string
+                  labels:
+                    additionalProperties:
+                      type: string
+                    description: Labels selects events from pods with these labels
+                    type: object
+                  namespace:
+                    description: Namespace selects events from this pod namespace
+                    type: string
+                  podname:
+                    description: Podname selects events from this pod name
+                    type: string
+                type: object
+              gadget:
+                description: Gadget is the name of the gadget such as "seccomp"
+                type: string
+              node:
+                description: Node is the name of the node on which this trace should
+                  run
+                type: string
+              output:
+                description: Output allows a gadget to output the results in the specified
+                  location. * With OutputMode=Status|Stream, Output is unused * With
+                  OutputMode=File, Output specifies the file path * With OutputMode=ExternalResource,
+                  Output specifies the external   resource (such as   seccompprofiles.security-profiles-operator.x-k8s.io
+                  for the   seccomp gadget)
+                type: string
+              outputMode:
+                description: OutputMode is "Status", "Stream", "File" or "ExternalResource"
+                enum:
+                - Status
+                - Stream
+                - File
+                - ExternalResource
+                type: string
+              parameters:
+                additionalProperties:
+                  type: string
+                description: Parameters contains gadget specific configurations.
+                type: object
+              runMode:
+                description: RunMode is "Auto" to automatically start the trace as
+                  soon as the resource is created, or "Manual" to be controlled by
+                  the "gadget.kinvolk.io/operation" annotation
+                enum:
+                - Auto
+                - Manual
+                type: string
+            type: object
+          status:
+            description: TraceStatus defines the observed state of Trace
+            properties:
+              operationError:
+                description: OperationError is the error returned by the gadget when
+                  applying the annotation gadget.kinvolk.io/operation=
+                type: string
+              operationWarning:
+                description: OperationWarning is returned by the gadget to notify
+                  about a malfunction when applying the annotation gadget.kinvolk.io/operation=.
+                  Unlike the OperationError that represents a fatal error, the OperationWarning
+                  could be ignored according to the context.
+                type: string
+              output:
+                description: Output is the output of the gadget
+                type: string
+              state:
+                description: State is "Started", "Stopped" or "Completed"
+                enum:
+                - Started
+                - Stopped
+                - Completed
+                type: string
+            type: object
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []

deploy/addons/inspektor-gadget/ig-daemonset.yaml.tmpl

@@ -0,0 +1,199 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: gadget
+  namespace: gadget
+  labels:
+    k8s-app: gadget
+spec:
+  selector:
+    matchLabels:
+      k8s-app: gadget
+  template:
+    metadata:
+      labels:
+        k8s-app: gadget
+      annotations:
+        # We need to set gadget container as unconfined so it is able to write
+        # /sys/fs/bpf as well as /sys/kernel/debug/tracing.
+        # Otherwise, we can have error like:
+        # "failed to create server failed to create folder for pinning bpf maps: mkdir /sys/fs/bpf/gadget: permission denied"
+        # (For reference, see: https://github.com/inspektor-gadget/inspektor-gadget/runs/3966318270?check_suite_focus=true#step:20:221)
+        container.apparmor.security.beta.kubernetes.io/gadget: "unconfined"
+        inspektor-gadget.kinvolk.io/option-hook-mode: "auto"
+    spec:
+      serviceAccount: gadget
+      hostPID: true
+      hostNetwork: true
+      nodeSelector:
+        kubernetes.io/os: "linux"
+      containers:
+      - name: gadget
+        terminationMessagePolicy: FallbackToLogsOnError
+        image: {{.CustomRegistries.InspektorGadget  | default .ImageRepository | default .Registries.InspektorGadget }}{{.Images.InspektorGadget}}
+        imagePullPolicy: "Always"
+        command: [ "/entrypoint.sh" ]
+        lifecycle:
+          preStop:
+            exec:
+              command:
+                - "/cleanup.sh"
+        readinessProbe:
+          periodSeconds: 5
+          timeoutSeconds: 2
+          exec:
+            command:
+              - /bin/gadgettracermanager
+              - -liveness
+        livenessProbe:
+          periodSeconds: 5
+          timeoutSeconds: 2
+          exec:
+            command:
+              - /bin/gadgettracermanager
+              - -liveness
+        env:
+          - name: NODE_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: spec.nodeName
+          - name: GADGET_POD_UID
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.uid
+          - name: TRACELOOP_NODE_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: spec.nodeName
+          - name: TRACELOOP_POD_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.name
+          - name: TRACELOOP_POD_NAMESPACE
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.namespace
+          - name: GADGET_IMAGE
+            value: "ghcr.io/inspektor-gadget/inspektor-gadget"
+          - name: INSPEKTOR_GADGET_VERSION
+            value: "v0.16.1"
+          - name: INSPEKTOR_GADGET_OPTION_HOOK_MODE
+            value: "auto"
+          - name: INSPEKTOR_GADGET_OPTION_FALLBACK_POD_INFORMER
+            value: "true"
+          # Make sure to keep these settings in sync with pkg/container-utils/runtime-client/interface.go
+          - name: INSPEKTOR_GADGET_CONTAINERD_SOCKETPATH
+            value: "/run/containerd/containerd.sock"
+          - name: INSPEKTOR_GADGET_CRIO_SOCKETPATH
+            value: "/run/crio/crio.sock"
+          - name: INSPEKTOR_GADGET_DOCKER_SOCKETPATH
+            value: "/run/docker.sock"
+          - name: HOST_ROOT
+            value: "/host"
+        securityContext:
+          capabilities:
+            add:
+              # We need CAP_NET_ADMIN to be able to create BPF link.
+              # Indeed, link_create is called with prog->type which equals
+              # BPF_PROG_TYPE_CGROUP_SKB.
+              # This value is then checked in
+              # bpf_prog_attach_check_attach_type() which also checks if we have
+              # CAP_NET_ADMIN:
+              # https://elixir.bootlin.com/linux/v5.14.14/source/kernel/bpf/syscall.c#L4099
+              # https://elixir.bootlin.com/linux/v5.14.14/source/kernel/bpf/syscall.c#L2967
+              - NET_ADMIN
+
+              # We need CAP_SYS_ADMIN to use Python-BCC gadgets because bcc
+              # internally calls bpf_get_map_fd_by_id() which contains the
+              # following snippet:
+              # if (!capable(CAP_SYS_ADMIN))
+              # 	return -EPERM;
+              # (https://elixir.bootlin.com/linux/v5.10.73/source/kernel/bpf/syscall.c#L3254)
+              #
+              # Details about this are given in:
+              # > The important design decision is to allow ID->FD transition for
+              # CAP_SYS_ADMIN only. What it means that user processes can run
+              # with CAP_BPF and CAP_NET_ADMIN and they will not be able to affect each
+              # other unless they pass FDs via scm_rights or via pinning in bpffs.
+              # ID->FD is a mechanism for human override and introspection.
+              # An admin can do 'sudo bpftool prog ...'. It's possible to enforce via LSM that
+              # only bpftool binary does bpf syscall with CAP_SYS_ADMIN and the rest of user
+              # space processes do bpf syscall with CAP_BPF isolating bpf objects (progs, maps,
+              # links) that are owned by such processes from each other.
+              # (https://lwn.net/Articles/820560/)
+              #
+              # Note that even with a kernel providing CAP_BPF, the above
+              # statement is still true.
+              - SYS_ADMIN
+
+              # We need this capability to get addresses from /proc/kallsyms.
+              # Without it, addresses displayed when reading this file will be
+              # 0.
+              # Thus, bcc_procutils_each_ksym will never call callback, so KSyms
+              # syms_ vector will be empty and it will return false.
+              # As a consequence, no prefix will be found in
+              # get_syscall_prefix(), so a default prefix (_sys) will be
+              # returned.
+              # Sadly, this default prefix is not used by the running kernel,
+              # which instead uses: __x64_sys_
+              - SYSLOG
+
+              # traceloop gadget uses strace which in turns use ptrace()
+              # syscall.
+              # Within kernel code, ptrace() calls ptrace_attach() which in
+              # turns calls __ptrace_may_access() which calls ptrace_has_cap()
+              # where CAP_SYS_PTRACE is finally checked:
+              # https://elixir.bootlin.com/linux/v5.14.14/source/kernel/ptrace.c#L284
+              - SYS_PTRACE
+
+              # Needed by setrlimit in gadgettracermanager and by the traceloop
+              # gadget.
+              - SYS_RESOURCE
+
+              # Needed for gadgets that don't dumb the memory rlimit.
+              # (Currently only applies to BCC python-based gadgets)
+              - IPC_LOCK
+
+              # Needed by BCC python-based gadgets to load the kheaders module:
+              # https://github.com/iovisor/bcc/blob/v0.24.0/src/cc/frontends/clang/kbuild_helper.cc#L158
+              - SYS_MODULE
+
+              # Needed by gadgets that open a raw sock like dns and snisnoop
+              - NET_RAW
+        volumeMounts:
+        - name: host
+          mountPath: /host
+        - name: run
+          mountPath: /run
+        - name: modules
+          mountPath: /lib/modules
+        - name: debugfs
+          mountPath: /sys/kernel/debug
+        - name: cgroup
+          mountPath: /sys/fs/cgroup
+        - name: bpffs
+          mountPath: /sys/fs/bpf
+      tolerations:
+      - effect: NoSchedule
+        operator: Exists
+      - effect: NoExecute
+        operator: Exists
+      volumes:
+      - name: host
+        hostPath:
+          path: /
+      - name: run
+        hostPath:
+          path: /run
+      - name: cgroup
+        hostPath:
+          path: /sys/fs/cgroup
+      - name: modules
+        hostPath:
+          path: /lib/modules
+      - name: bpffs
+        hostPath:
+          path: /sys/fs/bpf
+      - name: debugfs
+        hostPath:
+          path: /sys/kernel/debug 

deploy/addons/inspektor-gadget/ig-namespace.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: gadget

deploy/addons/inspektor-gadget/ig-role.yaml

@@ -0,0 +1,10 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  namespace: gadget
+  name: gadget-role
+rules:
+- apiGroups: [""]
+  resources: ["pods"]
+  # update is needed by traceloop gadget.
+  verbs: ["update"]

deploy/addons/inspektor-gadget/ig-rolebinding.yaml

@@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: gadget-role-binding
+  namespace: gadget
+subjects:
+- kind: ServiceAccount
+  name: gadget
+roleRef:
+  kind: Role
+  name: gadget-role
+  apiGroup: rbac.authorization.k8s.io

deploy/addons/inspektor-gadget/ig-serviceaccount.yaml

@@ -0,0 +1,5 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: gadget
+  namespace: gadget

pkg/addons/config.go

@@ -99,6 +99,11 @@ var Addons = []*Addon{
 		set:       SetBool,
 		callbacks: []setFn{EnableOrDisableAddon},
 	},
+	{
+		name:      "inspektor-gadget",
+		set:       SetBool,
+		callbacks: []setFn{EnableOrDisableAddon},
+	},
 	{
 		name:      "kong",
 		set:       SetBool,

pkg/minikube/assets/addons.go

@@ -277,6 +277,21 @@ var Addons = map[string]*Addon{
 			"istio-default-profile.yaml",
 			"0640"),
 	}, false, "istio", "3rd party (Istio)", "", "https://istio.io/latest/docs/setup/platform-setup/minikube/", nil, nil),
+	"inspektor-gadget": NewAddon([]*BinAsset{
+		MustBinAsset(addons.InspektorGadgetAssets, "inspektor-gadget/ig-namespace.yaml", vmpath.GuestAddonsDir, "ig-namespace.yaml", "0640"),
+		MustBinAsset(addons.InspektorGadgetAssets, "inspektor-gadget/ig-serviceaccount.yaml", vmpath.GuestAddonsDir, "ig-serviceaccount.yaml", "0640"),
+		MustBinAsset(addons.InspektorGadgetAssets, "inspektor-gadget/ig-role.yaml", vmpath.GuestAddonsDir, "ig-role.yaml", "0640"),
+		MustBinAsset(addons.InspektorGadgetAssets, "inspektor-gadget/ig-rolebinding.yaml", vmpath.GuestAddonsDir, "ig-rolebinding.yaml", "0640"),
+		MustBinAsset(addons.InspektorGadgetAssets, "inspektor-gadget/ig-clusterrole.yaml", vmpath.GuestAddonsDir, "ig-clusterrole.yaml", "0640"),
+		MustBinAsset(addons.InspektorGadgetAssets, "inspektor-gadget/ig-clusterrolebinding.yaml", vmpath.GuestAddonsDir, "ig-clusterrolebinding.yaml", "0640"),
+		MustBinAsset(addons.InspektorGadgetAssets, "inspektor-gadget/ig-crd.yaml", vmpath.GuestAddonsDir, "ig-crd.yaml", "0640"),
+		MustBinAsset(addons.InspektorGadgetAssets, "inspektor-gadget/ig-daemonset.yaml.tmpl", vmpath.GuestAddonsDir, "ig-daemonset.yaml", "0640"),
+	}, false, "inspektor-gadget", "3rd party (inspektor-gadget.io)", "https://github.com/orgs/inspektor-gadget/people", "https://minikube.sigs.k8s.io/docs/handbook/addons/inspektor-gadget/",
+		map[string]string{
+			"InspektorGadget": "inspektor-gadget/inspektor-gadget:v0.16.1@sha256:05515b802480613ae9f41e45e3b73449cf3a71fa781c697f182be0ac07319a2e",
+		}, map[string]string{
+			"InspektorGadget": "ghcr.io",
+		}),
 	"kong": NewAddon([]*BinAsset{
 		MustBinAsset(addons.KongAssets,
 			"kong/kong-ingress-controller.yaml.tmpl",

site/content/en/docs/contrib/tests.en.md

@@ -47,6 +47,9 @@ tests the GCP Auth addon with either phony or real credentials and makes sure th
 
 #### validateHeadlampAddon
 
+#### validateInspektorGadgetAddon
+tests the inspektor-gadget addon by ensuring the pod has come up and addon disables
+
 #### validateCloudSpannerAddon
 tests the cloud-spanner addon by ensuring the deployment and pod come up and addon disables
 

site/content/en/docs/handbook/addons/inspektor-gadget.md

@@ -0,0 +1,33 @@
+---
+title: "Using Inspektor Gadget Addon"
+linkTitle: "Inspektor Gadget"
+weight: 1
+date: 2023-02-16
+---
+
+## Inspektor Gadget Addon
+
+[Inspektor Gadget](https://github.com/inspektor-gadget/inspektor-gadget)  is a collection of tools (or gadgets) to debug and inspect Kubernetes resources and applications. It manages the packaging, deployment and execution of [eBPF](https://ebpf.io/) programs in a Kubernetes cluster, including many based on [BCC](https://github.com/iovisor/bcc) tools, as well as some developed specifically for use in Inspektor Gadget. It automatically maps low-level kernel primitives to high-level Kubernetes resources, making it easier and quicker to find the relevant information.
+
+### Enable Inspektor Gadget on minikube
+
+To enable this addon, simply run:
+```shell script
+minikube addons enable inspektor-gadget
+```
+
+### Testing installation
+
+```shell script
+kubectl get pods -n gadget
+```
+
+If everything went well, there should be no errors about Inspektor Gadget's installation in your minikube cluster.
+
+### Disable Inspektor Gadget
+
+To disable this addon, simply run:
+
+```shell script
+minikube addons disable inspektor-gadget
+```

test/integration/addons_test.go

@@ -78,7 +78,7 @@ func TestAddons(t *testing.T) {
 		// so we override that here to let minikube auto-detect appropriate cgroup driver
 		os.Setenv(constants.MinikubeForceSystemdEnv, "")
 
-		args := append([]string{"start", "-p", profile, "--wait=true", "--memory=4000", "--alsologtostderr", "--addons=registry", "--addons=metrics-server", "--addons=volumesnapshots", "--addons=csi-hostpath-driver", "--addons=gcp-auth", "--addons=cloud-spanner"}, StartArgs()...)
+		args := append([]string{"start", "-p", profile, "--wait=true", "--memory=4000", "--alsologtostderr", "--addons=registry", "--addons=metrics-server", "--addons=volumesnapshots", "--addons=csi-hostpath-driver", "--addons=gcp-auth", "--addons=cloud-spanner", "--addons=inspektor-gadget"}, StartArgs()...)
 		if !NoneDriver() { // none driver does not support ingress
 			args = append(args, "--addons=ingress", "--addons=ingress-dns")
 		}
@@ -104,6 +104,7 @@ func TestAddons(t *testing.T) {
 		}{
 			{"Registry", validateRegistryAddon},
 			{"Ingress", validateIngressAddon},
+			{"InspektorGadget", validateInspektorGadgetAddon},
 			{"MetricsServer", validateMetricsServerAddon},
 			{"HelmTiller", validateHelmTillerAddon},
 			{"Olm", validateOlmAddon},
@@ -796,6 +797,18 @@ func validateHeadlampAddon(ctx context.Context, t *testing.T, profile string) {
 	}
 }
 
+// validateInspektorGadgetAddon tests the inspektor-gadget addon by ensuring the pod has come up and addon disables
+func validateInspektorGadgetAddon(ctx context.Context, t *testing.T, profile string) {
+	defer PostMortemLogs(t, profile)
+
+	if _, err := PodWait(ctx, t, profile, "gadget", "k8s-app=gadget", Minutes(8)); err != nil {
+		t.Fatalf("failed waiting for inspektor-gadget pod: %v", err)
+	}
+	if rr, err := Run(t, exec.CommandContext(ctx, Target(), "addons", "disable", "inspektor-gadget", "-p", profile)); err != nil {
+		t.Errorf("failed to disable inspektor-gadget addon: args %q : %v", rr.Command(), err)
+	}
+}
+
 // validateCloudSpannerAddon tests the cloud-spanner addon by ensuring the deployment and pod come up and addon disables
 func validateCloudSpannerAddon(ctx context.Context, t *testing.T, profile string) {
 	defer PostMortemLogs(t, profile)