diff --git a/_data/tasks.yml b/_data/tasks.yml index a9e667316f..f5bdfbca6b 100644 --- a/_data/tasks.yml +++ b/_data/tasks.yml @@ -78,6 +78,7 @@ toc: - docs/tasks/debug-application-cluster/get-shell-running-container.md - docs/tasks/debug-application-cluster/monitor-node-health.md - docs/tasks/debug-application-cluster/logging-stackdriver.md + - docs/tasks/debug-application-cluster/events-stackdriver.md - docs/tasks/debug-application-cluster/logging-elasticsearch-kibana.md - docs/tasks/debug-application-cluster/determine-reason-pod-failure.md - docs/tasks/debug-application-cluster/debug-init-containers.md diff --git a/docs/tasks/debug-application-cluster/event-exporter-deploy.yaml b/docs/tasks/debug-application-cluster/event-exporter-deploy.yaml new file mode 100644 index 0000000000..e4ecc4aa0b --- /dev/null +++ b/docs/tasks/debug-application-cluster/event-exporter-deploy.yaml @@ -0,0 +1,45 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: event-exporter-sa + namespace: default + labels: + app: event-exporter +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: event-exporter-rb + namespace: default + labels: + app: event-exporter +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: view +subjects: +- kind: ServiceAccount + name: event-exporter-sa + namespace: default +--- +apiVersion: apps/v1beta1 +kind: Deployment +metadata: + name: event-exporter-v0.1.0 + namespace: default + labels: + app: event-exporter +spec: + replicas: 1 + template: + metadata: + labels: + app: event-exporter + spec: + serviceAccountName: event-exporter-sa + containers: + - name: event-exporter + image: gcr.io/google-containers/event-exporter:v0.1.0 + command: + - '/event-exporter' + terminationGracePeriodSeconds: 30 \ No newline at end of file diff --git a/docs/tasks/debug-application-cluster/events-stackdriver.md b/docs/tasks/debug-application-cluster/events-stackdriver.md new file mode 100644 index 0000000000..826f84cf21 --- /dev/null +++ b/docs/tasks/debug-application-cluster/events-stackdriver.md @@ -0,0 +1,89 @@ +--- +assignees: +- crassirostris +- piosz +title: Events in Stackdriver +--- + + + +Kubernetes events are objects that provide insight into what is happening +inside a cluster, such as what decisions were made by scheduler or why some +pods were evicted from the node. You can read more about using events +for debugging your application in the [Application Introspection and Debugging +](/docs/tasks/debug-application-cluster/debug-application-introspection/) +section. + +Since events are API objects, they are stored in the apiserver on master. To +avoid filling up master's disk, a retention policy is enforced: events are +removed one hour after the last occurrence. To provide longer history +and aggregation capabilities, a third party solution should be installed +to capture events. + +This article describes a solution that exports Kubernetes events to +Stackdriver Logging, where they can be processed and analyzed. + +**Note:** it is not guaranteed that all events happening in a cluster will be +exported to Stackdriver. One possible scenario when events will not be +exported is when event exporter is not running (e.g. during restart or +upgrade). In most cases it's fine to use events for purposes like setting up +[metrics][sdLogMetrics] and [alerts][sdAlerts], but you should be aware +of the potential inaccuracy. + +[sdLogMetrics]: https://cloud.google.com/logging/docs/view/logs_based_metrics +[sdAlerts]: https://cloud.google.com/logging/docs/view/logs_based_metrics#creating_an_alerting_policy + +* TOC +{:toc} + +## Deployment + +### Google Container Engine + +In Google Container Engine (GKE), if cloud logging is enabled, event exporter +is deployed by default to the clusters with master running version 1.7 and +higher. To prevent disturbing your workloads, event exporter does not have +resources set and is in the best effort QOS class, which means that it will +be the first to be killed in the case of resource starvation. If you want +your events to be exported, make sure you have enough resources to facilitate +the event exporter pod. This may vary depending on the workload, but on +average, approximately 100Mb RAM and 100m CPU is needed. + +### Deploying to the Existing Cluster + +Deploy event exporter to your cluster using the following command: + +```shell +kubectl create -f https://k8s.io/docs/tasks/debug-application-cluster/event-exporter-deploy.yaml +``` + +Since event exporter accesses the Kubernetes API, it requires permissions to +do so. The following deployment is configured to work with RBAC +authorization. It sets up a service account and a cluster role binding +to allow event exporter to read events. To make sure that event exporter +pod will not be evicted from the node, you can additionally set up resource +requests. As mentioned earlier, 100Mb RAM and 100m CPU should be enough. + +{% include code.html language="yaml" file="event-exporter-deploy.yaml" ghlink="/docs/tasks/debug-application-cluster/event-exporter-deploy.yaml" %} + +## User Guide + +Events are exported to the `GKE Cluster` resource in Stackdriver Logging. +You can find them by selecting an appropriate option from a drop-down menu +of available resources: + +Events location in the Stackdriver Logging interface + +You can filter based on the event object fields using Stackdriver Logging +[filtering mechanism](https://cloud.google.com/logging/docs/view/advanced_filters). +For example, the following query will show events from the scheduler +about pods from deployment `nginx-deployment`: + +``` +resource.type="gke_cluster" +jsonPayload.kind="Event" +jsonPayload.source.component="default-scheduler" +jsonPayload.involvedObject.name:"nginx-deployment" +``` + +Filtered events in the Stackdriver Logging interface diff --git a/images/docs/stackdriver-event-exporter-filter.png b/images/docs/stackdriver-event-exporter-filter.png new file mode 100644 index 0000000000..017f7aa324 Binary files /dev/null and b/images/docs/stackdriver-event-exporter-filter.png differ diff --git a/images/docs/stackdriver-event-exporter-resource.png b/images/docs/stackdriver-event-exporter-resource.png new file mode 100644 index 0000000000..d853cddecf Binary files /dev/null and b/images/docs/stackdriver-event-exporter-resource.png differ