carlosedp
/
cluster-monitoring
mirror of https://github.com/carlosedp/cluster-monitoring.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Fix namespace definition reported in #144. Rebuild manifests.

pull/161/head
Carlos de Paula 2022-02-08 12:41:51 -03:00
parent 514aa37f9a
commit 46176b4671
8 changed files with 2439 additions and 7950 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
base_operator_stack.jsonnet
View File

@ -4,7 +4,7 @@ local vars = import 'vars.jsonnet';
{ {
_config+:: { _config+:: {
namespace: 'monitoring', namespace: vars._config.namespace,
urls+:: { urls+:: {
domains: [vars.suffixDomain] + vars.additionalDomains, domains: [vars.suffixDomain] + vars.additionalDomains,

249
manifests/prometheus-rules.yaml
View File

@ -708,24 +708,19 @@ spec:
record: node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile record: node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile
- name: kube-prometheus-node-recording.rules - name: kube-prometheus-node-recording.rules
rules: rules:
- expr: sum(rate(node_cpu_seconds_total{mode!="idle",mode!="iowait"}[3m])) BY - expr: sum(rate(node_cpu_seconds_total{mode!="idle",mode!="iowait"}[3m])) BY (instance)
(instance)
record: instance:node_cpu:rate:sum record: instance:node_cpu:rate:sum
- expr: sum((node_filesystem_size_bytes{mountpoint="/"} - node_filesystem_free_bytes{mountpoint="/"})) - expr: sum((node_filesystem_size_bytes{mountpoint="/"} - node_filesystem_free_bytes{mountpoint="/"})) BY (instance)
BY (instance)
record: instance:node_filesystem_usage:sum record: instance:node_filesystem_usage:sum
- expr: sum(rate(node_network_receive_bytes_total[3m])) BY (instance) - expr: sum(rate(node_network_receive_bytes_total[3m])) BY (instance)
record: instance:node_network_receive_bytes:rate:sum record: instance:node_network_receive_bytes:rate:sum
- expr: sum(rate(node_network_transmit_bytes_total[3m])) BY (instance) - expr: sum(rate(node_network_transmit_bytes_total[3m])) BY (instance)
record: instance:node_network_transmit_bytes:rate:sum record: instance:node_network_transmit_bytes:rate:sum
- expr: sum(rate(node_cpu_seconds_total{mode!="idle",mode!="iowait"}[5m])) WITHOUT - expr: sum(rate(node_cpu_seconds_total{mode!="idle",mode!="iowait"}[5m])) WITHOUT (cpu, mode) / ON(instance) GROUP_LEFT() count(sum(node_cpu_seconds_total) BY (instance, cpu)) BY (instance)
(cpu, mode) / ON(instance) GROUP_LEFT() count(sum(node_cpu_seconds_total)
BY (instance, cpu)) BY (instance)
record: instance:node_cpu:ratio record: instance:node_cpu:ratio
- expr: sum(rate(node_cpu_seconds_total{mode!="idle",mode!="iowait"}[5m])) - expr: sum(rate(node_cpu_seconds_total{mode!="idle",mode!="iowait"}[5m]))
record: cluster:node_cpu:sum_rate5m record: cluster:node_cpu:sum_rate5m
- expr: cluster:node_cpu_seconds_total:rate5m / count(sum(node_cpu_seconds_total) - expr: cluster:node_cpu_seconds_total:rate5m / count(sum(node_cpu_seconds_total) BY (instance, cpu))
BY (instance, cpu))
record: cluster:node_cpu:ratio record: cluster:node_cpu:ratio
- name: kube-prometheus-general.rules - name: kube-prometheus-general.rules
rules: rules:
@ -737,9 +732,7 @@ spec:
rules: rules:
- alert: KubeStateMetricsListErrors - alert: KubeStateMetricsListErrors
annotations: annotations:
message: kube-state-metrics is experiencing errors at an elevated rate in message: kube-state-metrics is experiencing errors at an elevated rate in list operations. This is likely causing it to not be able to expose metrics about Kubernetes objects correctly or at all.
list operations. This is likely causing it to not be able to expose metrics
about Kubernetes objects correctly or at all.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubestatemetricslisterrors runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubestatemetricslisterrors
expr: | expr: |
(sum(rate(kube_state_metrics_list_total{job="kube-state-metrics",result="error"}[5m])) (sum(rate(kube_state_metrics_list_total{job="kube-state-metrics",result="error"}[5m]))
@ -751,9 +744,7 @@ spec:
severity: critical severity: critical
- alert: KubeStateMetricsWatchErrors - alert: KubeStateMetricsWatchErrors
annotations: annotations:
message: kube-state-metrics is experiencing errors at an elevated rate in message: kube-state-metrics is experiencing errors at an elevated rate in watch operations. This is likely causing it to not be able to expose metrics about Kubernetes objects correctly or at all.
watch operations. This is likely causing it to not be able to expose metrics
about Kubernetes objects correctly or at all.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubestatemetricswatcherrors runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubestatemetricswatcherrors
expr: | expr: |
(sum(rate(kube_state_metrics_watch_total{job="kube-state-metrics",result="error"}[5m])) (sum(rate(kube_state_metrics_watch_total{job="kube-state-metrics",result="error"}[5m]))
@ -767,9 +758,7 @@ spec:
rules: rules:
- alert: NodeFilesystemSpaceFillingUp - alert: NodeFilesystemSpaceFillingUp
annotations: annotations:
description: Filesystem on {{ $labels.device }} at {{ $labels.instance }} description: Filesystem on {{ $labels.device }} at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available space left and is filling up.
has only {{ printf "%.2f" $value }}% available space left and is filling
up.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-nodefilesystemspacefillingup runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-nodefilesystemspacefillingup
summary: Filesystem is predicted to run out of space within the next 24 hours. summary: Filesystem is predicted to run out of space within the next 24 hours.
expr: | expr: |
@ -785,9 +774,7 @@ spec:
severity: warning severity: warning
- alert: NodeFilesystemSpaceFillingUp - alert: NodeFilesystemSpaceFillingUp
annotations: annotations:
description: Filesystem on {{ $labels.device }} at {{ $labels.instance }} description: Filesystem on {{ $labels.device }} at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available space left and is filling up fast.
has only {{ printf "%.2f" $value }}% available space left and is filling
up fast.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-nodefilesystemspacefillingup runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-nodefilesystemspacefillingup
summary: Filesystem is predicted to run out of space within the next 4 hours. summary: Filesystem is predicted to run out of space within the next 4 hours.
expr: | expr: |
@ -803,8 +790,7 @@ spec:
severity: critical severity: critical
- alert: NodeFilesystemAlmostOutOfSpace - alert: NodeFilesystemAlmostOutOfSpace
annotations: annotations:
description: Filesystem on {{ $labels.device }} at {{ $labels.instance }} description: Filesystem on {{ $labels.device }} at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available space left.
has only {{ printf "%.2f" $value }}% available space left.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-nodefilesystemalmostoutofspace runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-nodefilesystemalmostoutofspace
summary: Filesystem has less than 5% space left. summary: Filesystem has less than 5% space left.
expr: | expr: |
@ -818,8 +804,7 @@ spec:
severity: warning severity: warning
- alert: NodeFilesystemAlmostOutOfSpace - alert: NodeFilesystemAlmostOutOfSpace
annotations: annotations:
description: Filesystem on {{ $labels.device }} at {{ $labels.instance }} description: Filesystem on {{ $labels.device }} at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available space left.
has only {{ printf "%.2f" $value }}% available space left.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-nodefilesystemalmostoutofspace runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-nodefilesystemalmostoutofspace
summary: Filesystem has less than 3% space left. summary: Filesystem has less than 3% space left.
expr: | expr: |
@ -833,9 +818,7 @@ spec:
severity: critical severity: critical
- alert: NodeFilesystemFilesFillingUp - alert: NodeFilesystemFilesFillingUp
annotations: annotations:
description: Filesystem on {{ $labels.device }} at {{ $labels.instance }} description: Filesystem on {{ $labels.device }} at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available inodes left and is filling up.
has only {{ printf "%.2f" $value }}% available inodes left and is filling
up.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-nodefilesystemfilesfillingup runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-nodefilesystemfilesfillingup
summary: Filesystem is predicted to run out of inodes within the next 24 hours. summary: Filesystem is predicted to run out of inodes within the next 24 hours.
expr: | expr: |
@ -851,9 +834,7 @@ spec:
severity: warning severity: warning
- alert: NodeFilesystemFilesFillingUp - alert: NodeFilesystemFilesFillingUp
annotations: annotations:
description: Filesystem on {{ $labels.device }} at {{ $labels.instance }} description: Filesystem on {{ $labels.device }} at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available inodes left and is filling up fast.
has only {{ printf "%.2f" $value }}% available inodes left and is filling
up fast.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-nodefilesystemfilesfillingup runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-nodefilesystemfilesfillingup
summary: Filesystem is predicted to run out of inodes within the next 4 hours. summary: Filesystem is predicted to run out of inodes within the next 4 hours.
expr: | expr: |
@ -869,8 +850,7 @@ spec:
severity: critical severity: critical
- alert: NodeFilesystemAlmostOutOfFiles - alert: NodeFilesystemAlmostOutOfFiles
annotations: annotations:
description: Filesystem on {{ $labels.device }} at {{ $labels.instance }} description: Filesystem on {{ $labels.device }} at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available inodes left.
has only {{ printf "%.2f" $value }}% available inodes left.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-nodefilesystemalmostoutoffiles runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-nodefilesystemalmostoutoffiles
summary: Filesystem has less than 5% inodes left. summary: Filesystem has less than 5% inodes left.
expr: | expr: |
@ -884,8 +864,7 @@ spec:
severity: warning severity: warning
- alert: NodeFilesystemAlmostOutOfFiles - alert: NodeFilesystemAlmostOutOfFiles
annotations: annotations:
description: Filesystem on {{ $labels.device }} at {{ $labels.instance }} description: Filesystem on {{ $labels.device }} at {{ $labels.instance }} has only {{ printf "%.2f" $value }}% available inodes left.
has only {{ printf "%.2f" $value }}% available inodes left.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-nodefilesystemalmostoutoffiles runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-nodefilesystemalmostoutoffiles
summary: Filesystem has less than 3% inodes left. summary: Filesystem has less than 3% inodes left.
expr: | expr: |
@ -899,8 +878,7 @@ spec:
severity: critical severity: critical
- alert: NodeNetworkReceiveErrs - alert: NodeNetworkReceiveErrs
annotations: annotations:
description: '{{ $labels.instance }} interface {{ $labels.device }} has encountered description: '{{ $labels.instance }} interface {{ $labels.device }} has encountered {{ printf "%.0f" $value }} receive errors in the last two minutes.'
{{ printf "%.0f" $value }} receive errors in the last two minutes.'
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-nodenetworkreceiveerrs runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-nodenetworkreceiveerrs
summary: Network interface is reporting many receive errors. summary: Network interface is reporting many receive errors.
expr: | expr: |
@ -910,8 +888,7 @@ spec:
severity: warning severity: warning
- alert: NodeNetworkTransmitErrs - alert: NodeNetworkTransmitErrs
annotations: annotations:
description: '{{ $labels.instance }} interface {{ $labels.device }} has encountered description: '{{ $labels.instance }} interface {{ $labels.device }} has encountered {{ printf "%.0f" $value }} transmit errors in the last two minutes.'
{{ printf "%.0f" $value }} transmit errors in the last two minutes.'
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-nodenetworktransmiterrs runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-nodenetworktransmiterrs
summary: Network interface is reporting many transmit errors. summary: Network interface is reporting many transmit errors.
expr: | expr: |
@ -939,8 +916,7 @@ spec:
severity: warning severity: warning
- alert: NodeClockSkewDetected - alert: NodeClockSkewDetected
annotations: annotations:
message: Clock on {{ $labels.instance }} is out of sync by more than 300s. message: Clock on {{ $labels.instance }} is out of sync by more than 300s. Ensure NTP is configured correctly on this host.
Ensure NTP is configured correctly on this host.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-nodeclockskewdetected runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-nodeclockskewdetected
summary: Clock skew detected. summary: Clock skew detected.
expr: | expr: |
@ -960,8 +936,7 @@ spec:
severity: warning severity: warning
- alert: NodeClockNotSynchronising - alert: NodeClockNotSynchronising
annotations: annotations:
message: Clock on {{ $labels.instance }} is not synchronising. Ensure NTP message: Clock on {{ $labels.instance }} is not synchronising. Ensure NTP is configured on this host.
is configured on this host.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-nodeclocknotsynchronising runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-nodeclocknotsynchronising
summary: Clock not synchronising. summary: Clock not synchronising.
expr: | expr: |
@ -973,8 +948,7 @@ spec:
rules: rules:
- alert: KubePodCrashLooping - alert: KubePodCrashLooping
annotations: annotations:
message: Pod {{ $labels.namespace }}/{{ $labels.pod }} ({{ $labels.container message: Pod {{ $labels.namespace }}/{{ $labels.pod }} ({{ $labels.container }}) is restarting {{ printf "%.2f" $value }} times / 5 minutes.
}}) is restarting {{ printf "%.2f" $value }} times / 5 minutes.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepodcrashlooping runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepodcrashlooping
expr: | expr: |
rate(kube_pod_container_status_restarts_total{job="kube-state-metrics"}[5m]) * 60 * 5 > 0 rate(kube_pod_container_status_restarts_total{job="kube-state-metrics"}[5m]) * 60 * 5 > 0
@ -983,8 +957,7 @@ spec:
severity: warning severity: warning
- alert: KubePodNotReady - alert: KubePodNotReady
annotations: annotations:
message: Pod {{ $labels.namespace }}/{{ $labels.pod }} has been in a non-ready message: Pod {{ $labels.namespace }}/{{ $labels.pod }} has been in a non-ready state for longer than 15 minutes.
state for longer than 15 minutes.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepodnotready runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepodnotready
expr: | expr: |
sum by (namespace, pod) ( sum by (namespace, pod) (
@ -999,9 +972,7 @@ spec:
severity: warning severity: warning
- alert: KubeDeploymentGenerationMismatch - alert: KubeDeploymentGenerationMismatch
annotations: annotations:
message: Deployment generation for {{ $labels.namespace }}/{{ $labels.deployment message: Deployment generation for {{ $labels.namespace }}/{{ $labels.deployment }} does not match, this indicates that the Deployment has failed but has not been rolled back.
}} does not match, this indicates that the Deployment has failed but has
not been rolled back.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedeploymentgenerationmismatch runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedeploymentgenerationmismatch
expr: | expr: |
kube_deployment_status_observed_generation{job="kube-state-metrics"} kube_deployment_status_observed_generation{job="kube-state-metrics"}
@ -1012,8 +983,7 @@ spec:
severity: warning severity: warning
- alert: KubeDeploymentReplicasMismatch - alert: KubeDeploymentReplicasMismatch
annotations: annotations:
message: Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has not message: Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has not matched the expected number of replicas for longer than 15 minutes.
matched the expected number of replicas for longer than 15 minutes.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedeploymentreplicasmismatch runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedeploymentreplicasmismatch
expr: | expr: |
( (
@ -1030,8 +1000,7 @@ spec:
severity: warning severity: warning
- alert: KubeStatefulSetReplicasMismatch - alert: KubeStatefulSetReplicasMismatch
annotations: annotations:
message: StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} has message: StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} has not matched the expected number of replicas for longer than 15 minutes.
not matched the expected number of replicas for longer than 15 minutes.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubestatefulsetreplicasmismatch runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubestatefulsetreplicasmismatch
expr: | expr: |
( (
@ -1048,9 +1017,7 @@ spec:
severity: warning severity: warning
- alert: KubeStatefulSetGenerationMismatch - alert: KubeStatefulSetGenerationMismatch
annotations: annotations:
message: StatefulSet generation for {{ $labels.namespace }}/{{ $labels.statefulset message: StatefulSet generation for {{ $labels.namespace }}/{{ $labels.statefulset }} does not match, this indicates that the StatefulSet has failed but has not been rolled back.
}} does not match, this indicates that the StatefulSet has failed but has
not been rolled back.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubestatefulsetgenerationmismatch runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubestatefulsetgenerationmismatch
expr: | expr: |
kube_statefulset_status_observed_generation{job="kube-state-metrics"} kube_statefulset_status_observed_generation{job="kube-state-metrics"}
@ -1061,8 +1028,7 @@ spec:
severity: warning severity: warning
- alert: KubeStatefulSetUpdateNotRolledOut - alert: KubeStatefulSetUpdateNotRolledOut
annotations: annotations:
message: StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} update message: StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} update has not been rolled out.
has not been rolled out.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubestatefulsetupdatenotrolledout runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubestatefulsetupdatenotrolledout
expr: | expr: |
( (
@ -1087,8 +1053,7 @@ spec:
severity: warning severity: warning
- alert: KubeDaemonSetRolloutStuck - alert: KubeDaemonSetRolloutStuck
annotations: annotations:
message: Only {{ $value | humanizePercentage }} of the desired Pods of DaemonSet message: Only {{ $value | humanizePercentage }} of the desired Pods of DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} are scheduled and ready.
{{ $labels.namespace }}/{{ $labels.daemonset }} are scheduled and ready.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedaemonsetrolloutstuck runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedaemonsetrolloutstuck
expr: | expr: |
kube_daemonset_status_number_ready{job="kube-state-metrics"} kube_daemonset_status_number_ready{job="kube-state-metrics"}
@ -1099,8 +1064,7 @@ spec:
severity: warning severity: warning
- alert: KubeContainerWaiting - alert: KubeContainerWaiting
annotations: annotations:
message: Pod {{ $labels.namespace }}/{{ $labels.pod }} container {{ $labels.container}} message: Pod {{ $labels.namespace }}/{{ $labels.pod }} container {{ $labels.container}} has been in waiting state for longer than 1 hour.
has been in waiting state for longer than 1 hour.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecontainerwaiting runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecontainerwaiting
expr: | expr: |
sum by (namespace, pod, container) (kube_pod_container_status_waiting_reason{job="kube-state-metrics"}) > 0 sum by (namespace, pod, container) (kube_pod_container_status_waiting_reason{job="kube-state-metrics"}) > 0
@ -1109,8 +1073,7 @@ spec:
severity: warning severity: warning
- alert: KubeDaemonSetNotScheduled - alert: KubeDaemonSetNotScheduled
annotations: annotations:
message: '{{ $value }} Pods of DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset message: '{{ $value }} Pods of DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} are not scheduled.'
}} are not scheduled.'
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedaemonsetnotscheduled runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedaemonsetnotscheduled
expr: | expr: |
kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics"} kube_daemonset_status_desired_number_scheduled{job="kube-state-metrics"}
@ -1121,8 +1084,7 @@ spec:
severity: warning severity: warning
- alert: KubeDaemonSetMisScheduled - alert: KubeDaemonSetMisScheduled
annotations: annotations:
message: '{{ $value }} Pods of DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset message: '{{ $value }} Pods of DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} are running where they are not supposed to run.'
}} are running where they are not supposed to run.'
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedaemonsetmisscheduled runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedaemonsetmisscheduled
expr: | expr: |
kube_daemonset_status_number_misscheduled{job="kube-state-metrics"} > 0 kube_daemonset_status_number_misscheduled{job="kube-state-metrics"} > 0
@ -1131,8 +1093,7 @@ spec:
severity: warning severity: warning
- alert: KubeCronJobRunning - alert: KubeCronJobRunning
annotations: annotations:
message: CronJob {{ $labels.namespace }}/{{ $labels.cronjob }} is taking more message: CronJob {{ $labels.namespace }}/{{ $labels.cronjob }} is taking more than 1h to complete.
than 1h to complete.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecronjobrunning runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecronjobrunning
expr: | expr: |
time() - kube_cronjob_next_schedule_time{job="kube-state-metrics"} > 3600 time() - kube_cronjob_next_schedule_time{job="kube-state-metrics"} > 3600
@ -1141,8 +1102,7 @@ spec:
severity: warning severity: warning
- alert: KubeJobCompletion - alert: KubeJobCompletion
annotations: annotations:
message: Job {{ $labels.namespace }}/{{ $labels.job_name }} is taking more message: Job {{ $labels.namespace }}/{{ $labels.job_name }} is taking more than one hour to complete.
than one hour to complete.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubejobcompletion runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubejobcompletion
expr: | expr: |
kube_job_spec_completions{job="kube-state-metrics"} - kube_job_status_succeeded{job="kube-state-metrics"} > 0 kube_job_spec_completions{job="kube-state-metrics"} - kube_job_status_succeeded{job="kube-state-metrics"} > 0
@ -1160,8 +1120,7 @@ spec:
severity: warning severity: warning
- alert: KubeHpaReplicasMismatch - alert: KubeHpaReplicasMismatch
annotations: annotations:
message: HPA {{ $labels.namespace }}/{{ $labels.hpa }} has not matched the message: HPA {{ $labels.namespace }}/{{ $labels.hpa }} has not matched the desired number of replicas for longer than 15 minutes.
desired number of replicas for longer than 15 minutes.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubehpareplicasmismatch runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubehpareplicasmismatch
expr: | expr: |
(kube_hpa_status_desired_replicas{job="kube-state-metrics"} (kube_hpa_status_desired_replicas{job="kube-state-metrics"}
@ -1174,8 +1133,7 @@ spec:
severity: warning severity: warning
- alert: KubeHpaMaxedOut - alert: KubeHpaMaxedOut
annotations: annotations:
message: HPA {{ $labels.namespace }}/{{ $labels.hpa }} has been running at message: HPA {{ $labels.namespace }}/{{ $labels.hpa }} has been running at max replicas for longer than 15 minutes.
max replicas for longer than 15 minutes.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubehpamaxedout runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubehpamaxedout
expr: | expr: |
kube_hpa_status_current_replicas{job="kube-state-metrics"} kube_hpa_status_current_replicas{job="kube-state-metrics"}
@ -1188,8 +1146,7 @@ spec:
rules: rules:
- alert: KubeCPUOvercommit - alert: KubeCPUOvercommit
annotations: annotations:
message: Cluster has overcommitted CPU resource requests for Pods and cannot message: Cluster has overcommitted CPU resource requests for Pods and cannot tolerate node failure.
tolerate node failure.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecpuovercommit runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecpuovercommit
expr: | expr: |
sum(namespace:kube_pod_container_resource_requests_cpu_cores:sum{}) sum(namespace:kube_pod_container_resource_requests_cpu_cores:sum{})
@ -1202,8 +1159,7 @@ spec:
severity: warning severity: warning
- alert: KubeMemoryOvercommit - alert: KubeMemoryOvercommit
annotations: annotations:
message: Cluster has overcommitted memory resource requests for Pods and cannot message: Cluster has overcommitted memory resource requests for Pods and cannot tolerate node failure.
tolerate node failure.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubememoryovercommit runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubememoryovercommit
expr: | expr: |
sum(namespace:kube_pod_container_resource_requests_memory_bytes:sum{}) sum(namespace:kube_pod_container_resource_requests_memory_bytes:sum{})
@ -1242,8 +1198,7 @@ spec:
severity: warning severity: warning
- alert: KubeQuotaExceeded - alert: KubeQuotaExceeded
annotations: annotations:
message: Namespace {{ $labels.namespace }} is using {{ $value | humanizePercentage message: Namespace {{ $labels.namespace }} is using {{ $value | humanizePercentage }} of its {{ $labels.resource }} quota.
}} of its {{ $labels.resource }} quota.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubequotaexceeded runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubequotaexceeded
expr: | expr: |
kube_resourcequota{job="kube-state-metrics", type="used"} kube_resourcequota{job="kube-state-metrics", type="used"}
@ -1255,9 +1210,7 @@ spec:
severity: warning severity: warning
- alert: CPUThrottlingHigh - alert: CPUThrottlingHigh
annotations: annotations:
message: '{{ $value | humanizePercentage }} throttling of CPU in namespace message: '{{ $value | humanizePercentage }} throttling of CPU in namespace {{ $labels.namespace }} for container {{ $labels.container }} in pod {{ $labels.pod }}.'
{{ $labels.namespace }} for container {{ $labels.container }} in pod {{
$labels.pod }}.'
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-cputhrottlinghigh runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-cputhrottlinghigh
expr: | expr: |
sum(increase(container_cpu_cfs_throttled_periods_total{container!="", }[5m])) by (container, pod, namespace) sum(increase(container_cpu_cfs_throttled_periods_total{container!="", }[5m])) by (container, pod, namespace)
@ -1271,9 +1224,7 @@ spec:
rules: rules:
- alert: KubePersistentVolumeFillingUp - alert: KubePersistentVolumeFillingUp
annotations: annotations:
message: The PersistentVolume claimed by {{ $labels.persistentvolumeclaim message: The PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} is only {{ $value | humanizePercentage }} free.
}} in Namespace {{ $labels.namespace }} is only {{ $value | humanizePercentage
}} free.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepersistentvolumefillingup runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepersistentvolumefillingup
expr: | expr: |
kubelet_volume_stats_available_bytes{job="kubelet", metrics_path="/metrics"} kubelet_volume_stats_available_bytes{job="kubelet", metrics_path="/metrics"}
@ -1285,9 +1236,7 @@ spec:
severity: critical severity: critical
- alert: KubePersistentVolumeFillingUp - alert: KubePersistentVolumeFillingUp
annotations: annotations:
message: Based on recent sampling, the PersistentVolume claimed by {{ $labels.persistentvolumeclaim message: Based on recent sampling, the PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} is expected to fill up within four days. Currently {{ $value | humanizePercentage }} is available.
}} in Namespace {{ $labels.namespace }} is expected to fill up within four
days. Currently {{ $value | humanizePercentage }} is available.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepersistentvolumefillingup runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepersistentvolumefillingup
expr: | expr: |
( (
@ -1302,8 +1251,7 @@ spec:
severity: warning severity: warning
- alert: KubePersistentVolumeErrors - alert: KubePersistentVolumeErrors
annotations: annotations:
message: The persistent volume {{ $labels.persistentvolume }} has status {{ message: The persistent volume {{ $labels.persistentvolume }} has status {{ $labels.phase }}.
$labels.phase }}.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepersistentvolumeerrors runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepersistentvolumeerrors
expr: | expr: |
kube_persistentvolume_status_phase{phase=~"Failed|Pending",job="kube-state-metrics"} > 0 kube_persistentvolume_status_phase{phase=~"Failed|Pending",job="kube-state-metrics"} > 0
@ -1314,8 +1262,7 @@ spec:
rules: rules:
- alert: KubeVersionMismatch - alert: KubeVersionMismatch
annotations: annotations:
message: There are {{ $value }} different semantic versions of Kubernetes message: There are {{ $value }} different semantic versions of Kubernetes components running.
components running.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeversionmismatch runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeversionmismatch
expr: | expr: |
count(count by (gitVersion) (label_replace(kubernetes_build_info{job!~"kube-dns|coredns"},"gitVersion","$1","gitVersion","(v[0-9]*.[0-9]*.[0-9]*).*"))) > 1 count(count by (gitVersion) (label_replace(kubernetes_build_info{job!~"kube-dns|coredns"},"gitVersion","$1","gitVersion","(v[0-9]*.[0-9]*.[0-9]*).*"))) > 1
@ -1324,8 +1271,7 @@ spec:
severity: warning severity: warning
- alert: KubeClientErrors - alert: KubeClientErrors
annotations: annotations:
message: Kubernetes API server client '{{ $labels.job }}/{{ $labels.instance message: Kubernetes API server client '{{ $labels.job }}/{{ $labels.instance }}' is experiencing {{ $value | humanizePercentage }} errors.'
}}' is experiencing {{ $value | humanizePercentage }} errors.'
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeclienterrors runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeclienterrors
expr: | expr: |
(sum(rate(rest_client_requests_total{code=~"5.."}[5m])) by (instance, job) (sum(rate(rest_client_requests_total{code=~"5.."}[5m])) by (instance, job)
@ -1393,8 +1339,7 @@ spec:
rules: rules:
- alert: KubeAPILatencyHigh - alert: KubeAPILatencyHigh
annotations: annotations:
message: The API server has an abnormal latency of {{ $value }} seconds for message: The API server has an abnormal latency of {{ $value }} seconds for {{ $labels.verb }} {{ $labels.resource }}.
{{ $labels.verb }} {{ $labels.resource }}.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapilatencyhigh runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapilatencyhigh
expr: | expr: |
cluster_quantile:apiserver_request_duration_seconds:histogram_quantile{job="apiserver",quantile="0.99"} cluster_quantile:apiserver_request_duration_seconds:histogram_quantile{job="apiserver",quantile="0.99"}
@ -1417,9 +1362,7 @@ spec:
severity: warning severity: warning
- alert: KubeAPIErrorsHigh - alert: KubeAPIErrorsHigh
annotations: annotations:
message: API server is returning errors for {{ $value | humanizePercentage message: API server is returning errors for {{ $value | humanizePercentage }} of requests for {{ $labels.verb }} {{ $labels.resource }} {{ $labels.subresource }}.
}} of requests for {{ $labels.verb }} {{ $labels.resource }} {{ $labels.subresource
}}.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapierrorshigh runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeapierrorshigh
expr: | expr: |
sum(rate(apiserver_request_total{job="apiserver",code=~"5.."}[5m])) by (resource,subresource,verb) sum(rate(apiserver_request_total{job="apiserver",code=~"5.."}[5m])) by (resource,subresource,verb)
@ -1430,8 +1373,7 @@ spec:
severity: warning severity: warning
- alert: KubeClientCertificateExpiration - alert: KubeClientCertificateExpiration
annotations: annotations:
message: A client certificate used to authenticate to the apiserver is expiring message: A client certificate used to authenticate to the apiserver is expiring in less than 7.0 days.
in less than 7.0 days.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeclientcertificateexpiration runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeclientcertificateexpiration
expr: | expr: |
apiserver_client_certificate_expiration_seconds_count{job="apiserver"} > 0 and on(job) histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m]))) < 604800 apiserver_client_certificate_expiration_seconds_count{job="apiserver"} > 0 and on(job) histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m]))) < 604800
@ -1439,8 +1381,7 @@ spec:
severity: warning severity: warning
- alert: KubeClientCertificateExpiration - alert: KubeClientCertificateExpiration
annotations: annotations:
message: A client certificate used to authenticate to the apiserver is expiring message: A client certificate used to authenticate to the apiserver is expiring in less than 24.0 hours.
in less than 24.0 hours.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeclientcertificateexpiration runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeclientcertificateexpiration
expr: | expr: |
apiserver_client_certificate_expiration_seconds_count{job="apiserver"} > 0 and on(job) histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m]))) < 86400 apiserver_client_certificate_expiration_seconds_count{job="apiserver"} > 0 and on(job) histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m]))) < 86400
@ -1448,10 +1389,7 @@ spec:
severity: critical severity: critical
- alert: AggregatedAPIErrors - alert: AggregatedAPIErrors
annotations: annotations:
message: An aggregated API {{ $labels.name }}/{{ $labels.namespace }} has message: An aggregated API {{ $labels.name }}/{{ $labels.namespace }} has reported errors. The number of errors have increased for it in the past five minutes. High values indicate that the availability of the service changes too often.
reported errors. The number of errors have increased for it in the past
five minutes. High values indicate that the availability of the service
changes too often.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-aggregatedapierrors runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-aggregatedapierrors
expr: | expr: |
sum by(name, namespace)(increase(aggregator_unavailable_apiservice_count[5m])) > 2 sum by(name, namespace)(increase(aggregator_unavailable_apiservice_count[5m])) > 2
@ -1459,8 +1397,7 @@ spec:
severity: warning severity: warning
- alert: AggregatedAPIDown - alert: AggregatedAPIDown
annotations: annotations:
message: An aggregated API {{ $labels.name }}/{{ $labels.namespace }} is down. message: An aggregated API {{ $labels.name }}/{{ $labels.namespace }} is down. It has not been available at least for the past five minutes.
It has not been available at least for the past five minutes.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-aggregatedapidown runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-aggregatedapidown
expr: | expr: |
sum by(name, namespace)(sum_over_time(aggregator_unavailable_apiservice[5m])) > 0 sum by(name, namespace)(sum_over_time(aggregator_unavailable_apiservice[5m])) > 0
@ -1497,8 +1434,7 @@ spec:
severity: warning severity: warning
- alert: KubeletTooManyPods - alert: KubeletTooManyPods
annotations: annotations:
message: Kubelet '{{ $labels.node }}' is running at {{ $value | humanizePercentage message: Kubelet '{{ $labels.node }}' is running at {{ $value | humanizePercentage }} of its Pod capacity.
}} of its Pod capacity.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubelettoomanypods runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubelettoomanypods
expr: | expr: |
max(max(kubelet_running_pod_count{job="kubelet", metrics_path="/metrics"}) by(instance) * on(instance) group_left(node) kubelet_node_name{job="kubelet", metrics_path="/metrics"}) by(node) / max(kube_node_status_capacity_pods{job="kube-state-metrics"} != 1) by(node) > 0.95 max(max(kubelet_running_pod_count{job="kubelet", metrics_path="/metrics"}) by(instance) * on(instance) group_left(node) kubelet_node_name{job="kubelet", metrics_path="/metrics"}) by(node) / max(kube_node_status_capacity_pods{job="kube-state-metrics"} != 1) by(node) > 0.95
@ -1507,8 +1443,7 @@ spec:
severity: warning severity: warning
- alert: KubeNodeReadinessFlapping - alert: KubeNodeReadinessFlapping
annotations: annotations:
message: The readiness status of node {{ $labels.node }} has changed {{ $value message: The readiness status of node {{ $labels.node }} has changed {{ $value }} times in the last 15 minutes.
}} times in the last 15 minutes.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubenodereadinessflapping runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubenodereadinessflapping
expr: | expr: |
sum(changes(kube_node_status_condition{status="true",condition="Ready"}[15m])) by (node) > 2 sum(changes(kube_node_status_condition{status="true",condition="Ready"}[15m])) by (node) > 2
@ -1517,8 +1452,7 @@ spec:
severity: warning severity: warning
- alert: KubeletPlegDurationHigh - alert: KubeletPlegDurationHigh
annotations: annotations:
message: The Kubelet Pod Lifecycle Event Generator has a 99th percentile duration message: The Kubelet Pod Lifecycle Event Generator has a 99th percentile duration of {{ $value }} seconds on node {{ $labels.node }}.
of {{ $value }} seconds on node {{ $labels.node }}.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeletplegdurationhigh runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeletplegdurationhigh
expr: | expr: |
node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile{quantile="0.99"} >= 10 node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile{quantile="0.99"} >= 10
@ -1527,8 +1461,7 @@ spec:
severity: warning severity: warning
- alert: KubeletPodStartUpLatencyHigh - alert: KubeletPodStartUpLatencyHigh
annotations: annotations:
message: Kubelet Pod startup 99th percentile latency is {{ $value }} seconds message: Kubelet Pod startup 99th percentile latency is {{ $value }} seconds on node {{ $labels.node }}.
on node {{ $labels.node }}.
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeletpodstartuplatencyhigh runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubeletpodstartuplatencyhigh
expr: | expr: |
histogram_quantile(0.99, sum(rate(kubelet_pod_worker_duration_seconds_bucket{job="kubelet", metrics_path="/metrics"}[5m])) by (instance, le)) * on(instance) group_left(node) kubelet_node_name{job="kubelet", metrics_path="/metrics"} > 60 histogram_quantile(0.99, sum(rate(kubelet_pod_worker_duration_seconds_bucket{job="kubelet", metrics_path="/metrics"}[5m])) by (instance, le)) * on(instance) group_left(node) kubelet_node_name{job="kubelet", metrics_path="/metrics"} > 60
@ -1570,8 +1503,7 @@ spec:
rules: rules:
- alert: PrometheusBadConfig - alert: PrometheusBadConfig
annotations: annotations:
description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has failed to description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has failed to reload its configuration.
reload its configuration.
summary: Failed Prometheus configuration reload. summary: Failed Prometheus configuration reload.
expr: | expr: |
# Without max_over_time, failed scrapes could create false negatives, see # Without max_over_time, failed scrapes could create false negatives, see
@ -1582,10 +1514,8 @@ spec:
severity: critical severity: critical
- alert: PrometheusNotificationQueueRunningFull - alert: PrometheusNotificationQueueRunningFull
annotations: annotations:
description: Alert notification queue of Prometheus {{$labels.namespace}}/{{$labels.pod}} description: Alert notification queue of Prometheus {{$labels.namespace}}/{{$labels.pod}} is running full.
is running full. summary: Prometheus alert notification queue predicted to run full in less than 30m.
summary: Prometheus alert notification queue predicted to run full in less
than 30m.
expr: | expr: |
# Without min_over_time, failed scrapes could create false negatives, see # Without min_over_time, failed scrapes could create false negatives, see
# https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details.
@ -1599,10 +1529,8 @@ spec:
severity: warning severity: warning
- alert: PrometheusErrorSendingAlertsToSomeAlertmanagers - alert: PrometheusErrorSendingAlertsToSomeAlertmanagers
annotations: annotations:
description: '{{ printf "%.1f" $value }}% errors while sending alerts from description: '{{ printf "%.1f" $value }}% errors while sending alerts from Prometheus {{$labels.namespace}}/{{$labels.pod}} to Alertmanager {{$labels.alertmanager}}.'
Prometheus {{$labels.namespace}}/{{$labels.pod}} to Alertmanager {{$labels.alertmanager}}.' summary: Prometheus has encountered more than 1% errors sending alerts to a specific Alertmanager.
summary: Prometheus has encountered more than 1% errors sending alerts to
a specific Alertmanager.
expr: | expr: |
( (
rate(prometheus_notifications_errors_total{job="prometheus-k8s",namespace="monitoring"}[5m]) rate(prometheus_notifications_errors_total{job="prometheus-k8s",namespace="monitoring"}[5m])
@ -1616,8 +1544,7 @@ spec:
severity: warning severity: warning
- alert: PrometheusErrorSendingAlertsToAnyAlertmanager - alert: PrometheusErrorSendingAlertsToAnyAlertmanager
annotations: annotations:
description: '{{ printf "%.1f" $value }}% minimum errors while sending alerts description: '{{ printf "%.1f" $value }}% minimum errors while sending alerts from Prometheus {{$labels.namespace}}/{{$labels.pod}} to any Alertmanager.'
from Prometheus {{$labels.namespace}}/{{$labels.pod}} to any Alertmanager.'
summary: Prometheus encounters more than 3% errors sending alerts to any Alertmanager. summary: Prometheus encounters more than 3% errors sending alerts to any Alertmanager.
expr: | expr: |
min without(alertmanager) ( min without(alertmanager) (
@ -1632,8 +1559,7 @@ spec:
severity: critical severity: critical
- alert: PrometheusNotConnectedToAlertmanagers - alert: PrometheusNotConnectedToAlertmanagers
annotations: annotations:
description: Prometheus {{$labels.namespace}}/{{$labels.pod}} is not connected description: Prometheus {{$labels.namespace}}/{{$labels.pod}} is not connected to any Alertmanagers.
to any Alertmanagers.
summary: Prometheus is not connected to any Alertmanagers. summary: Prometheus is not connected to any Alertmanagers.
expr: | expr: |
# Without max_over_time, failed scrapes could create false negatives, see # Without max_over_time, failed scrapes could create false negatives, see
@ -1644,8 +1570,7 @@ spec:
severity: warning severity: warning
- alert: PrometheusTSDBReloadsFailing - alert: PrometheusTSDBReloadsFailing
annotations: annotations:
description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has detected description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has detected {{$value | humanize}} reload failures over the last 3h.
{{$value | humanize}} reload failures over the last 3h.
summary: Prometheus has issues reloading blocks from disk. summary: Prometheus has issues reloading blocks from disk.
expr: | expr: |
increase(prometheus_tsdb_reloads_failures_total{job="prometheus-k8s",namespace="monitoring"}[3h]) > 0 increase(prometheus_tsdb_reloads_failures_total{job="prometheus-k8s",namespace="monitoring"}[3h]) > 0
@ -1654,8 +1579,7 @@ spec:
severity: warning severity: warning
- alert: PrometheusTSDBCompactionsFailing - alert: PrometheusTSDBCompactionsFailing
annotations: annotations:
description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has detected description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has detected {{$value | humanize}} compaction failures over the last 3h.
{{$value | humanize}} compaction failures over the last 3h.
summary: Prometheus has issues compacting blocks. summary: Prometheus has issues compacting blocks.
expr: | expr: |
increase(prometheus_tsdb_compactions_failed_total{job="prometheus-k8s",namespace="monitoring"}[3h]) > 0 increase(prometheus_tsdb_compactions_failed_total{job="prometheus-k8s",namespace="monitoring"}[3h]) > 0
@ -1664,8 +1588,7 @@ spec:
severity: warning severity: warning
- alert: PrometheusNotIngestingSamples - alert: PrometheusNotIngestingSamples
annotations: annotations:
description: Prometheus {{$labels.namespace}}/{{$labels.pod}} is not ingesting description: Prometheus {{$labels.namespace}}/{{$labels.pod}} is not ingesting samples.
samples.
summary: Prometheus is not ingesting samples. summary: Prometheus is not ingesting samples.
expr: | expr: |
rate(prometheus_tsdb_head_samples_appended_total{job="prometheus-k8s",namespace="monitoring"}[5m]) <= 0 rate(prometheus_tsdb_head_samples_appended_total{job="prometheus-k8s",namespace="monitoring"}[5m]) <= 0
@ -1674,9 +1597,7 @@ spec:
severity: warning severity: warning
- alert: PrometheusDuplicateTimestamps - alert: PrometheusDuplicateTimestamps
annotations: annotations:
description: Prometheus {{$labels.namespace}}/{{$labels.pod}} is dropping description: Prometheus {{$labels.namespace}}/{{$labels.pod}} is dropping {{ printf "%.4g" $value }} samples/s with different values but duplicated timestamp.
{{ printf "%.4g" $value }} samples/s with different values but duplicated
timestamp.
summary: Prometheus is dropping samples with duplicate timestamps. summary: Prometheus is dropping samples with duplicate timestamps.
expr: | expr: |
rate(prometheus_target_scrapes_sample_duplicate_timestamp_total{job="prometheus-k8s",namespace="monitoring"}[5m]) > 0 rate(prometheus_target_scrapes_sample_duplicate_timestamp_total{job="prometheus-k8s",namespace="monitoring"}[5m]) > 0
@ -1685,8 +1606,7 @@ spec:
severity: warning severity: warning
- alert: PrometheusOutOfOrderTimestamps - alert: PrometheusOutOfOrderTimestamps
annotations: annotations:
description: Prometheus {{$labels.namespace}}/{{$labels.pod}} is dropping description: Prometheus {{$labels.namespace}}/{{$labels.pod}} is dropping {{ printf "%.4g" $value }} samples/s with timestamps arriving out of order.
{{ printf "%.4g" $value }} samples/s with timestamps arriving out of order.
summary: Prometheus drops samples with out-of-order timestamps. summary: Prometheus drops samples with out-of-order timestamps.
expr: | expr: |
rate(prometheus_target_scrapes_sample_out_of_order_total{job="prometheus-k8s",namespace="monitoring"}[5m]) > 0 rate(prometheus_target_scrapes_sample_out_of_order_total{job="prometheus-k8s",namespace="monitoring"}[5m]) > 0
@ -1695,9 +1615,7 @@ spec:
severity: warning severity: warning
- alert: PrometheusRemoteStorageFailures - alert: PrometheusRemoteStorageFailures
annotations: annotations:
description: Prometheus {{$labels.namespace}}/{{$labels.pod}} failed to send description: Prometheus {{$labels.namespace}}/{{$labels.pod}} failed to send {{ printf "%.1f" $value }}% of the samples to {{ $labels.remote_name}}:{{ $labels.url }}
{{ printf "%.1f" $value }}% of the samples to {{ $labels.remote_name}}:{{
$labels.url }}
summary: Prometheus fails to send samples to remote storage. summary: Prometheus fails to send samples to remote storage.
expr: | expr: |
( (
@ -1716,9 +1634,7 @@ spec:
severity: critical severity: critical
- alert: PrometheusRemoteWriteBehind - alert: PrometheusRemoteWriteBehind
annotations: annotations:
description: Prometheus {{$labels.namespace}}/{{$labels.pod}} remote write description: Prometheus {{$labels.namespace}}/{{$labels.pod}} remote write is {{ printf "%.1f" $value }}s behind for {{ $labels.remote_name}}:{{ $labels.url }}.
is {{ printf "%.1f" $value }}s behind for {{ $labels.remote_name}}:{{ $labels.url
}}.
summary: Prometheus remote write is behind. summary: Prometheus remote write is behind.
expr: | expr: |
# Without max_over_time, failed scrapes could create false negatives, see # Without max_over_time, failed scrapes could create false negatives, see
@ -1734,13 +1650,8 @@ spec:
severity: critical severity: critical
- alert: PrometheusRemoteWriteDesiredShards - alert: PrometheusRemoteWriteDesiredShards
annotations: annotations:
description: Prometheus {{$labels.namespace}}/{{$labels.pod}} remote write description: Prometheus {{$labels.namespace}}/{{$labels.pod}} remote write desired shards calculation wants to run {{ $value }} shards for queue {{ $labels.remote_name}}:{{ $labels.url }}, which is more than the max of {{ printf `prometheus_remote_storage_shards_max{instance="%s",job="prometheus-k8s",namespace="monitoring"}` $labels.instance | query | first | value }}.
desired shards calculation wants to run {{ $value }} shards for queue {{ summary: Prometheus remote write desired shards calculation wants to run more than configured max shards.
$labels.remote_name}}:{{ $labels.url }}, which is more than the max of {{
printf `prometheus_remote_storage_shards_max{instance="%s",job="prometheus-k8s",namespace="monitoring"}`
$labels.instance | query | first | value }}.
summary: Prometheus remote write desired shards calculation wants to run more
than configured max shards.
expr: | expr: |
# Without max_over_time, failed scrapes could create false negatives, see # Without max_over_time, failed scrapes could create false negatives, see
# https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details. # https://www.robustperception.io/alerting-on-gauges-in-prometheus-2-0 for details.
@ -1754,8 +1665,7 @@ spec:
severity: warning severity: warning
- alert: PrometheusRuleFailures - alert: PrometheusRuleFailures
annotations: annotations:
description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has failed to description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has failed to evaluate {{ printf "%.0f" $value }} rules in the last 5m.
evaluate {{ printf "%.0f" $value }} rules in the last 5m.
summary: Prometheus is failing rule evaluations. summary: Prometheus is failing rule evaluations.
expr: | expr: |
increase(prometheus_rule_evaluation_failures_total{job="prometheus-k8s",namespace="monitoring"}[5m]) > 0 increase(prometheus_rule_evaluation_failures_total{job="prometheus-k8s",namespace="monitoring"}[5m]) > 0
@ -1764,8 +1674,7 @@ spec:
severity: critical severity: critical
- alert: PrometheusMissingRuleEvaluations - alert: PrometheusMissingRuleEvaluations
annotations: annotations:
description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has missed {{ description: Prometheus {{$labels.namespace}}/{{$labels.pod}} has missed {{ printf "%.0f" $value }} rule group evaluations in the last 5m.
printf "%.0f" $value }} rule group evaluations in the last 5m.
summary: Prometheus is missing rule evaluations due to slow rule group evaluation. summary: Prometheus is missing rule evaluations due to slow rule group evaluation.
expr: | expr: |
increase(prometheus_rule_group_iterations_missed_total{job="prometheus-k8s",namespace="monitoring"}[5m]) > 0 increase(prometheus_rule_group_iterations_missed_total{job="prometheus-k8s",namespace="monitoring"}[5m]) > 0
@ -1776,8 +1685,7 @@ spec:
rules: rules:
- alert: AlertmanagerConfigInconsistent - alert: AlertmanagerConfigInconsistent
annotations: annotations:
message: The configuration of the instances of the Alertmanager cluster `{{$labels.service}}` message: The configuration of the instances of the Alertmanager cluster `{{$labels.service}}` are out of sync.
are out of sync.
expr: | expr: |
count_values("config_hash", alertmanager_config_hash{job="alertmanager-main",namespace="monitoring"}) BY (service) / ON(service) GROUP_LEFT() label_replace(max(prometheus_operator_spec_replicas{job="prometheus-operator",namespace="monitoring",controller="alertmanager"}) by (name, job, namespace, controller), "service", "alertmanager-$1", "name", "(.*)") != 1 count_values("config_hash", alertmanager_config_hash{job="alertmanager-main",namespace="monitoring"}) BY (service) / ON(service) GROUP_LEFT() label_replace(max(prometheus_operator_spec_replicas{job="prometheus-operator",namespace="monitoring",controller="alertmanager"}) by (name, job, namespace, controller), "service", "alertmanager-$1", "name", "(.*)") != 1
for: 5m for: 5m
@ -1785,8 +1693,7 @@ spec:
severity: critical severity: critical
- alert: AlertmanagerFailedReload - alert: AlertmanagerFailedReload
annotations: annotations:
message: Reloading Alertmanager's configuration has failed for {{ $labels.namespace message: Reloading Alertmanager's configuration has failed for {{ $labels.namespace }}/{{ $labels.pod}}.
}}/{{ $labels.pod}}.
expr: | expr: |
alertmanager_config_last_reload_successful{job="alertmanager-main",namespace="monitoring"} == 0 alertmanager_config_last_reload_successful{job="alertmanager-main",namespace="monitoring"} == 0
for: 10m for: 10m
@ -1806,10 +1713,8 @@ spec:
rules: rules:
- alert: TargetDown - alert: TargetDown
annotations: annotations:
message: '{{ printf "%.4g" $value }}% of the {{ $labels.job }}/{{ $labels.service message: '{{ printf "%.4g" $value }}% of the {{ $labels.job }}/{{ $labels.service }} targets in {{ $labels.namespace }} namespace are down.'
}} targets in {{ $labels.namespace }} namespace are down.' expr: 100 * (count(up == 0) BY (job, namespace, service) / count(up) BY (job, namespace, service)) > 10
expr: 100 * (count(up == 0) BY (job, namespace, service) / count(up) BY (job,
namespace, service)) > 10
for: 10m for: 10m
labels: labels:
severity: warning severity: warning
@ -1828,8 +1733,7 @@ spec:
rules: rules:
- alert: NodeNetworkInterfaceFlapping - alert: NodeNetworkInterfaceFlapping
annotations: annotations:
message: Network interface "{{ $labels.device }}" changing it's up status message: Network interface "{{ $labels.device }}" changing it's up status often on node-exporter {{ $labels.namespace }}/{{ $labels.pod }}"
often on node-exporter {{ $labels.namespace }}/{{ $labels.pod }}"
expr: | expr: |
changes(node_network_up{job="node-exporter",device!~"veth.+"}[2m]) > 2 changes(node_network_up{job="node-exporter",device!~"veth.+"}[2m]) > 2
for: 2m for: 2m
@ -1839,8 +1743,7 @@ spec:
rules: rules:
- alert: PrometheusOperatorReconcileErrors - alert: PrometheusOperatorReconcileErrors
annotations: annotations:
message: Errors while reconciling {{ $labels.controller }} in {{ $labels.namespace message: Errors while reconciling {{ $labels.controller }} in {{ $labels.namespace }} Namespace.
}} Namespace.
expr: | expr: |
rate(prometheus_operator_reconcile_errors_total{job="prometheus-operator",namespace="monitoring"}[5m]) > 0.1 rate(prometheus_operator_reconcile_errors_total{job="prometheus-operator",namespace="monitoring"}[5m]) > 0.1
for: 10m for: 10m

2910
manifests/setup/prometheus-operator-0alertmanagerCustomResourceDefinition.yaml
View File

File diff suppressed because it is too large Load Diff

130
manifests/setup/prometheus-operator-0podmonitorCustomResourceDefinition.yaml
View File

@ -20,31 +20,24 @@ spec:
description: PodMonitor defines monitoring for a set of pods. description: PodMonitor defines monitoring for a set of pods.
properties: properties:
apiVersion: apiVersion:
description: 'APIVersion defines the versioned schema of this representation description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
of an object. Servers should convert recognized schemas to the latest
internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
type: string type: string
kind: kind:
description: 'Kind is a string value representing the REST resource this description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
object represents. Servers may infer this from the endpoint the client
submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
type: string type: string
metadata: metadata:
type: object type: object
spec: spec:
description: Specification of desired Pod selection for target discovery description: Specification of desired Pod selection for target discovery by Prometheus.
by Prometheus.
properties: properties:
jobLabel: jobLabel:
description: The label to use to retrieve the job name from. description: The label to use to retrieve the job name from.
type: string type: string
namespaceSelector: namespaceSelector:
description: Selector to select which namespaces the Endpoints objects description: Selector to select which namespaces the Endpoints objects are discovered from.
are discovered from.
properties: properties:
any: any:
description: Boolean describing whether all namespaces are selected description: Boolean describing whether all namespaces are selected in contrast to a list restricting them.
in contrast to a list restricting them.
type: boolean type: boolean
matchNames: matchNames:
description: List of namespace names. description: List of namespace names.
@ -55,63 +48,45 @@ spec:
podMetricsEndpoints: podMetricsEndpoints:
description: A list of endpoints allowed as part of this PodMonitor. description: A list of endpoints allowed as part of this PodMonitor.
items: items:
description: PodMetricsEndpoint defines a scrapeable endpoint of description: PodMetricsEndpoint defines a scrapeable endpoint of a Kubernetes Pod serving Prometheus metrics.
a Kubernetes Pod serving Prometheus metrics.
properties: properties:
honorLabels: honorLabels:
description: HonorLabels chooses the metric's labels on collisions description: HonorLabels chooses the metric's labels on collisions with target labels.
with target labels.
type: boolean type: boolean
honorTimestamps: honorTimestamps:
description: HonorTimestamps controls whether Prometheus respects description: HonorTimestamps controls whether Prometheus respects the timestamps present in scraped data.
the timestamps present in scraped data.
type: boolean type: boolean
interval: interval:
description: Interval at which metrics should be scraped description: Interval at which metrics should be scraped
type: string type: string
metricRelabelings: metricRelabelings:
description: MetricRelabelConfigs to apply to samples before description: MetricRelabelConfigs to apply to samples before ingestion.
ingestion.
items: items:
description: 'RelabelConfig allows dynamic rewriting of the description: 'RelabelConfig allows dynamic rewriting of the label set, being applied to samples before ingestion. It defines `<metric_relabel_configs>`-section of Prometheus configuration. More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#metric_relabel_configs'
label set, being applied to samples before ingestion. It
defines `<metric_relabel_configs>`-section of Prometheus
configuration. More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#metric_relabel_configs'
properties: properties:
action: action:
description: Action to perform based on regex matching. description: Action to perform based on regex matching. Default is 'replace'
Default is 'replace'
type: string type: string
modulus: modulus:
description: Modulus to take of the hash of the source description: Modulus to take of the hash of the source label values.
label values.
format: int64 format: int64
type: integer type: integer
regex: regex:
description: Regular expression against which the extracted description: Regular expression against which the extracted value is matched. Default is '(.*)'
value is matched. Default is '(.*)'
type: string type: string
replacement: replacement:
description: Replacement value against which a regex replace description: Replacement value against which a regex replace is performed if the regular expression matches. Regex capture groups are available. Default is '$1'
is performed if the regular expression matches. Regex
capture groups are available. Default is '$1'
type: string type: string
separator: separator:
description: Separator placed between concatenated source description: Separator placed between concatenated source label values. default is ';'.
label values. default is ';'.
type: string type: string
sourceLabels: sourceLabels:
description: The source labels select values from existing description: The source labels select values from existing labels. Their content is concatenated using the configured separator and matched against the configured regular expression for the replace, keep, and drop actions.
labels. Their content is concatenated using the configured
separator and matched against the configured regular
expression for the replace, keep, and drop actions.
items: items:
type: string type: string
type: array type: array
targetLabel: targetLabel:
description: Label to which the resulting value is written description: Label to which the resulting value is written in a replace action. It is mandatory for replace actions. Regex capture groups are available.
in a replace action. It is mandatory for replace actions.
Regex capture groups are available.
type: string type: string
type: object type: object
type: array type: array
@ -126,56 +101,39 @@ spec:
description: HTTP path to scrape for metrics. description: HTTP path to scrape for metrics.
type: string type: string
port: port:
description: Name of the pod port this endpoint refers to. Mutually description: Name of the pod port this endpoint refers to. Mutually exclusive with targetPort.
exclusive with targetPort.
type: string type: string
proxyUrl: proxyUrl:
description: ProxyURL eg http://proxyserver:2195 Directs scrapes description: ProxyURL eg http://proxyserver:2195 Directs scrapes to proxy through this endpoint.
to proxy through this endpoint.
type: string type: string
relabelings: relabelings:
description: 'RelabelConfigs to apply to samples before ingestion. description: 'RelabelConfigs to apply to samples before ingestion. More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config'
More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config'
items: items:
description: 'RelabelConfig allows dynamic rewriting of the description: 'RelabelConfig allows dynamic rewriting of the label set, being applied to samples before ingestion. It defines `<metric_relabel_configs>`-section of Prometheus configuration. More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#metric_relabel_configs'
label set, being applied to samples before ingestion. It
defines `<metric_relabel_configs>`-section of Prometheus
configuration. More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#metric_relabel_configs'
properties: properties:
action: action:
description: Action to perform based on regex matching. description: Action to perform based on regex matching. Default is 'replace'
Default is 'replace'
type: string type: string
modulus: modulus:
description: Modulus to take of the hash of the source description: Modulus to take of the hash of the source label values.
label values.
format: int64 format: int64
type: integer type: integer
regex: regex:
description: Regular expression against which the extracted description: Regular expression against which the extracted value is matched. Default is '(.*)'
value is matched. Default is '(.*)'
type: string type: string
replacement: replacement:
description: Replacement value against which a regex replace description: Replacement value against which a regex replace is performed if the regular expression matches. Regex capture groups are available. Default is '$1'
is performed if the regular expression matches. Regex
capture groups are available. Default is '$1'
type: string type: string
separator: separator:
description: Separator placed between concatenated source description: Separator placed between concatenated source label values. default is ';'.
label values. default is ';'.
type: string type: string
sourceLabels: sourceLabels:
description: The source labels select values from existing description: The source labels select values from existing labels. Their content is concatenated using the configured separator and matched against the configured regular expression for the replace, keep, and drop actions.
labels. Their content is concatenated using the configured
separator and matched against the configured regular
expression for the replace, keep, and drop actions.
items: items:
type: string type: string
type: array type: array
targetLabel: targetLabel:
description: Label to which the resulting value is written description: Label to which the resulting value is written in a replace action. It is mandatory for replace actions. Regex capture groups are available.
in a replace action. It is mandatory for replace actions.
Regex capture groups are available.
type: string type: string
type: object type: object
type: array type: array
@ -194,42 +152,30 @@ spec:
type: object type: object
type: array type: array
podTargetLabels: podTargetLabels:
description: PodTargetLabels transfers labels on the Kubernetes Pod description: PodTargetLabels transfers labels on the Kubernetes Pod onto the target.
onto the target.
items: items:
type: string type: string
type: array type: array
sampleLimit: sampleLimit:
description: SampleLimit defines per-scrape limit on number of scraped description: SampleLimit defines per-scrape limit on number of scraped samples that will be accepted.
samples that will be accepted.
format: int64 format: int64
type: integer type: integer
selector: selector:
description: Selector to select Pod objects. description: Selector to select Pod objects.
properties: properties:
matchExpressions: matchExpressions:
description: matchExpressions is a list of label selector requirements. description: matchExpressions is a list of label selector requirements. The requirements are ANDed.
The requirements are ANDed.
items: items:
description: A label selector requirement is a selector that description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.
contains values, a key, and an operator that relates the key
and values.
properties: properties:
key: key:
description: key is the label key that the selector applies description: key is the label key that the selector applies to.
to.
type: string type: string
operator: operator:
description: operator represents a key's relationship to description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.
a set of values. Valid operators are In, NotIn, Exists
and DoesNotExist.
type: string type: string
values: values:
description: values is an array of string values. If the description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.
operator is In or NotIn, the values array must be non-empty.
If the operator is Exists or DoesNotExist, the values
array must be empty. This array is replaced during a strategic
merge patch.
items: items:
type: string type: string
type: array type: array
@ -241,11 +187,7 @@ spec:
matchLabels: matchLabels:
additionalProperties: additionalProperties:
type: string type: string
description: matchLabels is a map of {key,value} pairs. A single description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed.
{key,value} in the matchLabels map is equivalent to an element
of matchExpressions, whose key field is "key", the operator
is "In", and the values array contains only "value". The requirements
are ANDed.
type: object type: object
type: object type: object
required: required:

3814
manifests/setup/prometheus-operator-0prometheusCustomResourceDefinition.yaml
View File

File diff suppressed because it is too large Load Diff

13
manifests/setup/prometheus-operator-0prometheusruleCustomResourceDefinition.yaml
View File

@ -20,14 +20,10 @@ spec:
description: PrometheusRule defines alerting rules for a Prometheus instance description: PrometheusRule defines alerting rules for a Prometheus instance
properties: properties:
apiVersion: apiVersion:
description: 'APIVersion defines the versioned schema of this representation description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
of an object. Servers should convert recognized schemas to the latest
internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
type: string type: string
kind: kind:
description: 'Kind is a string value representing the REST resource this description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
object represents. Servers may infer this from the endpoint the client
submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
type: string type: string
metadata: metadata:
type: object type: object
@ -37,10 +33,7 @@ spec:
groups: groups:
description: Content of Prometheus rule file description: Content of Prometheus rule file
items: items:
description: 'RuleGroup is a list of sequentially evaluated recording description: 'RuleGroup is a list of sequentially evaluated recording and alerting rules. Note: PartialResponseStrategy is only used by ThanosRuler and will be ignored by Prometheus instances. Valid values for this field are ''warn'' or ''abort''. More info: https://github.com/thanos-io/thanos/blob/master/docs/components/rule.md#partial-response'
and alerting rules. Note: PartialResponseStrategy is only used
by ThanosRuler and will be ignored by Prometheus instances. Valid
values for this field are ''warn'' or ''abort''. More info: https://github.com/thanos-io/thanos/blob/master/docs/components/rule.md#partial-response'
properties: properties:
interval: interval:
type: string type: string

243
manifests/setup/prometheus-operator-0servicemonitorCustomResourceDefinition.yaml
View File

@ -20,65 +20,50 @@ spec:
description: ServiceMonitor defines monitoring for a set of services. description: ServiceMonitor defines monitoring for a set of services.
properties: properties:
apiVersion: apiVersion:
description: 'APIVersion defines the versioned schema of this representation description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
of an object. Servers should convert recognized schemas to the latest
internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
type: string type: string
kind: kind:
description: 'Kind is a string value representing the REST resource this description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
object represents. Servers may infer this from the endpoint the client
submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
type: string type: string
metadata: metadata:
type: object type: object
spec: spec:
description: Specification of desired Service selection for target discovery description: Specification of desired Service selection for target discovery by Prometheus.
by Prometheus.
properties: properties:
endpoints: endpoints:
description: A list of endpoints allowed as part of this ServiceMonitor. description: A list of endpoints allowed as part of this ServiceMonitor.
items: items:
description: Endpoint defines a scrapeable endpoint serving Prometheus description: Endpoint defines a scrapeable endpoint serving Prometheus metrics.
metrics.
properties: properties:
basicAuth: basicAuth:
description: 'BasicAuth allow an endpoint to authenticate over description: 'BasicAuth allow an endpoint to authenticate over basic authentication More info: https://prometheus.io/docs/operating/configuration/#endpoints'
basic authentication More info: https://prometheus.io/docs/operating/configuration/#endpoints'
properties: properties:
password: password:
description: The secret in the service monitor namespace description: The secret in the service monitor namespace that contains the password for authentication.
that contains the password for authentication.
properties: properties:
key: key:
description: The key of the secret to select from. Must description: The key of the secret to select from. Must be a valid secret key.
be a valid secret key.
type: string type: string
name: name:
description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'
TODO: Add other useful fields. apiVersion, kind, uid?'
type: string type: string
optional: optional:
description: Specify whether the Secret or its key must description: Specify whether the Secret or its key must be defined
be defined
type: boolean type: boolean
required: required:
- key - key
type: object type: object
username: username:
description: The secret in the service monitor namespace description: The secret in the service monitor namespace that contains the username for authentication.
that contains the username for authentication.
properties: properties:
key: key:
description: The key of the secret to select from. Must description: The key of the secret to select from. Must be a valid secret key.
be a valid secret key.
type: string type: string
name: name:
description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'
TODO: Add other useful fields. apiVersion, kind, uid?'
type: string type: string
optional: optional:
description: Specify whether the Secret or its key must description: Specify whether the Secret or its key must be defined
be defined
type: boolean type: boolean
required: required:
- key - key
@ -88,79 +73,57 @@ spec:
description: File to read bearer token for scraping targets. description: File to read bearer token for scraping targets.
type: string type: string
bearerTokenSecret: bearerTokenSecret:
description: Secret to mount to read bearer token for scraping description: Secret to mount to read bearer token for scraping targets. The secret needs to be in the same namespace as the service monitor and accessible by the Prometheus Operator.
targets. The secret needs to be in the same namespace as the
service monitor and accessible by the Prometheus Operator.
properties: properties:
key: key:
description: The key of the secret to select from. Must description: The key of the secret to select from. Must be a valid secret key.
be a valid secret key.
type: string type: string
name: name:
description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'
TODO: Add other useful fields. apiVersion, kind, uid?'
type: string type: string
optional: optional:
description: Specify whether the Secret or its key must description: Specify whether the Secret or its key must be defined
be defined
type: boolean type: boolean
required: required:
- key - key
type: object type: object
honorLabels: honorLabels:
description: HonorLabels chooses the metric's labels on collisions description: HonorLabels chooses the metric's labels on collisions with target labels.
with target labels.
type: boolean type: boolean
honorTimestamps: honorTimestamps:
description: HonorTimestamps controls whether Prometheus respects description: HonorTimestamps controls whether Prometheus respects the timestamps present in scraped data.
the timestamps present in scraped data.
type: boolean type: boolean
interval: interval:
description: Interval at which metrics should be scraped description: Interval at which metrics should be scraped
type: string type: string
metricRelabelings: metricRelabelings:
description: MetricRelabelConfigs to apply to samples before description: MetricRelabelConfigs to apply to samples before ingestion.
ingestion.
items: items:
description: 'RelabelConfig allows dynamic rewriting of the description: 'RelabelConfig allows dynamic rewriting of the label set, being applied to samples before ingestion. It defines `<metric_relabel_configs>`-section of Prometheus configuration. More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#metric_relabel_configs'
label set, being applied to samples before ingestion. It
defines `<metric_relabel_configs>`-section of Prometheus
configuration. More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#metric_relabel_configs'
properties: properties:
action: action:
description: Action to perform based on regex matching. description: Action to perform based on regex matching. Default is 'replace'
Default is 'replace'
type: string type: string
modulus: modulus:
description: Modulus to take of the hash of the source description: Modulus to take of the hash of the source label values.
label values.
format: int64 format: int64
type: integer type: integer
regex: regex:
description: Regular expression against which the extracted description: Regular expression against which the extracted value is matched. Default is '(.*)'
value is matched. Default is '(.*)'
type: string type: string
replacement: replacement:
description: Replacement value against which a regex replace description: Replacement value against which a regex replace is performed if the regular expression matches. Regex capture groups are available. Default is '$1'
is performed if the regular expression matches. Regex
capture groups are available. Default is '$1'
type: string type: string
separator: separator:
description: Separator placed between concatenated source description: Separator placed between concatenated source label values. default is ';'.
label values. default is ';'.
type: string type: string
sourceLabels: sourceLabels:
description: The source labels select values from existing description: The source labels select values from existing labels. Their content is concatenated using the configured separator and matched against the configured regular expression for the replace, keep, and drop actions.
labels. Their content is concatenated using the configured
separator and matched against the configured regular
expression for the replace, keep, and drop actions.
items: items:
type: string type: string
type: array type: array
targetLabel: targetLabel:
description: Label to which the resulting value is written description: Label to which the resulting value is written in a replace action. It is mandatory for replace actions. Regex capture groups are available.
in a replace action. It is mandatory for replace actions.
Regex capture groups are available.
type: string type: string
type: object type: object
type: array type: array
@ -175,56 +138,39 @@ spec:
description: HTTP path to scrape for metrics. description: HTTP path to scrape for metrics.
type: string type: string
port: port:
description: Name of the service port this endpoint refers to. description: Name of the service port this endpoint refers to. Mutually exclusive with targetPort.
Mutually exclusive with targetPort.
type: string type: string
proxyUrl: proxyUrl:
description: ProxyURL eg http://proxyserver:2195 Directs scrapes description: ProxyURL eg http://proxyserver:2195 Directs scrapes to proxy through this endpoint.
to proxy through this endpoint.
type: string type: string
relabelings: relabelings:
description: 'RelabelConfigs to apply to samples before scraping. description: 'RelabelConfigs to apply to samples before scraping. More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config'
More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config'
items: items:
description: 'RelabelConfig allows dynamic rewriting of the description: 'RelabelConfig allows dynamic rewriting of the label set, being applied to samples before ingestion. It defines `<metric_relabel_configs>`-section of Prometheus configuration. More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#metric_relabel_configs'
label set, being applied to samples before ingestion. It
defines `<metric_relabel_configs>`-section of Prometheus
configuration. More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#metric_relabel_configs'
properties: properties:
action: action:
description: Action to perform based on regex matching. description: Action to perform based on regex matching. Default is 'replace'
Default is 'replace'
type: string type: string
modulus: modulus:
description: Modulus to take of the hash of the source description: Modulus to take of the hash of the source label values.
label values.
format: int64 format: int64
type: integer type: integer
regex: regex:
description: Regular expression against which the extracted description: Regular expression against which the extracted value is matched. Default is '(.*)'
value is matched. Default is '(.*)'
type: string type: string
replacement: replacement:
description: Replacement value against which a regex replace description: Replacement value against which a regex replace is performed if the regular expression matches. Regex capture groups are available. Default is '$1'
is performed if the regular expression matches. Regex
capture groups are available. Default is '$1'
type: string type: string
separator: separator:
description: Separator placed between concatenated source description: Separator placed between concatenated source label values. default is ';'.
label values. default is ';'.
type: string type: string
sourceLabels: sourceLabels:
description: The source labels select values from existing description: The source labels select values from existing labels. Their content is concatenated using the configured separator and matched against the configured regular expression for the replace, keep, and drop actions.
labels. Their content is concatenated using the configured
separator and matched against the configured regular
expression for the replace, keep, and drop actions.
items: items:
type: string type: string
type: array type: array
targetLabel: targetLabel:
description: Label to which the resulting value is written description: Label to which the resulting value is written in a replace action. It is mandatory for replace actions. Regex capture groups are available.
in a replace action. It is mandatory for replace actions.
Regex capture groups are available.
type: string type: string
type: object type: object
type: array type: array
@ -238,31 +184,25 @@ spec:
anyOf: anyOf:
- type: integer - type: integer
- type: string - type: string
description: Name or number of the pod port this endpoint refers description: Name or number of the pod port this endpoint refers to. Mutually exclusive with port.
to. Mutually exclusive with port.
x-kubernetes-int-or-string: true x-kubernetes-int-or-string: true
tlsConfig: tlsConfig:
description: TLS configuration to use when scraping the endpoint description: TLS configuration to use when scraping the endpoint
properties: properties:
ca: ca:
description: Stuct containing the CA cert to use for the description: Stuct containing the CA cert to use for the targets.
targets.
properties: properties:
configMap: configMap:
description: ConfigMap containing data to use for the description: ConfigMap containing data to use for the targets.
targets.
properties: properties:
key: key:
description: The key to select. description: The key to select.
type: string type: string
name: name:
description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'
TODO: Add other useful fields. apiVersion, kind,
uid?'
type: string type: string
optional: optional:
description: Specify whether the ConfigMap or its description: Specify whether the ConfigMap or its key must be defined
key must be defined
type: boolean type: boolean
required: required:
- key - key
@ -271,45 +211,35 @@ spec:
description: Secret containing data to use for the targets. description: Secret containing data to use for the targets.
properties: properties:
key: key:
description: The key of the secret to select from. Must description: The key of the secret to select from. Must be a valid secret key.
be a valid secret key.
type: string type: string
name: name:
description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'
TODO: Add other useful fields. apiVersion, kind,
uid?'
type: string type: string
optional: optional:
description: Specify whether the Secret or its key description: Specify whether the Secret or its key must be defined
must be defined
type: boolean type: boolean
required: required:
- key - key
type: object type: object
type: object type: object
caFile: caFile:
description: Path to the CA cert in the Prometheus container description: Path to the CA cert in the Prometheus container to use for the targets.
to use for the targets.
type: string type: string
cert: cert:
description: Struct containing the client cert file for description: Struct containing the client cert file for the targets.
the targets.
properties: properties:
configMap: configMap:
description: ConfigMap containing data to use for the description: ConfigMap containing data to use for the targets.
targets.
properties: properties:
key: key:
description: The key to select. description: The key to select.
type: string type: string
name: name:
description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'
TODO: Add other useful fields. apiVersion, kind,
uid?'
type: string type: string
optional: optional:
description: Specify whether the ConfigMap or its description: Specify whether the ConfigMap or its key must be defined
key must be defined
type: boolean type: boolean
required: required:
- key - key
@ -318,48 +248,38 @@ spec:
description: Secret containing data to use for the targets. description: Secret containing data to use for the targets.
properties: properties:
key: key:
description: The key of the secret to select from. Must description: The key of the secret to select from. Must be a valid secret key.
be a valid secret key.
type: string type: string
name: name:
description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'
TODO: Add other useful fields. apiVersion, kind,
uid?'
type: string type: string
optional: optional:
description: Specify whether the Secret or its key description: Specify whether the Secret or its key must be defined
must be defined
type: boolean type: boolean
required: required:
- key - key
type: object type: object
type: object type: object
certFile: certFile:
description: Path to the client cert file in the Prometheus description: Path to the client cert file in the Prometheus container for the targets.
container for the targets.
type: string type: string
insecureSkipVerify: insecureSkipVerify:
description: Disable target certificate validation. description: Disable target certificate validation.
type: boolean type: boolean
keyFile: keyFile:
description: Path to the client key file in the Prometheus description: Path to the client key file in the Prometheus container for the targets.
container for the targets.
type: string type: string
keySecret: keySecret:
description: Secret containing the client key file for the description: Secret containing the client key file for the targets.
targets.
properties: properties:
key: key:
description: The key of the secret to select from. Must description: The key of the secret to select from. Must be a valid secret key.
be a valid secret key.
type: string type: string
name: name:
description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names TODO: Add other useful fields. apiVersion, kind, uid?'
TODO: Add other useful fields. apiVersion, kind, uid?'
type: string type: string
optional: optional:
description: Specify whether the Secret or its key must description: Specify whether the Secret or its key must be defined
be defined
type: boolean type: boolean
required: required:
- key - key
@ -374,12 +294,10 @@ spec:
description: The label to use to retrieve the job name from. description: The label to use to retrieve the job name from.
type: string type: string
namespaceSelector: namespaceSelector:
description: Selector to select which namespaces the Endpoints objects description: Selector to select which namespaces the Endpoints objects are discovered from.
are discovered from.
properties: properties:
any: any:
description: Boolean describing whether all namespaces are selected description: Boolean describing whether all namespaces are selected in contrast to a list restricting them.
in contrast to a list restricting them.
type: boolean type: boolean
matchNames: matchNames:
description: List of namespace names. description: List of namespace names.
@ -388,42 +306,30 @@ spec:
type: array type: array
type: object type: object
podTargetLabels: podTargetLabels:
description: PodTargetLabels transfers labels on the Kubernetes Pod description: PodTargetLabels transfers labels on the Kubernetes Pod onto the target.
onto the target.
items: items:
type: string type: string
type: array type: array
sampleLimit: sampleLimit:
description: SampleLimit defines per-scrape limit on number of scraped description: SampleLimit defines per-scrape limit on number of scraped samples that will be accepted.
samples that will be accepted.
format: int64 format: int64
type: integer type: integer
selector: selector:
description: Selector to select Endpoints objects. description: Selector to select Endpoints objects.
properties: properties:
matchExpressions: matchExpressions:
description: matchExpressions is a list of label selector requirements. description: matchExpressions is a list of label selector requirements. The requirements are ANDed.
The requirements are ANDed.
items: items:
description: A label selector requirement is a selector that description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values.
contains values, a key, and an operator that relates the key
and values.
properties: properties:
key: key:
description: key is the label key that the selector applies description: key is the label key that the selector applies to.
to.
type: string type: string
operator: operator:
description: operator represents a key's relationship to description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist.
a set of values. Valid operators are In, NotIn, Exists
and DoesNotExist.
type: string type: string
values: values:
description: values is an array of string values. If the description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch.
operator is In or NotIn, the values array must be non-empty.
If the operator is Exists or DoesNotExist, the values
array must be empty. This array is replaced during a strategic
merge patch.
items: items:
type: string type: string
type: array type: array
@ -435,16 +341,11 @@ spec:
matchLabels: matchLabels:
additionalProperties: additionalProperties:
type: string type: string
description: matchLabels is a map of {key,value} pairs. A single description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed.
{key,value} in the matchLabels map is equivalent to an element
of matchExpressions, whose key field is "key", the operator
is "In", and the values array contains only "value". The requirements
are ANDed.
type: object type: object
type: object type: object
targetLabels: targetLabels:
description: TargetLabels transfers labels on the Kubernetes Service description: TargetLabels transfers labels on the Kubernetes Service onto the target.
onto the target.
items: items:
type: string type: string
type: array type: array

3028
manifests/setup/prometheus-operator-0thanosrulerCustomResourceDefinition.yaml
View File

File diff suppressed because it is too large Load Diff