From aa4e652e4314a20852e1118cfd4b692d64c6d895 Mon Sep 17 00:00:00 2001
From: Edd Robinson <me@edd.io>
Date: Mon, 25 Mar 2019 15:25:03 +0000
Subject: [PATCH] Add reason to total compaction metric
This commit adds a reason label to the total compaction metric. For
snapshots, the reason will indicate why the cache was snapshotted. For
other compactions, the reason label will be blank.
---
go.mod | 8 ++++----
go.sum | 8 ++++++++
tsdb/tsm1/cachestatus_string.go | 26 ++++++++++++++++++++++++++
tsdb/tsm1/engine.go | 20 +++++++++++---------
tsdb/tsm1/metrics.go | 2 +-
tsdb/tsm1/metrics_test.go | 2 ++
6 files changed, 52 insertions(+), 14 deletions(-)
create mode 100644 tsdb/tsm1/cachestatus_string.go
diff --git a/go.mod b/go.mod
index 4e44c651d5..0923599b8b 100644
--- a/go.mod
+++ b/go.mod
@@ -121,13 +121,13 @@ require (
github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82 // indirect
github.com/yudai/pp v2.0.1+incompatible // indirect
go.uber.org/zap v1.9.1
- golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9
- golang.org/x/net v0.0.0-20181106065722-10aee1819953
+ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2
+ golang.org/x/net v0.0.0-20190311183353-d8887717615a
golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f
- golang.org/x/sys v0.0.0-20181228144115-9a3f9b0469bb
+ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c
- golang.org/x/tools v0.0.0-20181221154417-3ad2d988d5e2
+ golang.org/x/tools v0.0.0-20190322203728-c1a832b0ad89
google.golang.org/api v0.0.0-20181021000519-a2651947f503
google.golang.org/genproto v0.0.0-20190108161440-ae2f86662275 // indirect
google.golang.org/grpc v1.17.0
diff --git a/go.sum b/go.sum
index d7ca6b3de2..18931cff77 100644
--- a/go.sum
+++ b/go.sum
@@ -430,6 +430,8 @@ golang.org/x/crypto v0.0.0-20180505025534-4ec37c66abab/go.mod h1:6SG95UA2DQfeDnf
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9 h1:mKdxBk7AujPs8kU4m80U72y/zjbZ3UcXC7dClwKbUI0=
golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20181112044915-a3060d491354 h1:6UAgZ8309zQ9+1iWkHzfszFguqzOdHGyGkd1HmhJ+UE=
golang.org/x/exp v0.0.0-20181112044915-a3060d491354/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -441,6 +443,8 @@ golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73r
golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181106065722-10aee1819953 h1:LuZIitY8waaxUfNIdtajyE/YzA/zyf0YxXG27VpLrkg=
golang.org/x/net v0.0.0-20181106065722-10aee1819953/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4 h1:99CA0JJbUX4ozCnLon680Jc9e0T1i8HCaLVJMwtI8Hc=
golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -456,6 +460,8 @@ golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a h1:1n5lsVfiQW3yfsRGu98756EH1
golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181228144115-9a3f9b0469bb h1:pf3XwC90UUdNPYWZdFjhGBE7DUFuK3Ct1zWmZ65QN30=
golang.org/x/sys v0.0.0-20181228144115-9a3f9b0469bb/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c h1:fqgJT0MGcGpPgpWU7VRdRjuArfcOvC4AoJmILihzhDg=
@@ -464,6 +470,8 @@ golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGm
golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20181221154417-3ad2d988d5e2 h1:M7NLB69gFpUH4s6SJLwXiVs45aZfVjqGKynfNFKSGcI=
golang.org/x/tools v0.0.0-20181221154417-3ad2d988d5e2/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190322203728-c1a832b0ad89 h1:iWXXYN3edZ3Nd/7I6Rt1sXrWVmhF9bgVtlEJ7BbH124=
+golang.org/x/tools v0.0.0-20190322203728-c1a832b0ad89/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
gonum.org/v1/gonum v0.0.0-20181121035319-3f7ecaa7e8ca h1:PupagGYwj8+I4ubCxcmcBRk3VlUWtTg5huQpZR9flmE=
gonum.org/v1/gonum v0.0.0-20181121035319-3f7ecaa7e8ca/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo=
gonum.org/v1/netlib v0.0.0-20181029234149-ec6d1f5cefe6 h1:4WsZyVtkthqrHTbDCJfiTs8IWNYE4uvsSDgaV6xpp+o=
diff --git a/tsdb/tsm1/cachestatus_string.go b/tsdb/tsm1/cachestatus_string.go
new file mode 100644
index 0000000000..651b7524d7
--- /dev/null
+++ b/tsdb/tsm1/cachestatus_string.go
@@ -0,0 +1,26 @@
+// Code generated by "stringer -type=CacheStatus"; DO NOT EDIT.
+
+package tsm1
+
+import "strconv"
+
+func _() {
+ // An "invalid array index" compiler error signifies that the constant values have changed.
+ // Re-run the stringer command to generate them again.
+ var x [1]struct{}
+ _ = x[CacheStatusOkay-0]
+ _ = x[CacheStatusSizeExceeded-1]
+ _ = x[CacheStatusAgeExceeded-2]
+ _ = x[CacheStatusColdNoWrites-3]
+}
+
+const _CacheStatus_name = "CacheStatusOkayCacheStatusSizeExceededCacheStatusAgeExceededCacheStatusColdNoWrites"
+
+var _CacheStatus_index = [...]uint8{0, 15, 38, 60, 83}
+
+func (i CacheStatus) String() string {
+ if i < 0 || i >= CacheStatus(len(_CacheStatus_index)-1) {
+ return "CacheStatus(" + strconv.FormatInt(int64(i), 10) + ")"
+ }
+ return _CacheStatus_name[_CacheStatus_index[i]:_CacheStatus_index[i+1]]
+}
diff --git a/tsdb/tsm1/engine.go b/tsdb/tsm1/engine.go
index 8f8a365b22..f634e3e2a3 100644
--- a/tsdb/tsm1/engine.go
+++ b/tsdb/tsm1/engine.go
@@ -36,6 +36,7 @@ import (
//go:generate env GO111MODULE=on go run github.com/benbjohnson/tmpl -data=@encoding.gen.go.tmpldata encoding.gen.go.tmpl
//go:generate env GO111MODULE=on go run github.com/benbjohnson/tmpl -data=@compact.gen.go.tmpldata compact.gen.go.tmpl
//go:generate env GO111MODULE=on go run github.com/benbjohnson/tmpl -data=@reader.gen.go.tmpldata reader.gen.go.tmpl
+//go:generate stringer -type=CacheStatus
var (
// Static objects to prevent small allocs.
@@ -739,17 +740,17 @@ func (t *compactionTracker) DecActive(level compactionLevel) {
func (t *compactionTracker) DecFullActive() { t.DecActive(5) }
// Attempted updates the number of compactions attempted for the provided level.
-func (t *compactionTracker) Attempted(level compactionLevel, success bool, duration time.Duration) {
+func (t *compactionTracker) Attempted(level compactionLevel, success bool, reason string, duration time.Duration) {
if success {
atomic.AddUint64(&t.ok[level], 1)
labels := t.Labels(level)
-
t.metrics.CompactionDuration.With(labels).Observe(duration.Seconds())
+ // Total compactions metric has reason and status.
+ labels["reason"] = reason
labels["status"] = "ok"
t.metrics.Compactions.With(labels).Inc()
-
return
}
@@ -757,12 +758,13 @@ func (t *compactionTracker) Attempted(level compactionLevel, success bool, durat
labels := t.Labels(level)
labels["status"] = "error"
+ labels["reason"] = reason
t.metrics.Compactions.With(labels).Inc()
}
// SnapshotAttempted updates the number of snapshots attempted.
-func (t *compactionTracker) SnapshotAttempted(success bool, duration time.Duration) {
- t.Attempted(0, success, duration)
+func (t *compactionTracker) SnapshotAttempted(success bool, reason CacheStatus, duration time.Duration) {
+ t.Attempted(0, success, reason.String(), duration)
}
// SetQueue sets the compaction queue depth for the provided level.
@@ -886,7 +888,7 @@ func (e *Engine) compactCache() {
if err != nil && err != errCompactionsDisabled {
e.logger.Info("Error writing snapshot", zap.Error(err))
}
- e.compactionTracker.SnapshotAttempted(err == nil || err == errCompactionsDisabled, time.Since(start))
+ e.compactionTracker.SnapshotAttempted(err == nil || err == errCompactionsDisabled, status, time.Since(start))
span.Finish()
}
@@ -1140,14 +1142,14 @@ func (s *compactionStrategy) compactGroup() {
}
log.Info("Error compacting TSM files", zap.Error(err))
- s.tracker.Attempted(s.level, false, 0)
+ s.tracker.Attempted(s.level, false, "", 0)
time.Sleep(time.Second)
return
}
if err := s.fileStore.ReplaceWithCallback(group, files, nil); err != nil {
log.Info("Error replacing new TSM files", zap.Error(err))
- s.tracker.Attempted(s.level, false, 0)
+ s.tracker.Attempted(s.level, false, "", 0)
time.Sleep(time.Second)
return
}
@@ -1156,7 +1158,7 @@ func (s *compactionStrategy) compactGroup() {
log.Info("Compacted file", zap.Int("tsm1_index", i), zap.String("tsm1_file", f))
}
log.Info("Finished compacting files", zap.Int("tsm1_files_n", len(files)))
- s.tracker.Attempted(s.level, true, time.Since(now))
+ s.tracker.Attempted(s.level, true, "", time.Since(now))
}
// levelCompactionStrategy returns a compactionStrategy for the given level.
diff --git a/tsdb/tsm1/metrics.go b/tsdb/tsm1/metrics.go
index 7677f4aaf4..fb7de95429 100644
--- a/tsdb/tsm1/metrics.go
+++ b/tsdb/tsm1/metrics.go
@@ -81,7 +81,7 @@ func newCompactionMetrics(labels prometheus.Labels) *compactionMetrics {
}
sort.Strings(names)
- totalCompactionsNames := append(append([]string(nil), names...), "status")
+ totalCompactionsNames := append(append([]string(nil), names...), []string{"reason", "status"}...)
sort.Strings(totalCompactionsNames)
return &compactionMetrics{
diff --git a/tsdb/tsm1/metrics_test.go b/tsdb/tsm1/metrics_test.go
index 3372adf3f0..6729affa60 100644
--- a/tsdb/tsm1/metrics_test.go
+++ b/tsdb/tsm1/metrics_test.go
@@ -159,6 +159,7 @@ func TestMetrics_Compactions(t *testing.T) {
labels = tracker.Labels(2)
labels["status"] = "ok"
+ labels["reason"] = CacheStatusAgeExceeded.String()
tracker.metrics.Compactions.With(labels).Add(float64(i + len(counters[0])))
labels = tracker.Labels(2)
@@ -197,6 +198,7 @@ func TestMetrics_Compactions(t *testing.T) {
l[k] = v
}
l["status"] = "ok"
+ l["reason"] = CacheStatusAgeExceeded.String()
metric := promtest.MustFindMetric(t, mfs, name, l)
if got := metric.GetCounter().GetValue(); got != exp {