Add reason to total compaction metric
This commit adds a reason label to the total compaction metric. For snapshots, the reason will indicate why the cache was snapshotted. For other compactions, the reason label will be blank.pull/12847/head
parent
dbca30dac5
commit
aa4e652e43
8
go.mod
8
go.mod
|
@ -121,13 +121,13 @@ require (
|
|||
github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82 // indirect
|
||||
github.com/yudai/pp v2.0.1+incompatible // indirect
|
||||
go.uber.org/zap v1.9.1
|
||||
golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9
|
||||
golang.org/x/net v0.0.0-20181106065722-10aee1819953
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2
|
||||
golang.org/x/net v0.0.0-20190311183353-d8887717615a
|
||||
golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4
|
||||
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f
|
||||
golang.org/x/sys v0.0.0-20181228144115-9a3f9b0469bb
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a
|
||||
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c
|
||||
golang.org/x/tools v0.0.0-20181221154417-3ad2d988d5e2
|
||||
golang.org/x/tools v0.0.0-20190322203728-c1a832b0ad89
|
||||
google.golang.org/api v0.0.0-20181021000519-a2651947f503
|
||||
google.golang.org/genproto v0.0.0-20190108161440-ae2f86662275 // indirect
|
||||
google.golang.org/grpc v1.17.0
|
||||
|
|
8
go.sum
8
go.sum
|
@ -430,6 +430,8 @@ golang.org/x/crypto v0.0.0-20180505025534-4ec37c66abab/go.mod h1:6SG95UA2DQfeDnf
|
|||
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9 h1:mKdxBk7AujPs8kU4m80U72y/zjbZ3UcXC7dClwKbUI0=
|
||||
golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20181112044915-a3060d491354 h1:6UAgZ8309zQ9+1iWkHzfszFguqzOdHGyGkd1HmhJ+UE=
|
||||
golang.org/x/exp v0.0.0-20181112044915-a3060d491354/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
|
@ -441,6 +443,8 @@ golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73r
|
|||
golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20181106065722-10aee1819953 h1:LuZIitY8waaxUfNIdtajyE/YzA/zyf0YxXG27VpLrkg=
|
||||
golang.org/x/net v0.0.0-20181106065722-10aee1819953/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628=
|
||||
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4 h1:99CA0JJbUX4ozCnLon680Jc9e0T1i8HCaLVJMwtI8Hc=
|
||||
golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
|
@ -456,6 +460,8 @@ golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a h1:1n5lsVfiQW3yfsRGu98756EH1
|
|||
golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20181228144115-9a3f9b0469bb h1:pf3XwC90UUdNPYWZdFjhGBE7DUFuK3Ct1zWmZ65QN30=
|
||||
golang.org/x/sys v0.0.0-20181228144115-9a3f9b0469bb/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c h1:fqgJT0MGcGpPgpWU7VRdRjuArfcOvC4AoJmILihzhDg=
|
||||
|
@ -464,6 +470,8 @@ golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGm
|
|||
golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20181221154417-3ad2d988d5e2 h1:M7NLB69gFpUH4s6SJLwXiVs45aZfVjqGKynfNFKSGcI=
|
||||
golang.org/x/tools v0.0.0-20181221154417-3ad2d988d5e2/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190322203728-c1a832b0ad89 h1:iWXXYN3edZ3Nd/7I6Rt1sXrWVmhF9bgVtlEJ7BbH124=
|
||||
golang.org/x/tools v0.0.0-20190322203728-c1a832b0ad89/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
gonum.org/v1/gonum v0.0.0-20181121035319-3f7ecaa7e8ca h1:PupagGYwj8+I4ubCxcmcBRk3VlUWtTg5huQpZR9flmE=
|
||||
gonum.org/v1/gonum v0.0.0-20181121035319-3f7ecaa7e8ca/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo=
|
||||
gonum.org/v1/netlib v0.0.0-20181029234149-ec6d1f5cefe6 h1:4WsZyVtkthqrHTbDCJfiTs8IWNYE4uvsSDgaV6xpp+o=
|
||||
|
|
|
@ -0,0 +1,26 @@
|
|||
// Code generated by "stringer -type=CacheStatus"; DO NOT EDIT.
|
||||
|
||||
package tsm1
|
||||
|
||||
import "strconv"
|
||||
|
||||
func _() {
|
||||
// An "invalid array index" compiler error signifies that the constant values have changed.
|
||||
// Re-run the stringer command to generate them again.
|
||||
var x [1]struct{}
|
||||
_ = x[CacheStatusOkay-0]
|
||||
_ = x[CacheStatusSizeExceeded-1]
|
||||
_ = x[CacheStatusAgeExceeded-2]
|
||||
_ = x[CacheStatusColdNoWrites-3]
|
||||
}
|
||||
|
||||
const _CacheStatus_name = "CacheStatusOkayCacheStatusSizeExceededCacheStatusAgeExceededCacheStatusColdNoWrites"
|
||||
|
||||
var _CacheStatus_index = [...]uint8{0, 15, 38, 60, 83}
|
||||
|
||||
func (i CacheStatus) String() string {
|
||||
if i < 0 || i >= CacheStatus(len(_CacheStatus_index)-1) {
|
||||
return "CacheStatus(" + strconv.FormatInt(int64(i), 10) + ")"
|
||||
}
|
||||
return _CacheStatus_name[_CacheStatus_index[i]:_CacheStatus_index[i+1]]
|
||||
}
|
|
@ -36,6 +36,7 @@ import (
|
|||
//go:generate env GO111MODULE=on go run github.com/benbjohnson/tmpl -data=@encoding.gen.go.tmpldata encoding.gen.go.tmpl
|
||||
//go:generate env GO111MODULE=on go run github.com/benbjohnson/tmpl -data=@compact.gen.go.tmpldata compact.gen.go.tmpl
|
||||
//go:generate env GO111MODULE=on go run github.com/benbjohnson/tmpl -data=@reader.gen.go.tmpldata reader.gen.go.tmpl
|
||||
//go:generate stringer -type=CacheStatus
|
||||
|
||||
var (
|
||||
// Static objects to prevent small allocs.
|
||||
|
@ -739,17 +740,17 @@ func (t *compactionTracker) DecActive(level compactionLevel) {
|
|||
func (t *compactionTracker) DecFullActive() { t.DecActive(5) }
|
||||
|
||||
// Attempted updates the number of compactions attempted for the provided level.
|
||||
func (t *compactionTracker) Attempted(level compactionLevel, success bool, duration time.Duration) {
|
||||
func (t *compactionTracker) Attempted(level compactionLevel, success bool, reason string, duration time.Duration) {
|
||||
if success {
|
||||
atomic.AddUint64(&t.ok[level], 1)
|
||||
|
||||
labels := t.Labels(level)
|
||||
|
||||
t.metrics.CompactionDuration.With(labels).Observe(duration.Seconds())
|
||||
|
||||
// Total compactions metric has reason and status.
|
||||
labels["reason"] = reason
|
||||
labels["status"] = "ok"
|
||||
t.metrics.Compactions.With(labels).Inc()
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
|
@ -757,12 +758,13 @@ func (t *compactionTracker) Attempted(level compactionLevel, success bool, durat
|
|||
|
||||
labels := t.Labels(level)
|
||||
labels["status"] = "error"
|
||||
labels["reason"] = reason
|
||||
t.metrics.Compactions.With(labels).Inc()
|
||||
}
|
||||
|
||||
// SnapshotAttempted updates the number of snapshots attempted.
|
||||
func (t *compactionTracker) SnapshotAttempted(success bool, duration time.Duration) {
|
||||
t.Attempted(0, success, duration)
|
||||
func (t *compactionTracker) SnapshotAttempted(success bool, reason CacheStatus, duration time.Duration) {
|
||||
t.Attempted(0, success, reason.String(), duration)
|
||||
}
|
||||
|
||||
// SetQueue sets the compaction queue depth for the provided level.
|
||||
|
@ -886,7 +888,7 @@ func (e *Engine) compactCache() {
|
|||
if err != nil && err != errCompactionsDisabled {
|
||||
e.logger.Info("Error writing snapshot", zap.Error(err))
|
||||
}
|
||||
e.compactionTracker.SnapshotAttempted(err == nil || err == errCompactionsDisabled, time.Since(start))
|
||||
e.compactionTracker.SnapshotAttempted(err == nil || err == errCompactionsDisabled, status, time.Since(start))
|
||||
|
||||
span.Finish()
|
||||
}
|
||||
|
@ -1140,14 +1142,14 @@ func (s *compactionStrategy) compactGroup() {
|
|||
}
|
||||
|
||||
log.Info("Error compacting TSM files", zap.Error(err))
|
||||
s.tracker.Attempted(s.level, false, 0)
|
||||
s.tracker.Attempted(s.level, false, "", 0)
|
||||
time.Sleep(time.Second)
|
||||
return
|
||||
}
|
||||
|
||||
if err := s.fileStore.ReplaceWithCallback(group, files, nil); err != nil {
|
||||
log.Info("Error replacing new TSM files", zap.Error(err))
|
||||
s.tracker.Attempted(s.level, false, 0)
|
||||
s.tracker.Attempted(s.level, false, "", 0)
|
||||
time.Sleep(time.Second)
|
||||
return
|
||||
}
|
||||
|
@ -1156,7 +1158,7 @@ func (s *compactionStrategy) compactGroup() {
|
|||
log.Info("Compacted file", zap.Int("tsm1_index", i), zap.String("tsm1_file", f))
|
||||
}
|
||||
log.Info("Finished compacting files", zap.Int("tsm1_files_n", len(files)))
|
||||
s.tracker.Attempted(s.level, true, time.Since(now))
|
||||
s.tracker.Attempted(s.level, true, "", time.Since(now))
|
||||
}
|
||||
|
||||
// levelCompactionStrategy returns a compactionStrategy for the given level.
|
||||
|
|
|
@ -81,7 +81,7 @@ func newCompactionMetrics(labels prometheus.Labels) *compactionMetrics {
|
|||
}
|
||||
sort.Strings(names)
|
||||
|
||||
totalCompactionsNames := append(append([]string(nil), names...), "status")
|
||||
totalCompactionsNames := append(append([]string(nil), names...), []string{"reason", "status"}...)
|
||||
sort.Strings(totalCompactionsNames)
|
||||
|
||||
return &compactionMetrics{
|
||||
|
|
|
@ -159,6 +159,7 @@ func TestMetrics_Compactions(t *testing.T) {
|
|||
|
||||
labels = tracker.Labels(2)
|
||||
labels["status"] = "ok"
|
||||
labels["reason"] = CacheStatusAgeExceeded.String()
|
||||
tracker.metrics.Compactions.With(labels).Add(float64(i + len(counters[0])))
|
||||
|
||||
labels = tracker.Labels(2)
|
||||
|
@ -197,6 +198,7 @@ func TestMetrics_Compactions(t *testing.T) {
|
|||
l[k] = v
|
||||
}
|
||||
l["status"] = "ok"
|
||||
l["reason"] = CacheStatusAgeExceeded.String()
|
||||
|
||||
metric := promtest.MustFindMetric(t, mfs, name, l)
|
||||
if got := metric.GetCounter().GetValue(); got != exp {
|
||||
|
|
Loading…
Reference in New Issue