Batch up writes for monitor service
The monitor service was writing one big batch for all stats. If this batch was large, it causes some slower and more expensive write paths to be taken that incur a lot of memory allocations. This changes the monitor service to write in batches of up to 5000 points which should avoid the slower paths.pull/8711/head
parent
59e137e543
commit
c8ba179731
|
@ -440,17 +440,26 @@ func (m *Monitor) storeStatistics() {
|
|||
return
|
||||
}
|
||||
|
||||
points := make(models.Points, 0, len(stats))
|
||||
// Write all stats in batches
|
||||
batch := make(models.Points, 0, 5000)
|
||||
for _, s := range stats {
|
||||
pt, err := models.NewPoint(s.Name, models.NewTags(s.Tags), s.Values, now)
|
||||
if err != nil {
|
||||
m.Logger.Info(fmt.Sprintf("Dropping point %v: %v", s.Name, err))
|
||||
return
|
||||
}
|
||||
points = append(points, pt)
|
||||
batch = append(batch, pt)
|
||||
if len(batch) == cap(batch) {
|
||||
m.writePoints(batch)
|
||||
batch = batch[:0]
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
m.writePoints(points)
|
||||
// Write the last batch
|
||||
if len(batch) > 0 {
|
||||
m.writePoints(batch)
|
||||
}
|
||||
case <-m.done:
|
||||
m.Logger.Info(fmt.Sprintf("terminating storage of statistics"))
|
||||
return
|
||||
|
|
Loading…
Reference in New Issue