Batch up writes for monitor service

The monitor service was writing one big batch for all stats.  If this
batch was large, it causes some slower and more expensive write paths
to be taken that incur a lot of memory allocations.  This changes the
monitor service to write in batches of up to 5000 points which should
avoid the slower paths.
pull/8711/head
Jason Wilder 2017-08-16 13:09:25 -06:00
parent 59e137e543
commit c8ba179731
1 changed files with 12 additions and 3 deletions

View File

@ -440,17 +440,26 @@ func (m *Monitor) storeStatistics() {
return
}
points := make(models.Points, 0, len(stats))
// Write all stats in batches
batch := make(models.Points, 0, 5000)
for _, s := range stats {
pt, err := models.NewPoint(s.Name, models.NewTags(s.Tags), s.Values, now)
if err != nil {
m.Logger.Info(fmt.Sprintf("Dropping point %v: %v", s.Name, err))
return
}
points = append(points, pt)
batch = append(batch, pt)
if len(batch) == cap(batch) {
m.writePoints(batch)
batch = batch[:0]
}
}
m.writePoints(points)
// Write the last batch
if len(batch) > 0 {
m.writePoints(batch)
}
case <-m.done:
m.Logger.Info(fmt.Sprintf("terminating storage of statistics"))
return