diff --git a/models/points.go b/models/points.go index 537a051fbe..a1643c78ed 100644 --- a/models/points.go +++ b/models/points.go @@ -233,6 +233,9 @@ func ParsePointsString(buf string) ([]Point, error) { } // ParseKey returns the measurement name and tags from a point. +// +// NOTE: to minimize heap allocations, the returned Tags will refer to subslices of buf. +// This can have the unintended effect preventing buf from being garbage collected. func ParseKey(buf []byte) (string, Tags, error) { // Ignore the error because scanMeasurement returns "missing fields" which we ignore // when just parsing a key @@ -249,6 +252,9 @@ func ParseKey(buf []byte) (string, Tags, error) { // ParsePointsWithPrecision is similar to ParsePoints, but allows the // caller to provide a precision for time. +// +// NOTE: to minimize heap allocations, the returned Points will refer to subslices of buf. +// This can have the unintended effect preventing buf from being garbage collected. func ParsePointsWithPrecision(buf []byte, defaultTime time.Time, precision string) ([]Point, error) { points := make([]Point, 0, bytes.Count(buf, []byte{'\n'})+1) var ( @@ -1617,6 +1623,22 @@ type Tag struct { Value []byte } +// Clone returns a shallow copy of Tag. +// +// Tags associated with a Point created by ParsePointsWithPrecision will hold references to the byte slice that was parsed. +// Use Clone to create a Tag with new byte slices that do not refer to the argument to ParsePointsWithPrecision. +func (t Tag) Clone() Tag { + other := Tag{ + Key: make([]byte, len(t.Key)), + Value: make([]byte, len(t.Value)), + } + + copy(other.Key, t.Key) + copy(other.Value, t.Value) + + return other +} + // Tags represents a sorted list of tags. type Tags []Tag @@ -1633,6 +1655,19 @@ func NewTags(m map[string]string) Tags { return a } +// Clone returns a shallow copy of Tags. +// +// Tags associated with a Point created by ParsePointsWithPrecision will hold references to the byte slice that was parsed. +// Use Clone to create Tags with new byte slices that do not refer to the argument to ParsePointsWithPrecision. +func (a Tags) Clone() Tags { + others := make(Tags, len(a)) + for i := range a { + others[i] = a[i].Clone() + } + + return others +} + // Len implements sort.Interface. func (a Tags) Len() int { return len(a) } diff --git a/models/points_test.go b/models/points_test.go index 67f717a7aa..26977b19af 100644 --- a/models/points_test.go +++ b/models/points_test.go @@ -89,6 +89,38 @@ func testPoint_cube(t *testing.T, f func(p models.Point)) { } } +func TestTag_Clone(t *testing.T) { + tag := models.Tag{Key: []byte("key"), Value: []byte("value")} + + c := tag.Clone() + + if &c.Key == &tag.Key || !bytes.Equal(c.Key, tag.Key) { + t.Fatalf("key %s should have been a clone of %s", c.Key, tag.Key) + } + + if &c.Value == &tag.Value || !bytes.Equal(c.Value, tag.Value) { + t.Fatalf("value %s should have been a clone of %s", c.Value, tag.Value) + } +} + +func TestTags_Clone(t *testing.T) { + tags := models.NewTags(map[string]string{"k1": "v1", "k2": "v2", "k3": "v3"}) + + clone := tags.Clone() + + for i := range tags { + tag := tags[i] + c := clone[i] + if &c.Key == &tag.Key || !bytes.Equal(c.Key, tag.Key) { + t.Fatalf("key %s should have been a clone of %s", c.Key, tag.Key) + } + + if &c.Value == &tag.Value || !bytes.Equal(c.Value, tag.Value) { + t.Fatalf("value %s should have been a clone of %s", c.Value, tag.Value) + } + } +} + var p models.Point func BenchmarkNewPoint(b *testing.B) { diff --git a/tsdb/shard.go b/tsdb/shard.go index 418558cf49..9f9857ac3c 100644 --- a/tsdb/shard.go +++ b/tsdb/shard.go @@ -580,6 +580,10 @@ func (s *Shard) validateSeriesAndFields(points []models.Point) ([]models.Point, continue } + // If the tags were created by models.ParsePointsWithPrecision, + // they refer to subslices of the buffer containing line protocol. + // To ensure we don't refer to that buffer, preventing that buffer from being garbage collected, clone the tags. + tags = tags.Clone() ss = s.index.CreateSeriesIndexIfNotExists(p.Name(), NewSeries(string(p.Key()), tags)) atomic.AddInt64(&s.stats.SeriesCreated, 1) }