diff --git a/models/inline_fnv.go b/models/inline_fnv.go new file mode 100644 index 0000000000..47dcac10bf --- /dev/null +++ b/models/inline_fnv.go @@ -0,0 +1,32 @@ +package models // import "github.com/influxdata/platform/models" + +// from stdlib hash/fnv/fnv.go +const ( + prime64 = 1099511628211 + offset64 = 14695981039346656037 +) + +// InlineFNV64a is an alloc-free port of the standard library's fnv64a. +// See https://en.wikipedia.org/wiki/Fowler%E2%80%93Noll%E2%80%93Vo_hash_function. +type InlineFNV64a uint64 + +// NewInlineFNV64a returns a new instance of InlineFNV64a. +func NewInlineFNV64a() InlineFNV64a { + return offset64 +} + +// Write adds data to the running hash. +func (s *InlineFNV64a) Write(data []byte) (int, error) { + hash := uint64(*s) + for _, c := range data { + hash ^= uint64(c) + hash *= prime64 + } + *s = InlineFNV64a(hash) + return len(data), nil +} + +// Sum64 returns the uint64 of the current resulting hash. +func (s *InlineFNV64a) Sum64() uint64 { + return uint64(*s) +} diff --git a/models/inline_fnv_test.go b/models/inline_fnv_test.go new file mode 100644 index 0000000000..2dab254f9a --- /dev/null +++ b/models/inline_fnv_test.go @@ -0,0 +1,29 @@ +package models_test + +import ( + "hash/fnv" + "testing" + "testing/quick" + + "github.com/influxdata/platform/models" +) + +func TestInlineFNV64aEquivalenceFuzz(t *testing.T) { + f := func(data []byte) bool { + stdlibFNV := fnv.New64a() + stdlibFNV.Write(data) + want := stdlibFNV.Sum64() + + inlineFNV := models.NewInlineFNV64a() + inlineFNV.Write(data) + got := inlineFNV.Sum64() + + return want == got + } + cfg := &quick.Config{ + MaxCount: 10000, + } + if err := quick.Check(f, cfg); err != nil { + t.Fatal(err) + } +} diff --git a/models/inline_strconv_parse.go b/models/inline_strconv_parse.go new file mode 100644 index 0000000000..241dc59183 --- /dev/null +++ b/models/inline_strconv_parse.go @@ -0,0 +1,44 @@ +package models // import "github.com/influxdata/platform/models" + +import ( + "reflect" + "strconv" + "unsafe" +) + +// parseIntBytes is a zero-alloc wrapper around strconv.ParseInt. +func parseIntBytes(b []byte, base int, bitSize int) (i int64, err error) { + s := unsafeBytesToString(b) + return strconv.ParseInt(s, base, bitSize) +} + +// parseUintBytes is a zero-alloc wrapper around strconv.ParseUint. +func parseUintBytes(b []byte, base int, bitSize int) (i uint64, err error) { + s := unsafeBytesToString(b) + return strconv.ParseUint(s, base, bitSize) +} + +// parseFloatBytes is a zero-alloc wrapper around strconv.ParseFloat. +func parseFloatBytes(b []byte, bitSize int) (float64, error) { + s := unsafeBytesToString(b) + return strconv.ParseFloat(s, bitSize) +} + +// parseBoolBytes is a zero-alloc wrapper around strconv.ParseBool. +func parseBoolBytes(b []byte) (bool, error) { + return strconv.ParseBool(unsafeBytesToString(b)) +} + +// unsafeBytesToString converts a []byte to a string without a heap allocation. +// +// It is unsafe, and is intended to prepare input to short-lived functions +// that require strings. +func unsafeBytesToString(in []byte) string { + src := *(*reflect.SliceHeader)(unsafe.Pointer(&in)) + dst := reflect.StringHeader{ + Data: src.Data, + Len: src.Len, + } + s := *(*string)(unsafe.Pointer(&dst)) + return s +} diff --git a/models/inline_strconv_parse_test.go b/models/inline_strconv_parse_test.go new file mode 100644 index 0000000000..119f543d78 --- /dev/null +++ b/models/inline_strconv_parse_test.go @@ -0,0 +1,103 @@ +package models + +import ( + "strconv" + "testing" + "testing/quick" +) + +func TestParseIntBytesEquivalenceFuzz(t *testing.T) { + f := func(b []byte, base int, bitSize int) bool { + exp, expErr := strconv.ParseInt(string(b), base, bitSize) + got, gotErr := parseIntBytes(b, base, bitSize) + + return exp == got && checkErrs(expErr, gotErr) + } + + cfg := &quick.Config{ + MaxCount: 10000, + } + + if err := quick.Check(f, cfg); err != nil { + t.Fatal(err) + } +} + +func TestParseIntBytesValid64bitBase10EquivalenceFuzz(t *testing.T) { + buf := []byte{} + f := func(n int64) bool { + buf = strconv.AppendInt(buf[:0], n, 10) + + exp, expErr := strconv.ParseInt(string(buf), 10, 64) + got, gotErr := parseIntBytes(buf, 10, 64) + + return exp == got && checkErrs(expErr, gotErr) + } + + cfg := &quick.Config{ + MaxCount: 10000, + } + + if err := quick.Check(f, cfg); err != nil { + t.Fatal(err) + } +} + +func TestParseFloatBytesEquivalenceFuzz(t *testing.T) { + f := func(b []byte, bitSize int) bool { + exp, expErr := strconv.ParseFloat(string(b), bitSize) + got, gotErr := parseFloatBytes(b, bitSize) + + return exp == got && checkErrs(expErr, gotErr) + } + + cfg := &quick.Config{ + MaxCount: 10000, + } + + if err := quick.Check(f, cfg); err != nil { + t.Fatal(err) + } +} + +func TestParseFloatBytesValid64bitEquivalenceFuzz(t *testing.T) { + buf := []byte{} + f := func(n float64) bool { + buf = strconv.AppendFloat(buf[:0], n, 'f', -1, 64) + + exp, expErr := strconv.ParseFloat(string(buf), 64) + got, gotErr := parseFloatBytes(buf, 64) + + return exp == got && checkErrs(expErr, gotErr) + } + + cfg := &quick.Config{ + MaxCount: 10000, + } + + if err := quick.Check(f, cfg); err != nil { + t.Fatal(err) + } +} + +func TestParseBoolBytesEquivalence(t *testing.T) { + var buf []byte + for _, s := range []string{"1", "t", "T", "TRUE", "true", "True", "0", "f", "F", "FALSE", "false", "False", "fail", "TrUe", "FAlSE", "numbers", ""} { + buf = append(buf[:0], s...) + + exp, expErr := strconv.ParseBool(s) + got, gotErr := parseBoolBytes(buf) + + if got != exp || !checkErrs(expErr, gotErr) { + t.Errorf("Failed to parse boolean value %q correctly: wanted (%t, %v), got (%t, %v)", s, exp, expErr, got, gotErr) + } + } +} + +func checkErrs(a, b error) bool { + if (a == nil) != (b == nil) { + return false + } + + return a == nil || a.Error() == b.Error() +} diff --git a/models/points.go b/models/points.go new file mode 100644 index 0000000000..9095fdde8c --- /dev/null +++ b/models/points.go @@ -0,0 +1,2463 @@ +// Package models implements basic objects used throughout the TICK stack. +package models // import "github.com/influxdata/platform/models" + +import ( + "bytes" + "encoding/binary" + "errors" + "fmt" + "io" + "math" + "sort" + "strconv" + "strings" + "time" + "unicode" + "unicode/utf8" + + "github.com/influxdata/influxdb/pkg/escape" +) + +type escapeSet struct { + k [1]byte + esc [2]byte +} + +var ( + measurementEscapeCodes = [...]escapeSet{ + {k: [1]byte{','}, esc: [2]byte{'\\', ','}}, + {k: [1]byte{' '}, esc: [2]byte{'\\', ' '}}, + } + + tagEscapeCodes = [...]escapeSet{ + {k: [1]byte{','}, esc: [2]byte{'\\', ','}}, + {k: [1]byte{' '}, esc: [2]byte{'\\', ' '}}, + {k: [1]byte{'='}, esc: [2]byte{'\\', '='}}, + } + + // ErrPointMustHaveAField is returned when operating on a point that does not have any fields. + ErrPointMustHaveAField = errors.New("point without fields is unsupported") + + // ErrInvalidNumber is returned when a number is expected but not provided. + ErrInvalidNumber = errors.New("invalid number") + + // ErrInvalidPoint is returned when a point cannot be parsed correctly. + ErrInvalidPoint = errors.New("point is invalid") +) + +const ( + // MaxKeyLength is the largest allowed size of the combined measurement and tag keys. + MaxKeyLength = 65535 +) + +// enableUint64Support will enable uint64 support if set to true. +var enableUint64Support = false + +// EnableUintSupport manually enables uint support for the point parser. +// This function will be removed in the future and only exists for unit tests during the +// transition. +func EnableUintSupport() { + enableUint64Support = true +} + +// Point defines the values that will be written to the database. +type Point interface { + // Name return the measurement name for the point. + Name() []byte + + // SetName updates the measurement name for the point. + SetName(string) + + // Tags returns the tag set for the point. + Tags() Tags + + // ForEachTag iterates over each tag invoking fn. If fn return false, iteration stops. + ForEachTag(fn func(k, v []byte) bool) + + // AddTag adds or replaces a tag value for a point. + AddTag(key, value string) + + // SetTags replaces the tags for the point. + SetTags(tags Tags) + + // HasTag returns true if the tag exists for the point. + HasTag(tag []byte) bool + + // Fields returns the fields for the point. + Fields() (Fields, error) + + // Time return the timestamp for the point. + Time() time.Time + + // SetTime updates the timestamp for the point. + SetTime(t time.Time) + + // UnixNano returns the timestamp of the point as nanoseconds since Unix epoch. + UnixNano() int64 + + // HashID returns a non-cryptographic checksum of the point's key. + HashID() uint64 + + // Key returns the key (measurement joined with tags) of the point. + Key() []byte + + // String returns a string representation of the point. If there is a + // timestamp associated with the point then it will be specified with the default + // precision of nanoseconds. + String() string + + // MarshalBinary returns a binary representation of the point. + MarshalBinary() ([]byte, error) + + // PrecisionString returns a string representation of the point. If there + // is a timestamp associated with the point then it will be specified in the + // given unit. + PrecisionString(precision string) string + + // RoundedString returns a string representation of the point. If there + // is a timestamp associated with the point, then it will be rounded to the + // given duration. + RoundedString(d time.Duration) string + + // Split will attempt to return multiple points with the same timestamp whose + // string representations are no longer than size. Points with a single field or + // a point without a timestamp may exceed the requested size. + Split(size int) []Point + + // Round will round the timestamp of the point to the given duration. + Round(d time.Duration) + + // StringSize returns the length of the string that would be returned by String(). + StringSize() int + + // AppendString appends the result of String() to the provided buffer and returns + // the result, potentially reducing string allocations. + AppendString(buf []byte) []byte + + // FieldIterator retuns a FieldIterator that can be used to traverse the + // fields of a point without constructing the in-memory map. + FieldIterator() FieldIterator +} + +// FieldType represents the type of a field. +type FieldType int + +const ( + // Integer indicates the field's type is integer. + Integer FieldType = iota + + // Float indicates the field's type is float. + Float + + // Boolean indicates the field's type is boolean. + Boolean + + // String indicates the field's type is string. + String + + // Empty is used to indicate that there is no field. + Empty + + // Unsigned indicates the field's type is an unsigned integer. + Unsigned +) + +// FieldIterator provides a low-allocation interface to iterate through a point's fields. +type FieldIterator interface { + // Next indicates whether there any fields remaining. + Next() bool + + // FieldKey returns the key of the current field. + FieldKey() []byte + + // Type returns the FieldType of the current field. + Type() FieldType + + // StringValue returns the string value of the current field. + StringValue() string + + // IntegerValue returns the integer value of the current field. + IntegerValue() (int64, error) + + // UnsignedValue returns the unsigned value of the current field. + UnsignedValue() (uint64, error) + + // BooleanValue returns the boolean value of the current field. + BooleanValue() (bool, error) + + // FloatValue returns the float value of the current field. + FloatValue() (float64, error) + + // Reset resets the iterator to its initial state. + Reset() +} + +// Points represents a sortable list of points by timestamp. +type Points []Point + +// Len implements sort.Interface. +func (a Points) Len() int { return len(a) } + +// Less implements sort.Interface. +func (a Points) Less(i, j int) bool { return a[i].Time().Before(a[j].Time()) } + +// Swap implements sort.Interface. +func (a Points) Swap(i, j int) { a[i], a[j] = a[j], a[i] } + +// point is the default implementation of Point. +type point struct { + time time.Time + + // text encoding of measurement and tags + // key must always be stored sorted by tags, if the original line was not sorted, + // we need to resort it + key []byte + + // text encoding of field data + fields []byte + + // text encoding of timestamp + ts []byte + + // cached version of parsed fields from data + cachedFields map[string]interface{} + + // cached version of parsed name from key + cachedName string + + // cached version of parsed tags + cachedTags Tags + + it fieldIterator +} + +// type assertions +var ( + _ Point = (*point)(nil) + _ FieldIterator = (*point)(nil) +) + +const ( + // the number of characters for the largest possible int64 (9223372036854775807) + maxInt64Digits = 19 + + // the number of characters for the smallest possible int64 (-9223372036854775808) + minInt64Digits = 20 + + // the number of characters for the largest possible uint64 (18446744073709551615) + maxUint64Digits = 20 + + // the number of characters required for the largest float64 before a range check + // would occur during parsing + maxFloat64Digits = 25 + + // the number of characters required for smallest float64 before a range check occur + // would occur during parsing + minFloat64Digits = 27 +) + +// ParsePoints returns a slice of Points from a text representation of a point +// with each point separated by newlines. If any points fail to parse, a non-nil error +// will be returned in addition to the points that parsed successfully. +func ParsePoints(buf []byte) ([]Point, error) { + return ParsePointsWithPrecision(buf, time.Now().UTC(), "n") +} + +// ParsePointsString is identical to ParsePoints but accepts a string. +func ParsePointsString(buf string) ([]Point, error) { + return ParsePoints([]byte(buf)) +} + +// ParseKey returns the measurement name and tags from a point. +// +// NOTE: to minimize heap allocations, the returned Tags will refer to subslices of buf. +// This can have the unintended effect preventing buf from being garbage collected. +func ParseKey(buf []byte) (string, Tags) { + name, tags := ParseKeyBytes(buf) + return string(name), tags +} + +func ParseKeyBytes(buf []byte) ([]byte, Tags) { + return ParseKeyBytesWithTags(buf, nil) +} + +func ParseKeyBytesWithTags(buf []byte, tags Tags) ([]byte, Tags) { + // Ignore the error because scanMeasurement returns "missing fields" which we ignore + // when just parsing a key + state, i, _ := scanMeasurement(buf, 0) + + var name []byte + if state == tagKeyState { + tags = parseTags(buf, tags) + // scanMeasurement returns the location of the comma if there are tags, strip that off + name = buf[:i-1] + } else { + name = buf[:i] + } + return unescapeMeasurement(name), tags +} + +func ParseTags(buf []byte) Tags { + return parseTags(buf, nil) +} + +func ParseName(buf []byte) []byte { + // Ignore the error because scanMeasurement returns "missing fields" which we ignore + // when just parsing a key + state, i, _ := scanMeasurement(buf, 0) + var name []byte + if state == tagKeyState { + name = buf[:i-1] + } else { + name = buf[:i] + } + + return unescapeMeasurement(name) +} + +// ParsePointsWithPrecision is similar to ParsePoints, but allows the +// caller to provide a precision for time. +// +// NOTE: to minimize heap allocations, the returned Points will refer to subslices of buf. +// This can have the unintended effect preventing buf from being garbage collected. +func ParsePointsWithPrecision(buf []byte, defaultTime time.Time, precision string) ([]Point, error) { + points := make([]Point, 0, bytes.Count(buf, []byte{'\n'})+1) + var ( + pos int + block []byte + failed []string + ) + for pos < len(buf) { + pos, block = scanLine(buf, pos) + pos++ + + if len(block) == 0 { + continue + } + + // lines which start with '#' are comments + start := skipWhitespace(block, 0) + + // If line is all whitespace, just skip it + if start >= len(block) { + continue + } + + if block[start] == '#' { + continue + } + + // strip the newline if one is present + if block[len(block)-1] == '\n' { + block = block[:len(block)-1] + } + + pt, err := parsePoint(block[start:], defaultTime, precision) + if err != nil { + failed = append(failed, fmt.Sprintf("unable to parse '%s': %v", string(block[start:]), err)) + } else { + points = append(points, pt) + } + + } + if len(failed) > 0 { + return points, fmt.Errorf("%s", strings.Join(failed, "\n")) + } + return points, nil + +} + +func parsePoint(buf []byte, defaultTime time.Time, precision string) (Point, error) { + // scan the first block which is measurement[,tag1=value1,tag2=value=2...] + pos, key, err := scanKey(buf, 0) + if err != nil { + return nil, err + } + + // measurement name is required + if len(key) == 0 { + return nil, fmt.Errorf("missing measurement") + } + + if len(key) > MaxKeyLength { + return nil, fmt.Errorf("max key length exceeded: %v > %v", len(key), MaxKeyLength) + } + + // scan the second block is which is field1=value1[,field2=value2,...] + pos, fields, err := scanFields(buf, pos) + if err != nil { + return nil, err + } + + // at least one field is required + if len(fields) == 0 { + return nil, fmt.Errorf("missing fields") + } + + var maxKeyErr error + err = walkFields(fields, func(k, v []byte) bool { + if sz := seriesKeySize(key, k); sz > MaxKeyLength { + maxKeyErr = fmt.Errorf("max key length exceeded: %v > %v", sz, MaxKeyLength) + return false + } + return true + }) + + if err != nil { + return nil, err + } + + if maxKeyErr != nil { + return nil, maxKeyErr + } + + // scan the last block which is an optional integer timestamp + pos, ts, err := scanTime(buf, pos) + if err != nil { + return nil, err + } + + pt := &point{ + key: key, + fields: fields, + ts: ts, + } + + if len(ts) == 0 { + pt.time = defaultTime + pt.SetPrecision(precision) + } else { + ts, err := parseIntBytes(ts, 10, 64) + if err != nil { + return nil, err + } + pt.time, err = SafeCalcTime(ts, precision) + if err != nil { + return nil, err + } + + // Determine if there are illegal non-whitespace characters after the + // timestamp block. + for pos < len(buf) { + if buf[pos] != ' ' { + return nil, ErrInvalidPoint + } + pos++ + } + } + return pt, nil +} + +// GetPrecisionMultiplier will return a multiplier for the precision specified. +func GetPrecisionMultiplier(precision string) int64 { + d := time.Nanosecond + switch precision { + case "u": + d = time.Microsecond + case "ms": + d = time.Millisecond + case "s": + d = time.Second + case "m": + d = time.Minute + case "h": + d = time.Hour + } + return int64(d) +} + +// scanKey scans buf starting at i for the measurement and tag portion of the point. +// It returns the ending position and the byte slice of key within buf. If there +// are tags, they will be sorted if they are not already. +func scanKey(buf []byte, i int) (int, []byte, error) { + start := skipWhitespace(buf, i) + + i = start + + // Determines whether the tags are sort, assume they are + sorted := true + + // indices holds the indexes within buf of the start of each tag. For example, + // a buf of 'cpu,host=a,region=b,zone=c' would have indices slice of [4,11,20] + // which indicates that the first tag starts at buf[4], seconds at buf[11], and + // last at buf[20] + indices := make([]int, 100) + + // tracks how many commas we've seen so we know how many values are indices. + // Since indices is an arbitrarily large slice, + // we need to know how many values in the buffer are in use. + commas := 0 + + // First scan the Point's measurement. + state, i, err := scanMeasurement(buf, i) + if err != nil { + return i, buf[start:i], err + } + + // Optionally scan tags if needed. + if state == tagKeyState { + i, commas, indices, err = scanTags(buf, i, indices) + if err != nil { + return i, buf[start:i], err + } + } + + // Now we know where the key region is within buf, and the location of tags, we + // need to determine if duplicate tags exist and if the tags are sorted. This iterates + // over the list comparing each tag in the sequence with each other. + for j := 0; j < commas-1; j++ { + // get the left and right tags + _, left := scanTo(buf[indices[j]:indices[j+1]-1], 0, '=') + _, right := scanTo(buf[indices[j+1]:indices[j+2]-1], 0, '=') + + // If left is greater than right, the tags are not sorted. We do not have to + // continue because the short path no longer works. + // If the tags are equal, then there are duplicate tags, and we should abort. + // If the tags are not sorted, this pass may not find duplicate tags and we + // need to do a more exhaustive search later. + if cmp := bytes.Compare(left, right); cmp > 0 { + sorted = false + break + } else if cmp == 0 { + return i, buf[start:i], fmt.Errorf("duplicate tags") + } + } + + // If the tags are not sorted, then sort them. This sort is inline and + // uses the tag indices we created earlier. The actual buffer is not sorted, the + // indices are using the buffer for value comparison. After the indices are sorted, + // the buffer is reconstructed from the sorted indices. + if !sorted && commas > 0 { + // Get the measurement name for later + measurement := buf[start : indices[0]-1] + + // Sort the indices + indices := indices[:commas] + insertionSort(0, commas, buf, indices) + + // Create a new key using the measurement and sorted indices + b := make([]byte, len(buf[start:i])) + pos := copy(b, measurement) + for _, i := range indices { + b[pos] = ',' + pos++ + _, v := scanToSpaceOr(buf, i, ',') + pos += copy(b[pos:], v) + } + + // Check again for duplicate tags now that the tags are sorted. + for j := 0; j < commas-1; j++ { + // get the left and right tags + _, left := scanTo(buf[indices[j]:], 0, '=') + _, right := scanTo(buf[indices[j+1]:], 0, '=') + + // If the tags are equal, then there are duplicate tags, and we should abort. + // If the tags are not sorted, this pass may not find duplicate tags and we + // need to do a more exhaustive search later. + if bytes.Equal(left, right) { + return i, b, fmt.Errorf("duplicate tags") + } + } + + return i, b, nil + } + + return i, buf[start:i], nil +} + +// The following constants allow us to specify which state to move to +// next, when scanning sections of a Point. +const ( + tagKeyState = iota + tagValueState + fieldsState +) + +// scanMeasurement examines the measurement part of a Point, returning +// the next state to move to, and the current location in the buffer. +func scanMeasurement(buf []byte, i int) (int, int, error) { + // Check first byte of measurement, anything except a comma is fine. + // It can't be a space, since whitespace is stripped prior to this + // function call. + if i >= len(buf) || buf[i] == ',' { + return -1, i, fmt.Errorf("missing measurement") + } + + for { + i++ + if i >= len(buf) { + // cpu + return -1, i, fmt.Errorf("missing fields") + } + + if buf[i-1] == '\\' { + // Skip character (it's escaped). + continue + } + + // Unescaped comma; move onto scanning the tags. + if buf[i] == ',' { + return tagKeyState, i + 1, nil + } + + // Unescaped space; move onto scanning the fields. + if buf[i] == ' ' { + // cpu value=1.0 + return fieldsState, i, nil + } + } +} + +// scanTags examines all the tags in a Point, keeping track of and +// returning the updated indices slice, number of commas and location +// in buf where to start examining the Point fields. +func scanTags(buf []byte, i int, indices []int) (int, int, []int, error) { + var ( + err error + commas int + state = tagKeyState + ) + + for { + switch state { + case tagKeyState: + // Grow our indices slice if we have too many tags. + if commas >= len(indices) { + newIndics := make([]int, cap(indices)*2) + copy(newIndics, indices) + indices = newIndics + } + indices[commas] = i + commas++ + + i, err = scanTagsKey(buf, i) + state = tagValueState // tag value always follows a tag key + case tagValueState: + state, i, err = scanTagsValue(buf, i) + case fieldsState: + indices[commas] = i + 1 + return i, commas, indices, nil + } + + if err != nil { + return i, commas, indices, err + } + } +} + +// scanTagsKey scans each character in a tag key. +func scanTagsKey(buf []byte, i int) (int, error) { + // First character of the key. + if i >= len(buf) || buf[i] == ' ' || buf[i] == ',' || buf[i] == '=' { + // cpu,{'', ' ', ',', '='} + return i, fmt.Errorf("missing tag key") + } + + // Examine each character in the tag key until we hit an unescaped + // equals (the tag value), or we hit an error (i.e., unescaped + // space or comma). + for { + i++ + + // Either we reached the end of the buffer or we hit an + // unescaped comma or space. + if i >= len(buf) || + ((buf[i] == ' ' || buf[i] == ',') && buf[i-1] != '\\') { + // cpu,tag{'', ' ', ','} + return i, fmt.Errorf("missing tag value") + } + + if buf[i] == '=' && buf[i-1] != '\\' { + // cpu,tag= + return i + 1, nil + } + } +} + +// scanTagsValue scans each character in a tag value. +func scanTagsValue(buf []byte, i int) (int, int, error) { + // Tag value cannot be empty. + if i >= len(buf) || buf[i] == ',' || buf[i] == ' ' { + // cpu,tag={',', ' '} + return -1, i, fmt.Errorf("missing tag value") + } + + // Examine each character in the tag value until we hit an unescaped + // comma (move onto next tag key), an unescaped space (move onto + // fields), or we error out. + for { + i++ + if i >= len(buf) { + // cpu,tag=value + return -1, i, fmt.Errorf("missing fields") + } + + // An unescaped equals sign is an invalid tag value. + if buf[i] == '=' && buf[i-1] != '\\' { + // cpu,tag={'=', 'fo=o'} + return -1, i, fmt.Errorf("invalid tag format") + } + + if buf[i] == ',' && buf[i-1] != '\\' { + // cpu,tag=foo, + return tagKeyState, i + 1, nil + } + + // cpu,tag=foo value=1.0 + // cpu, tag=foo\= value=1.0 + if buf[i] == ' ' && buf[i-1] != '\\' { + return fieldsState, i, nil + } + } +} + +func insertionSort(l, r int, buf []byte, indices []int) { + for i := l + 1; i < r; i++ { + for j := i; j > l && less(buf, indices, j, j-1); j-- { + indices[j], indices[j-1] = indices[j-1], indices[j] + } + } +} + +func less(buf []byte, indices []int, i, j int) bool { + // This grabs the tag names for i & j, it ignores the values + _, a := scanTo(buf, indices[i], '=') + _, b := scanTo(buf, indices[j], '=') + return bytes.Compare(a, b) < 0 +} + +// scanFields scans buf, starting at i for the fields section of a point. It returns +// the ending position and the byte slice of the fields within buf. +func scanFields(buf []byte, i int) (int, []byte, error) { + start := skipWhitespace(buf, i) + i = start + quoted := false + + // tracks how many '=' we've seen + equals := 0 + + // tracks how many commas we've seen + commas := 0 + + for { + // reached the end of buf? + if i >= len(buf) { + break + } + + // escaped characters? + if buf[i] == '\\' && i+1 < len(buf) { + i += 2 + continue + } + + // If the value is quoted, scan until we get to the end quote + // Only quote values in the field value since quotes are not significant + // in the field key + if buf[i] == '"' && equals > commas { + quoted = !quoted + i++ + continue + } + + // If we see an =, ensure that there is at least on char before and after it + if buf[i] == '=' && !quoted { + equals++ + + // check for "... =123" but allow "a\ =123" + if buf[i-1] == ' ' && buf[i-2] != '\\' { + return i, buf[start:i], fmt.Errorf("missing field key") + } + + // check for "...a=123,=456" but allow "a=123,a\,=456" + if buf[i-1] == ',' && buf[i-2] != '\\' { + return i, buf[start:i], fmt.Errorf("missing field key") + } + + // check for "... value=" + if i+1 >= len(buf) { + return i, buf[start:i], fmt.Errorf("missing field value") + } + + // check for "... value=,value2=..." + if buf[i+1] == ',' || buf[i+1] == ' ' { + return i, buf[start:i], fmt.Errorf("missing field value") + } + + if isNumeric(buf[i+1]) || buf[i+1] == '-' || buf[i+1] == 'N' || buf[i+1] == 'n' { + var err error + i, err = scanNumber(buf, i+1) + if err != nil { + return i, buf[start:i], err + } + continue + } + // If next byte is not a double-quote, the value must be a boolean + if buf[i+1] != '"' { + var err error + i, _, err = scanBoolean(buf, i+1) + if err != nil { + return i, buf[start:i], err + } + continue + } + } + + if buf[i] == ',' && !quoted { + commas++ + } + + // reached end of block? + if buf[i] == ' ' && !quoted { + break + } + i++ + } + + if quoted { + return i, buf[start:i], fmt.Errorf("unbalanced quotes") + } + + // check that all field sections had key and values (e.g. prevent "a=1,b" + if equals == 0 || commas != equals-1 { + return i, buf[start:i], fmt.Errorf("invalid field format") + } + + return i, buf[start:i], nil +} + +// scanTime scans buf, starting at i for the time section of a point. It +// returns the ending position and the byte slice of the timestamp within buf +// and and error if the timestamp is not in the correct numeric format. +func scanTime(buf []byte, i int) (int, []byte, error) { + start := skipWhitespace(buf, i) + i = start + + for { + // reached the end of buf? + if i >= len(buf) { + break + } + + // Reached end of block or trailing whitespace? + if buf[i] == '\n' || buf[i] == ' ' { + break + } + + // Handle negative timestamps + if i == start && buf[i] == '-' { + i++ + continue + } + + // Timestamps should be integers, make sure they are so we don't need + // to actually parse the timestamp until needed. + if buf[i] < '0' || buf[i] > '9' { + return i, buf[start:i], fmt.Errorf("bad timestamp") + } + i++ + } + return i, buf[start:i], nil +} + +func isNumeric(b byte) bool { + return (b >= '0' && b <= '9') || b == '.' +} + +// scanNumber returns the end position within buf, start at i after +// scanning over buf for an integer, or float. It returns an +// error if a invalid number is scanned. +func scanNumber(buf []byte, i int) (int, error) { + start := i + var isInt, isUnsigned bool + + // Is negative number? + if i < len(buf) && buf[i] == '-' { + i++ + // There must be more characters now, as just '-' is illegal. + if i == len(buf) { + return i, ErrInvalidNumber + } + } + + // how many decimal points we've see + decimal := false + + // indicates the number is float in scientific notation + scientific := false + + for { + if i >= len(buf) { + break + } + + if buf[i] == ',' || buf[i] == ' ' { + break + } + + if buf[i] == 'i' && i > start && !(isInt || isUnsigned) { + isInt = true + i++ + continue + } else if buf[i] == 'u' && i > start && !(isInt || isUnsigned) { + isUnsigned = true + i++ + continue + } + + if buf[i] == '.' { + // Can't have more than 1 decimal (e.g. 1.1.1 should fail) + if decimal { + return i, ErrInvalidNumber + } + decimal = true + } + + // `e` is valid for floats but not as the first char + if i > start && (buf[i] == 'e' || buf[i] == 'E') { + scientific = true + i++ + continue + } + + // + and - are only valid at this point if they follow an e (scientific notation) + if (buf[i] == '+' || buf[i] == '-') && (buf[i-1] == 'e' || buf[i-1] == 'E') { + i++ + continue + } + + // NaN is an unsupported value + if i+2 < len(buf) && (buf[i] == 'N' || buf[i] == 'n') { + return i, ErrInvalidNumber + } + + if !isNumeric(buf[i]) { + return i, ErrInvalidNumber + } + i++ + } + + if (isInt || isUnsigned) && (decimal || scientific) { + return i, ErrInvalidNumber + } + + numericDigits := i - start + if isInt { + numericDigits-- + } + if decimal { + numericDigits-- + } + if buf[start] == '-' { + numericDigits-- + } + + if numericDigits == 0 { + return i, ErrInvalidNumber + } + + // It's more common that numbers will be within min/max range for their type but we need to prevent + // out or range numbers from being parsed successfully. This uses some simple heuristics to decide + // if we should parse the number to the actual type. It does not do it all the time because it incurs + // extra allocations and we end up converting the type again when writing points to disk. + if isInt { + // Make sure the last char is an 'i' for integers (e.g. 9i10 is not valid) + if buf[i-1] != 'i' { + return i, ErrInvalidNumber + } + // Parse the int to check bounds the number of digits could be larger than the max range + // We subtract 1 from the index to remove the `i` from our tests + if len(buf[start:i-1]) >= maxInt64Digits || len(buf[start:i-1]) >= minInt64Digits { + if _, err := parseIntBytes(buf[start:i-1], 10, 64); err != nil { + return i, fmt.Errorf("unable to parse integer %s: %s", buf[start:i-1], err) + } + } + } else if isUnsigned { + // Return an error if uint64 support has not been enabled. + if !enableUint64Support { + return i, ErrInvalidNumber + } + // Make sure the last char is a 'u' for unsigned + if buf[i-1] != 'u' { + return i, ErrInvalidNumber + } + // Make sure the first char is not a '-' for unsigned + if buf[start] == '-' { + return i, ErrInvalidNumber + } + // Parse the uint to check bounds the number of digits could be larger than the max range + // We subtract 1 from the index to remove the `u` from our tests + if len(buf[start:i-1]) >= maxUint64Digits { + if _, err := parseUintBytes(buf[start:i-1], 10, 64); err != nil { + return i, fmt.Errorf("unable to parse unsigned %s: %s", buf[start:i-1], err) + } + } + } else { + // Parse the float to check bounds if it's scientific or the number of digits could be larger than the max range + if scientific || len(buf[start:i]) >= maxFloat64Digits || len(buf[start:i]) >= minFloat64Digits { + if _, err := parseFloatBytes(buf[start:i], 10); err != nil { + return i, fmt.Errorf("invalid float") + } + } + } + + return i, nil +} + +// scanBoolean returns the end position within buf, start at i after +// scanning over buf for boolean. Valid values for a boolean are +// t, T, true, TRUE, f, F, false, FALSE. It returns an error if a invalid boolean +// is scanned. +func scanBoolean(buf []byte, i int) (int, []byte, error) { + start := i + + if i < len(buf) && (buf[i] != 't' && buf[i] != 'f' && buf[i] != 'T' && buf[i] != 'F') { + return i, buf[start:i], fmt.Errorf("invalid boolean") + } + + i++ + for { + if i >= len(buf) { + break + } + + if buf[i] == ',' || buf[i] == ' ' { + break + } + i++ + } + + // Single char bool (t, T, f, F) is ok + if i-start == 1 { + return i, buf[start:i], nil + } + + // length must be 4 for true or TRUE + if (buf[start] == 't' || buf[start] == 'T') && i-start != 4 { + return i, buf[start:i], fmt.Errorf("invalid boolean") + } + + // length must be 5 for false or FALSE + if (buf[start] == 'f' || buf[start] == 'F') && i-start != 5 { + return i, buf[start:i], fmt.Errorf("invalid boolean") + } + + // Otherwise + valid := false + switch buf[start] { + case 't': + valid = bytes.Equal(buf[start:i], []byte("true")) + case 'f': + valid = bytes.Equal(buf[start:i], []byte("false")) + case 'T': + valid = bytes.Equal(buf[start:i], []byte("TRUE")) || bytes.Equal(buf[start:i], []byte("True")) + case 'F': + valid = bytes.Equal(buf[start:i], []byte("FALSE")) || bytes.Equal(buf[start:i], []byte("False")) + } + + if !valid { + return i, buf[start:i], fmt.Errorf("invalid boolean") + } + + return i, buf[start:i], nil + +} + +// skipWhitespace returns the end position within buf, starting at i after +// scanning over spaces in tags. +func skipWhitespace(buf []byte, i int) int { + for i < len(buf) { + if buf[i] != ' ' && buf[i] != '\t' && buf[i] != 0 { + break + } + i++ + } + return i +} + +// scanLine returns the end position in buf and the next line found within +// buf. +func scanLine(buf []byte, i int) (int, []byte) { + start := i + quoted := false + fields := false + + // tracks how many '=' and commas we've seen + // this duplicates some of the functionality in scanFields + equals := 0 + commas := 0 + for { + // reached the end of buf? + if i >= len(buf) { + break + } + + // skip past escaped characters + if buf[i] == '\\' && i+2 < len(buf) { + i += 2 + continue + } + + if buf[i] == ' ' { + fields = true + } + + // If we see a double quote, makes sure it is not escaped + if fields { + if !quoted && buf[i] == '=' { + i++ + equals++ + continue + } else if !quoted && buf[i] == ',' { + i++ + commas++ + continue + } else if buf[i] == '"' && equals > commas { + i++ + quoted = !quoted + continue + } + } + + if buf[i] == '\n' && !quoted { + break + } + + i++ + } + + return i, buf[start:i] +} + +// scanTo returns the end position in buf and the next consecutive block +// of bytes, starting from i and ending with stop byte, where stop byte +// has not been escaped. +// +// If there are leading spaces, they are skipped. +func scanTo(buf []byte, i int, stop byte) (int, []byte) { + start := i + for { + // reached the end of buf? + if i >= len(buf) { + break + } + + // Reached unescaped stop value? + if buf[i] == stop && (i == 0 || buf[i-1] != '\\') { + break + } + i++ + } + + return i, buf[start:i] +} + +// scanTo returns the end position in buf and the next consecutive block +// of bytes, starting from i and ending with stop byte. If there are leading +// spaces, they are skipped. +func scanToSpaceOr(buf []byte, i int, stop byte) (int, []byte) { + start := i + if buf[i] == stop || buf[i] == ' ' { + return i, buf[start:i] + } + + for { + i++ + if buf[i-1] == '\\' { + continue + } + + // reached the end of buf? + if i >= len(buf) { + return i, buf[start:i] + } + + // reached end of block? + if buf[i] == stop || buf[i] == ' ' { + return i, buf[start:i] + } + } +} + +func scanTagValue(buf []byte, i int) (int, []byte) { + start := i + for { + if i >= len(buf) { + break + } + + if buf[i] == ',' && buf[i-1] != '\\' { + break + } + i++ + } + if i > len(buf) { + return i, nil + } + return i, buf[start:i] +} + +func scanFieldValue(buf []byte, i int) (int, []byte) { + start := i + quoted := false + for i < len(buf) { + // Only escape char for a field value is a double-quote and backslash + if buf[i] == '\\' && i+1 < len(buf) && (buf[i+1] == '"' || buf[i+1] == '\\') { + i += 2 + continue + } + + // Quoted value? (e.g. string) + if buf[i] == '"' { + i++ + quoted = !quoted + continue + } + + if buf[i] == ',' && !quoted { + break + } + i++ + } + return i, buf[start:i] +} + +func EscapeMeasurement(in []byte) []byte { + for _, c := range measurementEscapeCodes { + if bytes.IndexByte(in, c.k[0]) != -1 { + in = bytes.Replace(in, c.k[:], c.esc[:], -1) + } + } + return in +} + +func unescapeMeasurement(in []byte) []byte { + if bytes.IndexByte(in, '\\') == -1 { + return in + } + + for i := range measurementEscapeCodes { + c := &measurementEscapeCodes[i] + if bytes.IndexByte(in, c.k[0]) != -1 { + in = bytes.Replace(in, c.esc[:], c.k[:], -1) + } + } + return in +} + +func escapeTag(in []byte) []byte { + for i := range tagEscapeCodes { + c := &tagEscapeCodes[i] + if bytes.IndexByte(in, c.k[0]) != -1 { + in = bytes.Replace(in, c.k[:], c.esc[:], -1) + } + } + return in +} + +func unescapeTag(in []byte) []byte { + if bytes.IndexByte(in, '\\') == -1 { + return in + } + + for i := range tagEscapeCodes { + c := &tagEscapeCodes[i] + if bytes.IndexByte(in, c.k[0]) != -1 { + in = bytes.Replace(in, c.esc[:], c.k[:], -1) + } + } + return in +} + +// escapeStringFieldReplacer replaces double quotes and backslashes +// with the same character preceded by a backslash. +// As of Go 1.7 this benchmarked better in allocations and CPU time +// compared to iterating through a string byte-by-byte and appending to a new byte slice, +// calling strings.Replace twice, and better than (*Regex).ReplaceAllString. +var escapeStringFieldReplacer = strings.NewReplacer(`"`, `\"`, `\`, `\\`) + +// EscapeStringField returns a copy of in with any double quotes or +// backslashes with escaped values. +func EscapeStringField(in string) string { + return escapeStringFieldReplacer.Replace(in) +} + +// unescapeStringField returns a copy of in with any escaped double-quotes +// or backslashes unescaped. +func unescapeStringField(in string) string { + if strings.IndexByte(in, '\\') == -1 { + return in + } + + var out []byte + i := 0 + for { + if i >= len(in) { + break + } + // unescape backslashes + if in[i] == '\\' && i+1 < len(in) && in[i+1] == '\\' { + out = append(out, '\\') + i += 2 + continue + } + // unescape double-quotes + if in[i] == '\\' && i+1 < len(in) && in[i+1] == '"' { + out = append(out, '"') + i += 2 + continue + } + out = append(out, in[i]) + i++ + + } + return string(out) +} + +// NewPoint returns a new point with the given measurement name, tags, fields and timestamp. If +// an unsupported field value (NaN, or +/-Inf) or out of range time is passed, this function +// returns an error. +func NewPoint(name string, tags Tags, fields Fields, t time.Time) (Point, error) { + key, err := pointKey(name, tags, fields, t) + if err != nil { + return nil, err + } + + return &point{ + key: key, + time: t, + fields: fields.MarshalBinary(), + }, nil +} + +// pointKey checks some basic requirements for valid points, and returns the +// key, along with an possible error. +func pointKey(measurement string, tags Tags, fields Fields, t time.Time) ([]byte, error) { + if len(fields) == 0 { + return nil, ErrPointMustHaveAField + } + + if !t.IsZero() { + if err := CheckTime(t); err != nil { + return nil, err + } + } + + for key, value := range fields { + switch value := value.(type) { + case float64: + // Ensure the caller validates and handles invalid field values + if math.IsInf(value, 0) { + return nil, fmt.Errorf("+/-Inf is an unsupported value for field %s", key) + } + if math.IsNaN(value) { + return nil, fmt.Errorf("NaN is an unsupported value for field %s", key) + } + case float32: + // Ensure the caller validates and handles invalid field values + if math.IsInf(float64(value), 0) { + return nil, fmt.Errorf("+/-Inf is an unsupported value for field %s", key) + } + if math.IsNaN(float64(value)) { + return nil, fmt.Errorf("NaN is an unsupported value for field %s", key) + } + } + if len(key) == 0 { + return nil, fmt.Errorf("all fields must have non-empty names") + } + } + + key := MakeKey([]byte(measurement), tags) + for field := range fields { + sz := seriesKeySize(key, []byte(field)) + if sz > MaxKeyLength { + return nil, fmt.Errorf("max key length exceeded: %v > %v", sz, MaxKeyLength) + } + } + + return key, nil +} + +func seriesKeySize(key, field []byte) int { + // 4 is the length of the tsm1.fieldKeySeparator constant. It's inlined here to avoid a circular + // dependency. + return len(key) + 4 + len(field) +} + +// NewPointFromBytes returns a new Point from a marshalled Point. +func NewPointFromBytes(b []byte) (Point, error) { + p := &point{} + if err := p.UnmarshalBinary(b); err != nil { + return nil, err + } + + // This does some basic validation to ensure there are fields and they + // can be unmarshalled as well. + iter := p.FieldIterator() + var hasField bool + for iter.Next() { + if len(iter.FieldKey()) == 0 { + continue + } + hasField = true + switch iter.Type() { + case Float: + _, err := iter.FloatValue() + if err != nil { + return nil, fmt.Errorf("unable to unmarshal field %s: %s", string(iter.FieldKey()), err) + } + case Integer: + _, err := iter.IntegerValue() + if err != nil { + return nil, fmt.Errorf("unable to unmarshal field %s: %s", string(iter.FieldKey()), err) + } + case Unsigned: + _, err := iter.UnsignedValue() + if err != nil { + return nil, fmt.Errorf("unable to unmarshal field %s: %s", string(iter.FieldKey()), err) + } + case String: + // Skip since this won't return an error + case Boolean: + _, err := iter.BooleanValue() + if err != nil { + return nil, fmt.Errorf("unable to unmarshal field %s: %s", string(iter.FieldKey()), err) + } + } + } + + if !hasField { + return nil, ErrPointMustHaveAField + } + + return p, nil +} + +// MustNewPoint returns a new point with the given measurement name, tags, fields and timestamp. If +// an unsupported field value (NaN) is passed, this function panics. +func MustNewPoint(name string, tags Tags, fields Fields, time time.Time) Point { + pt, err := NewPoint(name, tags, fields, time) + if err != nil { + panic(err.Error()) + } + return pt +} + +// Key returns the key (measurement joined with tags) of the point. +func (p *point) Key() []byte { + return p.key +} + +func (p *point) name() []byte { + _, name := scanTo(p.key, 0, ',') + return name +} + +func (p *point) Name() []byte { + return escape.Unescape(p.name()) +} + +// SetName updates the measurement name for the point. +func (p *point) SetName(name string) { + p.cachedName = "" + p.key = MakeKey([]byte(name), p.Tags()) +} + +// Time return the timestamp for the point. +func (p *point) Time() time.Time { + return p.time +} + +// SetTime updates the timestamp for the point. +func (p *point) SetTime(t time.Time) { + p.time = t +} + +// Round will round the timestamp of the point to the given duration. +func (p *point) Round(d time.Duration) { + p.time = p.time.Round(d) +} + +// Tags returns the tag set for the point. +func (p *point) Tags() Tags { + if p.cachedTags != nil { + return p.cachedTags + } + p.cachedTags = parseTags(p.key, nil) + return p.cachedTags +} + +func (p *point) ForEachTag(fn func(k, v []byte) bool) { + walkTags(p.key, fn) +} + +func (p *point) HasTag(tag []byte) bool { + if len(p.key) == 0 { + return false + } + + var exists bool + walkTags(p.key, func(key, value []byte) bool { + if bytes.Equal(tag, key) { + exists = true + return false + } + return true + }) + + return exists +} + +func walkTags(buf []byte, fn func(key, value []byte) bool) { + if len(buf) == 0 { + return + } + + pos, name := scanTo(buf, 0, ',') + + // it's an empty key, so there are no tags + if len(name) == 0 { + return + } + + hasEscape := bytes.IndexByte(buf, '\\') != -1 + i := pos + 1 + var key, value []byte + for { + if i >= len(buf) { + break + } + i, key = scanTo(buf, i, '=') + i, value = scanTagValue(buf, i+1) + + if len(value) == 0 { + continue + } + + if hasEscape { + if !fn(unescapeTag(key), unescapeTag(value)) { + return + } + } else { + if !fn(key, value) { + return + } + } + + i++ + } +} + +// walkFields walks each field key and value via fn. If fn returns false, the iteration +// is stopped. The values are the raw byte slices and not the converted types. +func walkFields(buf []byte, fn func(key, value []byte) bool) error { + var i int + var key, val []byte + for len(buf) > 0 { + i, key = scanTo(buf, 0, '=') + if i > len(buf)-2 { + return fmt.Errorf("invalid value: field-key=%s", key) + } + buf = buf[i+1:] + i, val = scanFieldValue(buf, 0) + buf = buf[i:] + if !fn(key, val) { + break + } + + // slice off comma + if len(buf) > 0 { + buf = buf[1:] + } + } + return nil +} + +// parseTags parses buf into the provided destination tags, returning destination +// Tags, which may have a different length and capacity. +func parseTags(buf []byte, dst Tags) Tags { + if len(buf) == 0 { + return nil + } + + n := bytes.Count(buf, []byte(",")) + if cap(dst) < n { + dst = make(Tags, n) + } else { + dst = dst[:n] + } + + // Ensure existing behaviour when point has no tags and nil slice passed in. + if dst == nil { + dst = Tags{} + } + + // Series keys can contain escaped commas, therefore the number of commas + // in a series key only gives an estimation of the upper bound on the number + // of tags. + var i int + walkTags(buf, func(key, value []byte) bool { + dst[i].Key, dst[i].Value = key, value + i++ + return true + }) + return dst[:i] +} + +// MakeKey creates a key for a set of tags. +func MakeKey(name []byte, tags Tags) []byte { + return AppendMakeKey(nil, name, tags) +} + +// AppendMakeKey appends the key derived from name and tags to dst and returns the extended buffer. +func AppendMakeKey(dst []byte, name []byte, tags Tags) []byte { + // unescape the name and then re-escape it to avoid double escaping. + // The key should always be stored in escaped form. + dst = append(dst, EscapeMeasurement(unescapeMeasurement(name))...) + dst = tags.AppendHashKey(dst) + return dst +} + +// SetTags replaces the tags for the point. +func (p *point) SetTags(tags Tags) { + p.key = MakeKey(p.Name(), tags) + p.cachedTags = tags +} + +// AddTag adds or replaces a tag value for a point. +func (p *point) AddTag(key, value string) { + tags := p.Tags() + tags = append(tags, Tag{Key: []byte(key), Value: []byte(value)}) + sort.Sort(tags) + p.cachedTags = tags + p.key = MakeKey(p.Name(), tags) +} + +// Fields returns the fields for the point. +func (p *point) Fields() (Fields, error) { + if p.cachedFields != nil { + return p.cachedFields, nil + } + cf, err := p.unmarshalBinary() + if err != nil { + return nil, err + } + p.cachedFields = cf + return p.cachedFields, nil +} + +// SetPrecision will round a time to the specified precision. +func (p *point) SetPrecision(precision string) { + switch precision { + case "n": + case "u": + p.SetTime(p.Time().Truncate(time.Microsecond)) + case "ms": + p.SetTime(p.Time().Truncate(time.Millisecond)) + case "s": + p.SetTime(p.Time().Truncate(time.Second)) + case "m": + p.SetTime(p.Time().Truncate(time.Minute)) + case "h": + p.SetTime(p.Time().Truncate(time.Hour)) + } +} + +// String returns the string representation of the point. +func (p *point) String() string { + if p.Time().IsZero() { + return string(p.Key()) + " " + string(p.fields) + } + return string(p.Key()) + " " + string(p.fields) + " " + strconv.FormatInt(p.UnixNano(), 10) +} + +// AppendString appends the string representation of the point to buf. +func (p *point) AppendString(buf []byte) []byte { + buf = append(buf, p.key...) + buf = append(buf, ' ') + buf = append(buf, p.fields...) + + if !p.time.IsZero() { + buf = append(buf, ' ') + buf = strconv.AppendInt(buf, p.UnixNano(), 10) + } + + return buf +} + +// StringSize returns the length of the string that would be returned by String(). +func (p *point) StringSize() int { + size := len(p.key) + len(p.fields) + 1 + + if !p.time.IsZero() { + digits := 1 // even "0" has one digit + t := p.UnixNano() + if t < 0 { + // account for negative sign, then negate + digits++ + t = -t + } + for t > 9 { // already accounted for one digit + digits++ + t /= 10 + } + size += digits + 1 // digits and a space + } + + return size +} + +// MarshalBinary returns a binary representation of the point. +func (p *point) MarshalBinary() ([]byte, error) { + if len(p.fields) == 0 { + return nil, ErrPointMustHaveAField + } + + tb, err := p.time.MarshalBinary() + if err != nil { + return nil, err + } + + b := make([]byte, 8+len(p.key)+len(p.fields)+len(tb)) + i := 0 + + binary.BigEndian.PutUint32(b[i:], uint32(len(p.key))) + i += 4 + + i += copy(b[i:], p.key) + + binary.BigEndian.PutUint32(b[i:i+4], uint32(len(p.fields))) + i += 4 + + i += copy(b[i:], p.fields) + + copy(b[i:], tb) + return b, nil +} + +// UnmarshalBinary decodes a binary representation of the point into a point struct. +func (p *point) UnmarshalBinary(b []byte) error { + var n int + + // Read key length. + if len(b) < 4 { + return io.ErrShortBuffer + } + n, b = int(binary.BigEndian.Uint32(b[:4])), b[4:] + + // Read key. + if len(b) < n { + return io.ErrShortBuffer + } + p.key, b = b[:n], b[n:] + + // Read fields length. + if len(b) < 4 { + return io.ErrShortBuffer + } + n, b = int(binary.BigEndian.Uint32(b[:4])), b[4:] + + // Read fields. + if len(b) < n { + return io.ErrShortBuffer + } + p.fields, b = b[:n], b[n:] + + // Read timestamp. + return p.time.UnmarshalBinary(b) +} + +// PrecisionString returns a string representation of the point. If there +// is a timestamp associated with the point then it will be specified in the +// given unit. +func (p *point) PrecisionString(precision string) string { + if p.Time().IsZero() { + return fmt.Sprintf("%s %s", p.Key(), string(p.fields)) + } + return fmt.Sprintf("%s %s %d", p.Key(), string(p.fields), + p.UnixNano()/GetPrecisionMultiplier(precision)) +} + +// RoundedString returns a string representation of the point. If there +// is a timestamp associated with the point, then it will be rounded to the +// given duration. +func (p *point) RoundedString(d time.Duration) string { + if p.Time().IsZero() { + return fmt.Sprintf("%s %s", p.Key(), string(p.fields)) + } + return fmt.Sprintf("%s %s %d", p.Key(), string(p.fields), + p.time.Round(d).UnixNano()) +} + +func (p *point) unmarshalBinary() (Fields, error) { + iter := p.FieldIterator() + fields := make(Fields, 8) + for iter.Next() { + if len(iter.FieldKey()) == 0 { + continue + } + switch iter.Type() { + case Float: + v, err := iter.FloatValue() + if err != nil { + return nil, fmt.Errorf("unable to unmarshal field %s: %s", string(iter.FieldKey()), err) + } + fields[string(iter.FieldKey())] = v + case Integer: + v, err := iter.IntegerValue() + if err != nil { + return nil, fmt.Errorf("unable to unmarshal field %s: %s", string(iter.FieldKey()), err) + } + fields[string(iter.FieldKey())] = v + case Unsigned: + v, err := iter.UnsignedValue() + if err != nil { + return nil, fmt.Errorf("unable to unmarshal field %s: %s", string(iter.FieldKey()), err) + } + fields[string(iter.FieldKey())] = v + case String: + fields[string(iter.FieldKey())] = iter.StringValue() + case Boolean: + v, err := iter.BooleanValue() + if err != nil { + return nil, fmt.Errorf("unable to unmarshal field %s: %s", string(iter.FieldKey()), err) + } + fields[string(iter.FieldKey())] = v + } + } + return fields, nil +} + +// HashID returns a non-cryptographic checksum of the point's key. +func (p *point) HashID() uint64 { + h := NewInlineFNV64a() + h.Write(p.key) + sum := h.Sum64() + return sum +} + +// UnixNano returns the timestamp of the point as nanoseconds since Unix epoch. +func (p *point) UnixNano() int64 { + return p.Time().UnixNano() +} + +// Split will attempt to return multiple points with the same timestamp whose +// string representations are no longer than size. Points with a single field or +// a point without a timestamp may exceed the requested size. +func (p *point) Split(size int) []Point { + if p.time.IsZero() || p.StringSize() <= size { + return []Point{p} + } + + // key string, timestamp string, spaces + size -= len(p.key) + len(strconv.FormatInt(p.time.UnixNano(), 10)) + 2 + + var points []Point + var start, cur int + + for cur < len(p.fields) { + end, _ := scanTo(p.fields, cur, '=') + end, _ = scanFieldValue(p.fields, end+1) + + if cur > start && end-start > size { + points = append(points, &point{ + key: p.key, + time: p.time, + fields: p.fields[start : cur-1], + }) + start = cur + } + + cur = end + 1 + } + + points = append(points, &point{ + key: p.key, + time: p.time, + fields: p.fields[start:], + }) + + return points +} + +// Tag represents a single key/value tag pair. +type Tag struct { + Key []byte + Value []byte +} + +// NewTag returns a new Tag. +func NewTag(key, value []byte) Tag { + return Tag{ + Key: key, + Value: value, + } +} + +// Size returns the size of the key and value. +func (t Tag) Size() int { return len(t.Key) + len(t.Value) } + +// Clone returns a shallow copy of Tag. +// +// Tags associated with a Point created by ParsePointsWithPrecision will hold references to the byte slice that was parsed. +// Use Clone to create a Tag with new byte slices that do not refer to the argument to ParsePointsWithPrecision. +func (t Tag) Clone() Tag { + other := Tag{ + Key: make([]byte, len(t.Key)), + Value: make([]byte, len(t.Value)), + } + + copy(other.Key, t.Key) + copy(other.Value, t.Value) + + return other +} + +// String returns the string reprsentation of the tag. +func (t *Tag) String() string { + var buf bytes.Buffer + buf.WriteByte('{') + buf.WriteString(string(t.Key)) + buf.WriteByte(' ') + buf.WriteString(string(t.Value)) + buf.WriteByte('}') + return buf.String() +} + +// Tags represents a sorted list of tags. +type Tags []Tag + +// NewTags returns a new Tags from a map. +func NewTags(m map[string]string) Tags { + if len(m) == 0 { + return nil + } + a := make(Tags, 0, len(m)) + for k, v := range m { + a = append(a, NewTag([]byte(k), []byte(v))) + } + sort.Sort(a) + return a +} + +// Keys returns the list of keys for a tag set. +func (a Tags) Keys() []string { + if len(a) == 0 { + return nil + } + keys := make([]string, len(a)) + for i, tag := range a { + keys[i] = string(tag.Key) + } + return keys +} + +// Values returns the list of values for a tag set. +func (a Tags) Values() []string { + if len(a) == 0 { + return nil + } + values := make([]string, len(a)) + for i, tag := range a { + values[i] = string(tag.Value) + } + return values +} + +// String returns the string representation of the tags. +func (a Tags) String() string { + var buf bytes.Buffer + buf.WriteByte('[') + for i := range a { + buf.WriteString(a[i].String()) + if i < len(a)-1 { + buf.WriteByte(' ') + } + } + buf.WriteByte(']') + return buf.String() +} + +// Size returns the number of bytes needed to store all tags. Note, this is +// the number of bytes needed to store all keys and values and does not account +// for data structures or delimiters for example. +func (a Tags) Size() int { + var total int + for i := range a { + total += a[i].Size() + } + return total +} + +// Clone returns a copy of the slice where the elements are a result of calling `Clone` on the original elements +// +// Tags associated with a Point created by ParsePointsWithPrecision will hold references to the byte slice that was parsed. +// Use Clone to create Tags with new byte slices that do not refer to the argument to ParsePointsWithPrecision. +func (a Tags) Clone() Tags { + if len(a) == 0 { + return nil + } + + others := make(Tags, len(a)) + for i := range a { + others[i] = a[i].Clone() + } + + return others +} + +func (a Tags) Len() int { return len(a) } +func (a Tags) Less(i, j int) bool { return bytes.Compare(a[i].Key, a[j].Key) == -1 } +func (a Tags) Swap(i, j int) { a[i], a[j] = a[j], a[i] } + +// Equal returns true if a equals other. +func (a Tags) Equal(other Tags) bool { + if len(a) != len(other) { + return false + } + for i := range a { + if !bytes.Equal(a[i].Key, other[i].Key) || !bytes.Equal(a[i].Value, other[i].Value) { + return false + } + } + return true +} + +// CompareTags returns -1 if a < b, 1 if a > b, and 0 if a == b. +func CompareTags(a, b Tags) int { + // Compare each key & value until a mismatch. + for i := 0; i < len(a) && i < len(b); i++ { + if cmp := bytes.Compare(a[i].Key, b[i].Key); cmp != 0 { + return cmp + } + if cmp := bytes.Compare(a[i].Value, b[i].Value); cmp != 0 { + return cmp + } + } + + // If all tags are equal up to this point then return shorter tagset. + if len(a) < len(b) { + return -1 + } else if len(a) > len(b) { + return 1 + } + + // All tags are equal. + return 0 +} + +// Get returns the value for a key. +func (a Tags) Get(key []byte) []byte { + // OPTIMIZE: Use sort.Search if tagset is large. + + for _, t := range a { + if bytes.Equal(t.Key, key) { + return t.Value + } + } + return nil +} + +// GetString returns the string value for a string key. +func (a Tags) GetString(key string) string { + return string(a.Get([]byte(key))) +} + +// Set sets the value for a key. +func (a *Tags) Set(key, value []byte) { + for i, t := range *a { + if bytes.Equal(t.Key, key) { + (*a)[i].Value = value + return + } + } + *a = append(*a, Tag{Key: key, Value: value}) + sort.Sort(*a) +} + +// SetString sets the string value for a string key. +func (a *Tags) SetString(key, value string) { + a.Set([]byte(key), []byte(value)) +} + +// Delete removes a tag by key. +func (a *Tags) Delete(key []byte) { + for i, t := range *a { + if bytes.Equal(t.Key, key) { + copy((*a)[i:], (*a)[i+1:]) + (*a)[len(*a)-1] = Tag{} + *a = (*a)[:len(*a)-1] + return + } + } +} + +// Map returns a map representation of the tags. +func (a Tags) Map() map[string]string { + m := make(map[string]string, len(a)) + for _, t := range a { + m[string(t.Key)] = string(t.Value) + } + return m +} + +// Merge merges the tags combining the two. If both define a tag with the +// same key, the merged value overwrites the old value. +// A new map is returned. +func (a Tags) Merge(other map[string]string) Tags { + merged := make(map[string]string, len(a)+len(other)) + for _, t := range a { + merged[string(t.Key)] = string(t.Value) + } + for k, v := range other { + merged[k] = v + } + return NewTags(merged) +} + +// HashKey hashes all of a tag's keys. +func (a Tags) HashKey() []byte { + return a.AppendHashKey(nil) +} + +func (a Tags) needsEscape() bool { + for i := range a { + t := &a[i] + for j := range tagEscapeCodes { + c := &tagEscapeCodes[j] + if bytes.IndexByte(t.Key, c.k[0]) != -1 || bytes.IndexByte(t.Value, c.k[0]) != -1 { + return true + } + } + } + return false +} + +// AppendHashKey appends the result of hashing all of a tag's keys and values to dst and returns the extended buffer. +func (a Tags) AppendHashKey(dst []byte) []byte { + // Empty maps marshal to empty bytes. + if len(a) == 0 { + return dst + } + + // Type invariant: Tags are sorted + + sz := 0 + var escaped Tags + if a.needsEscape() { + var tmp [20]Tag + if len(a) < len(tmp) { + escaped = tmp[:len(a)] + } else { + escaped = make(Tags, len(a)) + } + + for i := range a { + t := &a[i] + nt := &escaped[i] + nt.Key = escapeTag(t.Key) + nt.Value = escapeTag(t.Value) + sz += len(nt.Key) + len(nt.Value) + } + } else { + sz = a.Size() + escaped = a + } + + sz += len(escaped) + (len(escaped) * 2) // separators + + // Generate marshaled bytes. + if cap(dst)-len(dst) < sz { + nd := make([]byte, len(dst), len(dst)+sz) + copy(nd, dst) + dst = nd + } + buf := dst[len(dst) : len(dst)+sz] + idx := 0 + for i := range escaped { + k := &escaped[i] + if len(k.Value) == 0 { + continue + } + buf[idx] = ',' + idx++ + copy(buf[idx:], k.Key) + idx += len(k.Key) + buf[idx] = '=' + idx++ + copy(buf[idx:], k.Value) + idx += len(k.Value) + } + return dst[:len(dst)+idx] +} + +// CopyTags returns a shallow copy of tags. +func CopyTags(a Tags) Tags { + other := make(Tags, len(a)) + copy(other, a) + return other +} + +// DeepCopyTags returns a deep copy of tags. +func DeepCopyTags(a Tags) Tags { + // Calculate size of keys/values in bytes. + var n int + for _, t := range a { + n += len(t.Key) + len(t.Value) + } + + // Build single allocation for all key/values. + buf := make([]byte, n) + + // Copy tags to new set. + other := make(Tags, len(a)) + for i, t := range a { + copy(buf, t.Key) + other[i].Key, buf = buf[:len(t.Key)], buf[len(t.Key):] + + copy(buf, t.Value) + other[i].Value, buf = buf[:len(t.Value)], buf[len(t.Value):] + } + + return other +} + +// Fields represents a mapping between a Point's field names and their +// values. +type Fields map[string]interface{} + +// FieldIterator retuns a FieldIterator that can be used to traverse the +// fields of a point without constructing the in-memory map. +func (p *point) FieldIterator() FieldIterator { + p.Reset() + return p +} + +type fieldIterator struct { + start, end int + key, keybuf []byte + valueBuf []byte + fieldType FieldType +} + +// Next indicates whether there any fields remaining. +func (p *point) Next() bool { + p.it.start = p.it.end + if p.it.start >= len(p.fields) { + return false + } + + p.it.end, p.it.key = scanTo(p.fields, p.it.start, '=') + if escape.IsEscaped(p.it.key) { + p.it.keybuf = escape.AppendUnescaped(p.it.keybuf[:0], p.it.key) + p.it.key = p.it.keybuf + } + + p.it.end, p.it.valueBuf = scanFieldValue(p.fields, p.it.end+1) + p.it.end++ + + if len(p.it.valueBuf) == 0 { + p.it.fieldType = Empty + return true + } + + c := p.it.valueBuf[0] + + if c == '"' { + p.it.fieldType = String + return true + } + + if strings.IndexByte(`0123456789-.nNiIu`, c) >= 0 { + if p.it.valueBuf[len(p.it.valueBuf)-1] == 'i' { + p.it.fieldType = Integer + p.it.valueBuf = p.it.valueBuf[:len(p.it.valueBuf)-1] + } else if p.it.valueBuf[len(p.it.valueBuf)-1] == 'u' { + p.it.fieldType = Unsigned + p.it.valueBuf = p.it.valueBuf[:len(p.it.valueBuf)-1] + } else { + p.it.fieldType = Float + } + return true + } + + // to keep the same behavior that currently exists, default to boolean + p.it.fieldType = Boolean + return true +} + +// FieldKey returns the key of the current field. +func (p *point) FieldKey() []byte { + return p.it.key +} + +// Type returns the FieldType of the current field. +func (p *point) Type() FieldType { + return p.it.fieldType +} + +// StringValue returns the string value of the current field. +func (p *point) StringValue() string { + return unescapeStringField(string(p.it.valueBuf[1 : len(p.it.valueBuf)-1])) +} + +// IntegerValue returns the integer value of the current field. +func (p *point) IntegerValue() (int64, error) { + n, err := parseIntBytes(p.it.valueBuf, 10, 64) + if err != nil { + return 0, fmt.Errorf("unable to parse integer value %q: %v", p.it.valueBuf, err) + } + return n, nil +} + +// UnsignedValue returns the unsigned value of the current field. +func (p *point) UnsignedValue() (uint64, error) { + n, err := parseUintBytes(p.it.valueBuf, 10, 64) + if err != nil { + return 0, fmt.Errorf("unable to parse unsigned value %q: %v", p.it.valueBuf, err) + } + return n, nil +} + +// BooleanValue returns the boolean value of the current field. +func (p *point) BooleanValue() (bool, error) { + b, err := parseBoolBytes(p.it.valueBuf) + if err != nil { + return false, fmt.Errorf("unable to parse bool value %q: %v", p.it.valueBuf, err) + } + return b, nil +} + +// FloatValue returns the float value of the current field. +func (p *point) FloatValue() (float64, error) { + f, err := parseFloatBytes(p.it.valueBuf, 64) + if err != nil { + return 0, fmt.Errorf("unable to parse floating point value %q: %v", p.it.valueBuf, err) + } + return f, nil +} + +// Reset resets the iterator to its initial state. +func (p *point) Reset() { + p.it.fieldType = Empty + p.it.key = nil + p.it.valueBuf = nil + p.it.start = 0 + p.it.end = 0 +} + +// MarshalBinary encodes all the fields to their proper type and returns the binary +// represenation +// NOTE: uint64 is specifically not supported due to potential overflow when we decode +// again later to an int64 +// NOTE2: uint is accepted, and may be 64 bits, and is for some reason accepted... +func (p Fields) MarshalBinary() []byte { + var b []byte + keys := make([]string, 0, len(p)) + + for k := range p { + keys = append(keys, k) + } + + // Not really necessary, can probably be removed. + sort.Strings(keys) + + for i, k := range keys { + if i > 0 { + b = append(b, ',') + } + b = appendField(b, k, p[k]) + } + + return b +} + +func appendField(b []byte, k string, v interface{}) []byte { + b = append(b, []byte(escape.String(k))...) + b = append(b, '=') + + // check popular types first + switch v := v.(type) { + case float64: + b = strconv.AppendFloat(b, v, 'f', -1, 64) + case int64: + b = strconv.AppendInt(b, v, 10) + b = append(b, 'i') + case string: + b = append(b, '"') + b = append(b, []byte(EscapeStringField(v))...) + b = append(b, '"') + case bool: + b = strconv.AppendBool(b, v) + case int32: + b = strconv.AppendInt(b, int64(v), 10) + b = append(b, 'i') + case int16: + b = strconv.AppendInt(b, int64(v), 10) + b = append(b, 'i') + case int8: + b = strconv.AppendInt(b, int64(v), 10) + b = append(b, 'i') + case int: + b = strconv.AppendInt(b, int64(v), 10) + b = append(b, 'i') + case uint64: + b = strconv.AppendUint(b, v, 10) + b = append(b, 'u') + case uint32: + b = strconv.AppendInt(b, int64(v), 10) + b = append(b, 'i') + case uint16: + b = strconv.AppendInt(b, int64(v), 10) + b = append(b, 'i') + case uint8: + b = strconv.AppendInt(b, int64(v), 10) + b = append(b, 'i') + case uint: + // TODO: 'uint' should be converted to writing as an unsigned integer, + // but we cannot since that would break backwards compatibility. + b = strconv.AppendInt(b, int64(v), 10) + b = append(b, 'i') + case float32: + b = strconv.AppendFloat(b, float64(v), 'f', -1, 32) + case []byte: + b = append(b, v...) + case nil: + // skip + default: + // Can't determine the type, so convert to string + b = append(b, '"') + b = append(b, []byte(EscapeStringField(fmt.Sprintf("%v", v)))...) + b = append(b, '"') + + } + + return b +} + +// ValidKeyToken returns true if the token used for measurement, tag key, or tag +// value is a valid unicode string and only contains printable, non-replacement characters. +func ValidKeyToken(s string) bool { + if !utf8.ValidString(s) { + return false + } + for _, r := range s { + if !unicode.IsPrint(r) || r == unicode.ReplacementChar { + return false + } + } + return true +} + +// ValidKeyTokens returns true if the measurement name and all tags are valid. +func ValidKeyTokens(name string, tags Tags) bool { + if !ValidKeyToken(name) { + return false + } + for _, tag := range tags { + if !ValidKeyToken(string(tag.Key)) || !ValidKeyToken(string(tag.Value)) { + return false + } + } + return true +} diff --git a/models/points_internal_test.go b/models/points_internal_test.go new file mode 100644 index 0000000000..3a760d37b0 --- /dev/null +++ b/models/points_internal_test.go @@ -0,0 +1,17 @@ +package models + +import "testing" + +func TestMarshalPointNoFields(t *testing.T) { + points, err := ParsePointsString("m,k=v f=0i") + if err != nil { + t.Fatal(err) + } + + // It's unclear how this can ever happen, but we've observed points that were marshalled without any fields. + points[0].(*point).fields = []byte{} + + if _, err := points[0].MarshalBinary(); err != ErrPointMustHaveAField { + t.Fatalf("got error %v, exp %v", err, ErrPointMustHaveAField) + } +} diff --git a/models/points_test.go b/models/points_test.go new file mode 100644 index 0000000000..2a232dcfcf --- /dev/null +++ b/models/points_test.go @@ -0,0 +1,2551 @@ +package models_test + +import ( + "bytes" + "fmt" + "io" + "math" + "math/rand" + "reflect" + "strconv" + "strings" + "testing" + "time" + + "github.com/influxdata/platform/models" +) + +var ( + tags = models.NewTags(map[string]string{"foo": "bar", "apple": "orange", "host": "serverA", "region": "uswest"}) + fields = models.Fields{ + "int64": int64(math.MaxInt64), + "uint32": uint32(math.MaxUint32), + "string": "String field that has a decent length, probably some log message or something", + "boolean": false, + "float64-tiny": float64(math.SmallestNonzeroFloat64), + "float64-large": float64(math.MaxFloat64), + } + maxFloat64 = strconv.FormatFloat(math.MaxFloat64, 'f', 1, 64) + minFloat64 = strconv.FormatFloat(-math.MaxFloat64, 'f', 1, 64) + + sink interface{} +) + +func TestMarshal(t *testing.T) { + got := tags.HashKey() + if exp := ",apple=orange,foo=bar,host=serverA,region=uswest"; string(got) != exp { + t.Log("got: ", string(got)) + t.Log("exp: ", exp) + t.Error("invalid match") + } +} + +func TestMarshalFields(t *testing.T) { + for _, tt := range []struct { + name string + value interface{} + exp string + }{ + { + name: "Float", + value: float64(2), + exp: `value=2`, + }, + { + name: "Integer", + value: int64(2), + exp: `value=2i`, + }, + { + name: "Unsigned", + value: uint64(2), + exp: `value=2u`, + }, + { + name: "String", + value: "foobar", + exp: `value="foobar"`, + }, + { + name: "Boolean", + value: true, + exp: `value=true`, + }, + } { + t.Run(tt.name, func(t *testing.T) { + fields := map[string]interface{}{"value": tt.value} + if have, want := models.Fields(fields).MarshalBinary(), []byte(tt.exp); !bytes.Equal(have, want) { + t.Fatalf("unexpected field output: %s != %s", string(have), string(want)) + } + }) + } +} + +func TestTags_HashKey(t *testing.T) { + tags = models.NewTags(map[string]string{"A FOO": "bar", "APPLE": "orange", "host": "serverA", "region": "uswest"}) + got := tags.HashKey() + if exp := ",A\\ FOO=bar,APPLE=orange,host=serverA,region=uswest"; string(got) != exp { + t.Log("got: ", string(got)) + t.Log("exp: ", exp) + t.Error("invalid match") + } +} + +func BenchmarkMarshal(b *testing.B) { + for i := 0; i < b.N; i++ { + tags.HashKey() + } +} +func TestPoint_Tags(t *testing.T) { + examples := []struct { + Point string + Tags models.Tags + }{ + {`cpu value=1`, models.Tags{}}, + {"cpu,tag0=v0 value=1", models.NewTags(map[string]string{"tag0": "v0"})}, + {"cpu,tag0=v0,tag1=v0 value=1", models.NewTags(map[string]string{"tag0": "v0", "tag1": "v0"})}, + {`cpu,tag0=v\ 0 value=1`, models.NewTags(map[string]string{"tag0": "v 0"})}, + {`cpu,tag0=v\ 0\ 1,tag1=v2 value=1`, models.NewTags(map[string]string{"tag0": "v 0 1", "tag1": "v2"})}, + {`cpu,tag0=\, value=1`, models.NewTags(map[string]string{"tag0": ","})}, + {`cpu,ta\ g0=\, value=1`, models.NewTags(map[string]string{"ta g0": ","})}, + {`cpu,tag0=\,1 value=1`, models.NewTags(map[string]string{"tag0": ",1"})}, + {`cpu,tag0=1\"\",t=k value=1`, models.NewTags(map[string]string{"tag0": `1\"\"`, "t": "k"})}, + } + + for _, example := range examples { + t.Run(example.Point, func(t *testing.T) { + pts, err := models.ParsePointsString(example.Point) + if err != nil { + t.Fatal(err) + } else if len(pts) != 1 { + t.Fatalf("parsed %d points, expected 1", len(pts)) + } + + // Repeat to test Tags() caching + for i := 0; i < 2; i++ { + tags := pts[0].Tags() + if !reflect.DeepEqual(tags, example.Tags) { + t.Fatalf("got %#v (%s), expected %#v", tags, tags.String(), example.Tags) + } + } + + }) + } +} + +func TestPoint_StringSize(t *testing.T) { + testPoint_cube(t, func(p models.Point) { + l := p.StringSize() + s := p.String() + + if l != len(s) { + t.Errorf("Incorrect length for %q. got %v, exp %v", s, l, len(s)) + } + }) + +} + +func TestPoint_AppendString(t *testing.T) { + testPoint_cube(t, func(p models.Point) { + got := p.AppendString(nil) + exp := []byte(p.String()) + + if !reflect.DeepEqual(exp, got) { + t.Errorf("AppendString() didn't match String(): got %v, exp %v", got, exp) + } + }) +} + +func testPoint_cube(t *testing.T, f func(p models.Point)) { + // heard of a table-driven test? let's make a cube-driven test... + tagList := []models.Tags{nil, {models.NewTag([]byte("foo"), []byte("bar"))}, tags} + fieldList := []models.Fields{{"a": 42.0}, {"a": 42, "b": "things"}, fields} + timeList := []time.Time{time.Time{}, time.Unix(0, 0), time.Unix(-34526, 0), time.Unix(231845, 0), time.Now()} + + for _, tagSet := range tagList { + for _, fieldSet := range fieldList { + for _, pointTime := range timeList { + p, err := models.NewPoint("test", tagSet, fieldSet, pointTime) + if err != nil { + t.Errorf("unexpected error creating point: %v", err) + continue + } + + f(p) + } + } + } +} + +func TestTag_Clone(t *testing.T) { + tag := models.NewTag([]byte("key"), []byte("value")) + + c := tag.Clone() + + if &c.Key == &tag.Key || !bytes.Equal(c.Key, tag.Key) { + t.Fatalf("key %s should have been a clone of %s", c.Key, tag.Key) + } + + if &c.Value == &tag.Value || !bytes.Equal(c.Value, tag.Value) { + t.Fatalf("value %s should have been a clone of %s", c.Value, tag.Value) + } +} + +func TestTags_Clone(t *testing.T) { + tags := models.NewTags(map[string]string{"k1": "v1", "k2": "v2", "k3": "v3"}) + + clone := tags.Clone() + + for i := range tags { + tag := tags[i] + c := clone[i] + if &c.Key == &tag.Key || !bytes.Equal(c.Key, tag.Key) { + t.Fatalf("key %s should have been a clone of %s", c.Key, tag.Key) + } + + if &c.Value == &tag.Value || !bytes.Equal(c.Value, tag.Value) { + t.Fatalf("value %s should have been a clone of %s", c.Value, tag.Value) + } + } +} + +var p models.Point + +func BenchmarkNewPoint(b *testing.B) { + ts := time.Now() + for i := 0; i < b.N; i++ { + p, _ = models.NewPoint("measurement", tags, fields, ts) + } +} + +func BenchmarkNewPointFromBinary(b *testing.B) { + pts, err := models.ParsePointsString("cpu value1=1.0,value2=1.0,value3=3.0,value4=4,value5=\"five\" 1000000000") + if err != nil { + b.Fatalf("unexpected error ParsePointsString: %v", err) + } + + bytes, err := pts[0].MarshalBinary() + if err != nil { + b.Fatalf("unexpected error MarshalBinary: %v", err) + } + + for i := 0; i < b.N; i++ { + _, err := models.NewPointFromBytes(bytes) + if err != nil { + b.Fatalf("unexpected error NewPointsFromBytes: %v", err) + } + } +} + +func BenchmarkParsePointNoTags5000(b *testing.B) { + var batch [5000]string + for i := 0; i < len(batch); i++ { + batch[i] = `cpu value=1i 1000000000` + } + lines := strings.Join(batch[:], "\n") + b.ResetTimer() + for i := 0; i < b.N; i++ { + models.ParsePoints([]byte(lines)) + b.SetBytes(int64(len(lines))) + } +} + +func BenchmarkParsePointNoTags(b *testing.B) { + line := `cpu value=1i 1000000000` + for i := 0; i < b.N; i++ { + models.ParsePoints([]byte(line)) + b.SetBytes(int64(len(line))) + } +} + +func BenchmarkParsePointWithPrecisionN(b *testing.B) { + line := `cpu value=1i 1000000000` + defaultTime := time.Now().UTC() + for i := 0; i < b.N; i++ { + models.ParsePointsWithPrecision([]byte(line), defaultTime, "n") + b.SetBytes(int64(len(line))) + } +} + +func BenchmarkParsePointWithPrecisionU(b *testing.B) { + line := `cpu value=1i 1000000000` + defaultTime := time.Now().UTC() + for i := 0; i < b.N; i++ { + models.ParsePointsWithPrecision([]byte(line), defaultTime, "u") + b.SetBytes(int64(len(line))) + } +} + +func BenchmarkParsePointsTagsSorted2(b *testing.B) { + line := `cpu,host=serverA,region=us-west value=1i 1000000000` + for i := 0; i < b.N; i++ { + models.ParsePoints([]byte(line)) + b.SetBytes(int64(len(line))) + } +} + +func BenchmarkParsePointsTagsSorted5(b *testing.B) { + line := `cpu,env=prod,host=serverA,region=us-west,target=servers,zone=1c value=1i 1000000000` + for i := 0; i < b.N; i++ { + models.ParsePoints([]byte(line)) + b.SetBytes(int64(len(line))) + } +} + +func BenchmarkParsePointsTagsSorted10(b *testing.B) { + line := `cpu,env=prod,host=serverA,region=us-west,tag1=value1,tag2=value2,tag3=value3,tag4=value4,tag5=value5,target=servers,zone=1c value=1i 1000000000` + for i := 0; i < b.N; i++ { + models.ParsePoints([]byte(line)) + b.SetBytes(int64(len(line))) + } +} + +func BenchmarkParsePointsTagsUnSorted2(b *testing.B) { + line := `cpu,region=us-west,host=serverA value=1i 1000000000` + for i := 0; i < b.N; i++ { + pt, _ := models.ParsePoints([]byte(line)) + b.SetBytes(int64(len(line))) + pt[0].Key() + } +} + +func BenchmarkParsePointsTagsUnSorted5(b *testing.B) { + line := `cpu,region=us-west,host=serverA,env=prod,target=servers,zone=1c value=1i 1000000000` + for i := 0; i < b.N; i++ { + pt, _ := models.ParsePoints([]byte(line)) + b.SetBytes(int64(len(line))) + pt[0].Key() + } +} + +func BenchmarkParsePointsTagsUnSorted10(b *testing.B) { + line := `cpu,region=us-west,host=serverA,env=prod,target=servers,zone=1c,tag1=value1,tag2=value2,tag3=value3,tag4=value4,tag5=value5 value=1i 1000000000` + for i := 0; i < b.N; i++ { + pt, _ := models.ParsePoints([]byte(line)) + b.SetBytes(int64(len(line))) + pt[0].Key() + } +} + +func BenchmarkParseKey(b *testing.B) { + line := `cpu,region=us-west,host=serverA,env=prod,target=servers,zone=1c,tag1=value1,tag2=value2,tag3=value3,tag4=value4,tag5=value5` + for i := 0; i < b.N; i++ { + models.ParseKey([]byte(line)) + } +} + +// TestPoint wraps a models.Point but also makes available the raw +// arguments to the Point. +// +// This is useful for ensuring that comparisons between results of +// operations on Points match the expected input data to the Point, +// since models.Point does not expose the raw input data (e.g., tags) +// via its API. +type TestPoint struct { + RawFields models.Fields + RawTags models.Tags + RawTime time.Time + models.Point +} + +// NewTestPoint returns a new TestPoint. +// +// NewTestPoint panics if it is not a valid models.Point. +func NewTestPoint(name string, tags models.Tags, fields models.Fields, time time.Time) TestPoint { + return TestPoint{ + RawTags: tags, + RawFields: fields, + RawTime: time, + Point: models.MustNewPoint(name, tags, fields, time), + } +} + +func test(t *testing.T, line string, point TestPoint) { + pts, err := models.ParsePointsWithPrecision([]byte(line), time.Unix(0, 0), "n") + if err != nil { + t.Fatalf(`ParsePoints("%s") mismatch. got %v, exp nil`, line, err) + } + + if exp := 1; len(pts) != exp { + t.Fatalf(`ParsePoints("%s") len mismatch. got %d, exp %d`, line, len(pts), exp) + } + + if exp := point.Key(); !bytes.Equal(pts[0].Key(), exp) { + t.Errorf("ParsePoints(\"%s\") key mismatch.\ngot %v\nexp %v", line, string(pts[0].Key()), string(exp)) + } + + if exp := len(point.Tags()); len(pts[0].Tags()) != exp { + t.Errorf(`ParsePoints("%s") tags mismatch. got %v, exp %v`, line, pts[0].Tags(), exp) + } + + for _, tag := range pts[0].Tags() { + if !bytes.Equal(tag.Value, point.RawTags.Get(tag.Key)) { + t.Errorf(`ParsePoints("%s") tags mismatch. got %s, exp %s`, line, tag.Value, point.RawTags.Get(tag.Key)) + } + } + + for name, value := range point.RawFields { + fields, err := pts[0].Fields() + if err != nil { + t.Fatal(err) + } + val := fields[name] + expfval, ok := val.(float64) + + if ok && math.IsNaN(expfval) { + gotfval, ok := value.(float64) + if ok && !math.IsNaN(gotfval) { + t.Errorf(`ParsePoints("%s") field '%s' mismatch. exp NaN`, line, name) + } + } + if !reflect.DeepEqual(val, value) { + t.Errorf(`ParsePoints("%s") field '%s' mismatch. got %[3]v (%[3]T), exp %[4]v (%[4]T)`, line, name, val, value) + } + } + + if !pts[0].Time().Equal(point.Time()) { + t.Errorf(`ParsePoints("%s") time mismatch. got %v, exp %v`, line, pts[0].Time(), point.Time()) + } + + if !strings.HasPrefix(pts[0].String(), line) { + t.Errorf("ParsePoints string mismatch.\ngot: %v\nexp: %v", pts[0].String(), line) + } +} + +func TestParsePointNoValue(t *testing.T) { + pts, err := models.ParsePointsString("") + if err != nil { + t.Errorf(`ParsePoints("%s") mismatch. got %v, exp nil`, "", err) + } + + if exp := 0; len(pts) != exp { + t.Errorf(`ParsePoints("%s") len mismatch. got %v, exp %v`, "", len(pts), exp) + } +} + +func TestParsePointWhitespaceValue(t *testing.T) { + pts, err := models.ParsePointsString(" ") + if err != nil { + t.Errorf(`ParsePoints("%s") mismatch. got %v, exp nil`, "", err) + } + + if exp := 0; len(pts) != exp { + t.Errorf(`ParsePoints("%s") len mismatch. got %v, exp %v`, "", len(pts), exp) + } +} + +func TestParsePointNoFields(t *testing.T) { + expectedSuffix := "missing fields" + examples := []string{ + "cpu_load_short,host=server01,region=us-west", + "cpu", + "cpu,host==", + "=", + } + + for i, example := range examples { + _, err := models.ParsePointsString(example) + if err == nil { + t.Errorf(`[Example %d] ParsePoints("%s") mismatch. got nil, exp error`, i, example) + } else if !strings.HasSuffix(err.Error(), expectedSuffix) { + t.Errorf(`[Example %d] ParsePoints("%s") mismatch. got %q, exp suffix %q`, i, example, err, expectedSuffix) + } + } +} + +func TestParsePointNoTimestamp(t *testing.T) { + test(t, "cpu value=1", NewTestPoint("cpu", nil, models.Fields{"value": 1.0}, time.Unix(0, 0))) +} + +func TestParsePointMissingQuote(t *testing.T) { + expectedSuffix := "unbalanced quotes" + examples := []string{ + `cpu,host=serverA value="test`, + `cpu,host=serverA value="test""`, + } + + for i, example := range examples { + _, err := models.ParsePointsString(example) + if err == nil { + t.Errorf(`[Example %d] ParsePoints("%s") mismatch. got nil, exp error`, i, example) + } else if !strings.HasSuffix(err.Error(), expectedSuffix) { + t.Errorf(`[Example %d] ParsePoints("%s") mismatch. got %q, exp suffix %q`, i, example, err, expectedSuffix) + } + } +} + +func TestParsePointMissingTagKey(t *testing.T) { + expectedSuffix := "missing tag key" + examples := []string{ + `cpu, value=1`, + `cpu,`, + `cpu,,,`, + `cpu,host=serverA,=us-east value=1i`, + `cpu,host=serverAa\,,=us-east value=1i`, + `cpu,host=serverA\,,=us-east value=1i`, + `cpu, =serverA value=1i`, + } + + for i, example := range examples { + _, err := models.ParsePointsString(example) + if err == nil { + t.Errorf(`[Example %d] ParsePoints("%s") mismatch. got nil, exp error`, i, example) + } else if !strings.HasSuffix(err.Error(), expectedSuffix) { + t.Errorf(`[Example %d] ParsePoints("%s") mismatch. got %q, exp suffix %q`, i, example, err, expectedSuffix) + } + } + + _, err := models.ParsePointsString(`cpu,host=serverA,\ =us-east value=1i`) + if err != nil { + t.Errorf(`ParsePoints("%s") mismatch. got %v, exp nil`, `cpu,host=serverA,\ =us-east value=1i`, err) + } +} + +func TestParsePointMissingTagValue(t *testing.T) { + expectedSuffix := "missing tag value" + examples := []string{ + `cpu,host`, + `cpu,host,`, + `cpu,host=`, + `cpu,host value=1i`, + `cpu,host=serverA,region value=1i`, + `cpu,host=serverA,region= value=1i`, + `cpu,host=serverA,region=,zone=us-west value=1i`, + } + + for i, example := range examples { + _, err := models.ParsePointsString(example) + if err == nil { + t.Errorf(`[Example %d] ParsePoints("%s") mismatch. got nil, exp error`, i, example) + } else if !strings.HasSuffix(err.Error(), expectedSuffix) { + t.Errorf(`[Example %d] ParsePoints("%s") mismatch. got %q, exp suffix %q`, i, example, err, expectedSuffix) + } + } +} + +func TestParsePointInvalidTagFormat(t *testing.T) { + expectedSuffix := "invalid tag format" + examples := []string{ + `cpu,host=f=o,`, + `cpu,host=f\==o,`, + } + + for i, example := range examples { + _, err := models.ParsePointsString(example) + if err == nil { + t.Errorf(`[Example %d] ParsePoints("%s") mismatch. got nil, exp error`, i, example) + } else if !strings.HasSuffix(err.Error(), expectedSuffix) { + t.Errorf(`[Example %d] ParsePoints("%s") mismatch. got %q, exp suffix %q`, i, example, err, expectedSuffix) + } + } +} + +func TestParsePointMissingFieldName(t *testing.T) { + _, err := models.ParsePointsString(`cpu,host=serverA,region=us-west =`) + if err == nil { + t.Errorf(`ParsePoints("%s") mismatch. got nil, exp error`, `cpu,host=serverA,region=us-west =`) + } + + _, err = models.ParsePointsString(`cpu,host=serverA,region=us-west =123i`) + if err == nil { + t.Errorf(`ParsePoints("%s") mismatch. got nil, exp error`, `cpu,host=serverA,region=us-west =123i`) + } + + _, err = models.ParsePointsString(`cpu,host=serverA,region=us-west a\ =123i`) + if err != nil { + t.Errorf(`ParsePoints("%s") mismatch. got nil, exp error`, `cpu,host=serverA,region=us-west a\ =123i`) + } + _, err = models.ParsePointsString(`cpu,host=serverA,region=us-west value=123i,=456i`) + if err == nil { + t.Errorf(`ParsePoints("%s") mismatch. got nil, exp error`, `cpu,host=serverA,region=us-west value=123i,=456i`) + } +} + +func TestParsePointMissingFieldValue(t *testing.T) { + _, err := models.ParsePointsString(`cpu,host=serverA,region=us-west value=`) + if err == nil { + t.Errorf(`ParsePoints("%s") mismatch. got nil, exp error`, `cpu,host=serverA,region=us-west value=`) + } + + _, err = models.ParsePointsString(`cpu,host=serverA,region=us-west value= 1000000000i`) + if err == nil { + t.Errorf(`ParsePoints("%s") mismatch. got nil, exp error`, `cpu,host=serverA,region=us-west value= 1000000000i`) + } + + _, err = models.ParsePointsString(`cpu,host=serverA,region=us-west value=,value2=1i`) + if err == nil { + t.Errorf(`ParsePoints("%s") mismatch. got nil, exp error`, `cpu,host=serverA,region=us-west value=,value2=1i`) + } + + _, err = models.ParsePointsString(`cpu,host=server01,region=us-west 1434055562000000000i`) + if err == nil { + t.Errorf(`ParsePoints("%s") mismatch. got nil, exp error`, `cpu,host=server01,region=us-west 1434055562000000000i`) + } + + _, err = models.ParsePointsString(`cpu,host=server01,region=us-west value=1i,b`) + if err == nil { + t.Errorf(`ParsePoints("%s") mismatch. got nil, exp error`, `cpu,host=server01,region=us-west value=1i,b`) + } + + _, err = models.ParsePointsString(`m f="blah"=123,r 1531703600000000000`) + if err == nil { + t.Errorf(`ParsePoints("%s") mismatch. got nil, exp error`, `m f="blah"=123,r 1531703600000000000`) + } +} + +func TestParsePointBadNumber(t *testing.T) { + for _, tt := range []string{ + "cpu v=- ", + "cpu v=-i ", + "cpu v=-. ", + "cpu v=. ", + "cpu v=1.0i ", + "cpu v=1ii ", + "cpu v=1a ", + "cpu v=-e-e-e ", + "cpu v=42+3 ", + "cpu v= ", + "cpu v=-123u", + } { + _, err := models.ParsePointsString(tt) + if err == nil { + t.Errorf("Point %q should be invalid", tt) + } + } +} + +func TestParsePointMaxInt64(t *testing.T) { + // out of range + _, err := models.ParsePointsString(`cpu,host=serverA,region=us-west value=9223372036854775808i`) + exp := `unable to parse 'cpu,host=serverA,region=us-west value=9223372036854775808i': unable to parse integer 9223372036854775808: strconv.ParseInt: parsing "9223372036854775808": value out of range` + if err == nil || (err != nil && err.Error() != exp) { + t.Fatalf("Error mismatch:\nexp: %s\ngot: %v", exp, err) + } + + // max int + p, err := models.ParsePointsString(`cpu,host=serverA,region=us-west value=9223372036854775807i`) + if err != nil { + t.Fatalf(`ParsePoints("%s") mismatch. got %v, exp nil`, `cpu,host=serverA,region=us-west value=9223372036854775807i`, err) + } + fields, err := p[0].Fields() + if err != nil { + t.Fatal(err) + } + if exp, got := int64(9223372036854775807), fields["value"].(int64); exp != got { + t.Fatalf("ParsePoints Value mismatch. \nexp: %v\ngot: %v", exp, got) + } + + // leading zeros + _, err = models.ParsePointsString(`cpu,host=serverA,region=us-west value=0009223372036854775807i`) + if err != nil { + t.Fatalf(`ParsePoints("%s") mismatch. got %v, exp nil`, `cpu,host=serverA,region=us-west value=0009223372036854775807i`, err) + } +} + +func TestParsePointMinInt64(t *testing.T) { + // out of range + _, err := models.ParsePointsString(`cpu,host=serverA,region=us-west value=-9223372036854775809i`) + if err == nil { + t.Errorf(`ParsePoints("%s") mismatch. got nil, exp error`, `cpu,host=serverA,region=us-west value=-9223372036854775809i`) + } + + // min int + p, err := models.ParsePointsString(`cpu,host=serverA,region=us-west value=-9223372036854775808i`) + if err != nil { + t.Errorf(`ParsePoints("%s") mismatch. got %v, exp nil`, `cpu,host=serverA,region=us-west value=-9223372036854775808i`, err) + } + fields, err := p[0].Fields() + if err != nil { + t.Fatal(err) + } + if exp, got := int64(-9223372036854775808), fields["value"].(int64); exp != got { + t.Fatalf("ParsePoints Value mismatch. \nexp: %v\ngot: %v", exp, got) + } + + // leading zeros + _, err = models.ParsePointsString(`cpu,host=serverA,region=us-west value=-0009223372036854775808i`) + if err != nil { + t.Errorf(`ParsePoints("%s") mismatch. got %v, exp nil`, `cpu,host=serverA,region=us-west value=-0009223372036854775808i`, err) + } +} + +func TestParsePointMaxFloat64(t *testing.T) { + // out of range + _, err := models.ParsePointsString(fmt.Sprintf(`cpu,host=serverA,region=us-west value=%s`, "1"+string(maxFloat64))) + if err == nil { + t.Errorf(`ParsePoints("%s") mismatch. got nil, exp error`, `cpu,host=serverA,region=us-west value=...`) + } + + // max float + p, err := models.ParsePointsString(fmt.Sprintf(`cpu,host=serverA,region=us-west value=%s`, string(maxFloat64))) + if err != nil { + t.Errorf(`ParsePoints("%s") mismatch. got %v, exp nil`, `cpu,host=serverA,region=us-west value=9223372036854775807`, err) + } + fields, err := p[0].Fields() + if err != nil { + t.Fatal(err) + } + if exp, got := math.MaxFloat64, fields["value"].(float64); exp != got { + t.Fatalf("ParsePoints Value mismatch. \nexp: %v\ngot: %v", exp, got) + } + + // leading zeros + _, err = models.ParsePointsString(fmt.Sprintf(`cpu,host=serverA,region=us-west value=%s`, "0000"+string(maxFloat64))) + if err != nil { + t.Errorf(`ParsePoints("%s") mismatch. got %v, exp nil`, `cpu,host=serverA,region=us-west value=0009223372036854775807`, err) + } +} + +func TestParsePointMinFloat64(t *testing.T) { + // out of range + _, err := models.ParsePointsString(fmt.Sprintf(`cpu,host=serverA,region=us-west value=%s`, "-1"+string(minFloat64)[1:])) + if err == nil { + t.Errorf(`ParsePoints("%s") mismatch. got nil, exp error`, `cpu,host=serverA,region=us-west value=...`) + } + + // min float + p, err := models.ParsePointsString(fmt.Sprintf(`cpu,host=serverA,region=us-west value=%s`, string(minFloat64))) + if err != nil { + t.Errorf(`ParsePoints("%s") mismatch. got %v, exp nil`, `cpu,host=serverA,region=us-west value=...`, err) + } + fields, err := p[0].Fields() + if err != nil { + t.Fatal(err) + } + if exp, got := -math.MaxFloat64, fields["value"].(float64); exp != got { + t.Fatalf("ParsePoints Value mismatch. \nexp: %v\ngot: %v", exp, got) + } + + // leading zeros + _, err = models.ParsePointsString(fmt.Sprintf(`cpu,host=serverA,region=us-west value=%s`, "-0000000"+string(minFloat64)[1:])) + if err != nil { + t.Errorf(`ParsePoints("%s") mismatch. got %v, exp nil`, `cpu,host=serverA,region=us-west value=...`, err) + } +} + +func TestParsePointMaxUint64(t *testing.T) { + // out of range + _, err := models.ParsePointsString(`cpu,host=serverA,region=us-west value=18446744073709551616u`) + exp := `unable to parse 'cpu,host=serverA,region=us-west value=18446744073709551616u': unable to parse unsigned 18446744073709551616: strconv.ParseUint: parsing "18446744073709551616": value out of range` + if err == nil || (err != nil && err.Error() != exp) { + t.Fatalf("Error mismatch:\nexp: %s\ngot: %v", exp, err) + } + + // max int + p, err := models.ParsePointsString(`cpu,host=serverA,region=us-west value=18446744073709551615u`) + if err != nil { + t.Fatalf(`ParsePoints("%s") mismatch. got %v, exp nil`, `cpu,host=serverA,region=us-west value=18446744073709551615u`, err) + } + fields, err := p[0].Fields() + if err != nil { + t.Fatal(err) + } + if exp, got := uint64(18446744073709551615), fields["value"].(uint64); exp != got { + t.Fatalf("ParsePoints Value mismatch. \nexp: %v\ngot: %v", exp, got) + } + + // leading zeros + _, err = models.ParsePointsString(`cpu,host=serverA,region=us-west value=00018446744073709551615u`) + if err != nil { + t.Fatalf(`ParsePoints("%s") mismatch. got %v, exp nil`, `cpu,host=serverA,region=us-west value=00018446744073709551615u`, err) + } +} + +func TestParsePointMinUint64(t *testing.T) { + // out of range + _, err := models.ParsePointsString(`cpu,host=serverA,region=us-west value=--1u`) + if err == nil { + t.Errorf(`ParsePoints("%s") mismatch. got nil, exp error`, `cpu,host=serverA,region=us-west value=-1u`) + } + + // min int + p, err := models.ParsePointsString(`cpu,host=serverA,region=us-west value=0u`) + if err != nil { + t.Errorf(`ParsePoints("%s") mismatch. got %v, exp nil`, `cpu,host=serverA,region=us-west value=0u`, err) + } + fields, err := p[0].Fields() + if err != nil { + t.Fatal(err) + } + if exp, got := uint64(0), fields["value"].(uint64); exp != got { + t.Fatalf("ParsePoints Value mismatch. \nexp: %v\ngot: %v", exp, got) + } + + // leading zeros + _, err = models.ParsePointsString(`cpu,host=serverA,region=us-west value=0000u`) + if err != nil { + t.Errorf(`ParsePoints("%s") mismatch. got %v, exp nil`, `cpu,host=serverA,region=us-west value=0000u`, err) + } +} + +func TestParsePointNumberNonNumeric(t *testing.T) { + _, err := models.ParsePointsString(`cpu,host=serverA,region=us-west value=.1a`) + if err == nil { + t.Errorf(`ParsePoints("%s") mismatch. got nil, exp error`, `cpu,host=serverA,region=us-west value=.1a`) + } +} + +func TestParsePointNegativeWrongPlace(t *testing.T) { + _, err := models.ParsePointsString(`cpu,host=serverA,region=us-west value=0.-1`) + if err == nil { + t.Errorf(`ParsePoints("%s") mismatch. got nil, exp error`, `cpu,host=serverA,region=us-west value=0.-1`) + } +} + +func TestParsePointOnlyNegativeSign(t *testing.T) { + _, err := models.ParsePointsString(`cpu,host=serverA,region=us-west value=-`) + if err == nil { + t.Errorf(`ParsePoints("%s") mismatch. got nil, exp error`, `cpu,host=serverA,region=us-west value=-`) + } +} + +func TestParsePointFloatMultipleDecimals(t *testing.T) { + _, err := models.ParsePointsString(`cpu,host=serverA,region=us-west value=1.1.1`) + if err == nil { + t.Errorf(`ParsePoints("%s") mismatch. got nil, exp error`, `cpu,host=serverA,region=us-west value=1.1.1`) + } +} + +func TestParsePointInteger(t *testing.T) { + _, err := models.ParsePointsString(`cpu,host=serverA,region=us-west value=1i`) + if err != nil { + t.Errorf(`ParsePoints("%s") mismatch. got %v, exp nil`, `cpu,host=serverA,region=us-west value=1i`, err) + } +} + +func TestParsePointNegativeInteger(t *testing.T) { + _, err := models.ParsePointsString(`cpu,host=serverA,region=us-west value=-1i`) + if err != nil { + t.Errorf(`ParsePoints("%s") mismatch. got %v, exp nil`, `cpu,host=serverA,region=us-west value=-1i`, err) + } +} + +func TestParsePointNegativeFloat(t *testing.T) { + _, err := models.ParsePointsString(`cpu,host=serverA,region=us-west value=-1.0`) + if err != nil { + t.Errorf(`ParsePoints("%s") mismatch. got %v, exp nil`, `cpu,host=serverA,region=us-west value=-1.0`, err) + } +} + +func TestParsePointFloatNoLeadingDigit(t *testing.T) { + _, err := models.ParsePointsString(`cpu,host=serverA,region=us-west value=.1`) + if err != nil { + t.Errorf(`ParsePoints("%s") mismatch. got %v, exp nil`, `cpu,host=serverA,region=us-west value=-1.0`, err) + } +} + +func TestParsePointFloatScientific(t *testing.T) { + _, err := models.ParsePointsString(`cpu,host=serverA,region=us-west value=1.0e4`) + if err != nil { + t.Errorf(`ParsePoints("%s") mismatch. got %v, exp nil`, `cpu,host=serverA,region=us-west value=1.0e4`, err) + } + + pts, err := models.ParsePointsString(`cpu,host=serverA,region=us-west value=1e4`) + if err != nil { + t.Errorf(`ParsePoints("%s") mismatch. got %v, exp nil`, `cpu,host=serverA,region=us-west value=1.0e4`, err) + } + + fields, err := pts[0].Fields() + if err != nil { + t.Fatal(err) + } + if fields["value"] != 1e4 { + t.Errorf(`ParsePoints("%s") mismatch. got %v, exp nil`, `cpu,host=serverA,region=us-west value=1e4`, err) + } +} + +func TestParsePointFloatScientificUpper(t *testing.T) { + _, err := models.ParsePointsString(`cpu,host=serverA,region=us-west value=1.0E4`) + if err != nil { + t.Errorf(`ParsePoints("%s") mismatch. got %v, exp nil`, `cpu,host=serverA,region=us-west value=1.0E4`, err) + } + + pts, err := models.ParsePointsString(`cpu,host=serverA,region=us-west value=1E4`) + if err != nil { + t.Errorf(`ParsePoints("%s") mismatch. got %v, exp nil`, `cpu,host=serverA,region=us-west value=1.0E4`, err) + } + + fields, err := pts[0].Fields() + if err != nil { + t.Fatal(err) + } + if fields["value"] != 1e4 { + t.Errorf(`ParsePoints("%s") mismatch. got %v, exp nil`, `cpu,host=serverA,region=us-west value=1E4`, err) + } +} + +func TestParsePointFloatScientificDecimal(t *testing.T) { + _, err := models.ParsePointsString(`cpu,host=serverA,region=us-west value=1.0e-4`) + if err != nil { + t.Errorf(`ParsePoints("%s") mismatch. got %v, exp nil`, `cpu,host=serverA,region=us-west value=1.0e-4`, err) + } +} + +func TestParsePointFloatNegativeScientific(t *testing.T) { + _, err := models.ParsePointsString(`cpu,host=serverA,region=us-west value=-1.0e-4`) + if err != nil { + t.Errorf(`ParsePoints("%s") mismatch. got %v, exp nil`, `cpu,host=serverA,region=us-west value=-1.0e-4`, err) + } +} + +func TestParsePointBooleanInvalid(t *testing.T) { + _, err := models.ParsePointsString(`cpu,host=serverA,region=us-west value=a`) + if err == nil { + t.Errorf(`ParsePoints("%s") mismatch. got nil, exp error`, `cpu,host=serverA,region=us-west value=a`) + } +} + +func TestParsePointScientificIntInvalid(t *testing.T) { + _, err := models.ParsePointsString(`cpu,host=serverA,region=us-west value=9ie10`) + if err == nil { + t.Errorf(`ParsePoints("%s") mismatch. got nil, exp error`, `cpu,host=serverA,region=us-west value=9ie10`) + } + + _, err = models.ParsePointsString(`cpu,host=serverA,region=us-west value=9e10i`) + if err == nil { + t.Errorf(`ParsePoints("%s") mismatch. got nil, exp error`, `cpu,host=serverA,region=us-west value=9e10i`) + } +} + +func TestParsePointWhitespace(t *testing.T) { + examples := []string{ + `cpu value=1.0 1257894000000000000`, + `cpu value=1.0 1257894000000000000`, + `cpu value=1.0 1257894000000000000`, + `cpu value=1.0 1257894000000000000 `, + `cpu value=1.0 1257894000000000000 +`, + `cpu value=1.0 1257894000000000000 +`, + } + + expPoint := NewTestPoint("cpu", models.Tags{}, models.Fields{"value": 1.0}, time.Unix(0, 1257894000000000000)) + for i, example := range examples { + pts, err := models.ParsePoints([]byte(example)) + if err != nil { + t.Fatalf(`[Example %d] ParsePoints("%s") error. got %v, exp nil`, i, example, err) + } + + if got, exp := len(pts), 1; got != exp { + t.Fatalf("[Example %d] got %d points, expected %d", i, got, exp) + } + + if got, exp := string(pts[0].Name()), string(expPoint.Name()); got != exp { + t.Fatalf("[Example %d] got %v measurement, expected %v", i, got, exp) + } + + fields, err := pts[0].Fields() + if err != nil { + t.Fatal(err) + } + eFields, err := expPoint.Fields() + if err != nil { + t.Fatal(err) + } + if got, exp := len(fields), len(eFields); got != exp { + t.Fatalf("[Example %d] got %d fields, expected %d", i, got, exp) + } + + if got, exp := fields["value"], eFields["value"]; got != exp { + t.Fatalf(`[Example %d] got %v for field "value", expected %v`, i, got, exp) + } + + if got, exp := pts[0].Time().UnixNano(), expPoint.Time().UnixNano(); got != exp { + t.Fatalf(`[Example %d] got %d time, expected %d`, i, got, exp) + } + } +} + +func TestParsePointUnescape(t *testing.T) { + // commas in measurement name + test(t, `foo\,bar value=1i`, + NewTestPoint( + "foo,bar", // comma in the name + models.NewTags(map[string]string{}), + models.Fields{ + "value": int64(1), + }, + time.Unix(0, 0))) + + // comma in measurement name with tags + test(t, `cpu\,main,regions=east value=1.0`, + NewTestPoint( + "cpu,main", // comma in the name + models.NewTags(map[string]string{ + "regions": "east", + }), + models.Fields{ + "value": 1.0, + }, + time.Unix(0, 0))) + + // spaces in measurement name + test(t, `cpu\ load,region=east value=1.0`, + NewTestPoint( + "cpu load", // space in the name + models.NewTags(map[string]string{ + "region": "east", + }), + models.Fields{ + "value": 1.0, + }, + time.Unix(0, 0))) + + // equals in measurement name + test(t, `cpu\=load,region=east value=1.0`, + NewTestPoint( + `cpu\=load`, // backslash is literal + models.NewTags(map[string]string{ + "region": "east", + }), + models.Fields{ + "value": 1.0, + }, + time.Unix(0, 0))) + + // equals in measurement name + test(t, `cpu=load,region=east value=1.0`, + NewTestPoint( + `cpu=load`, // literal equals is fine in measurement name + models.NewTags(map[string]string{ + "region": "east", + }), + models.Fields{ + "value": 1.0, + }, + time.Unix(0, 0))) + + // commas in tag names + test(t, `cpu,region\,zone=east value=1.0`, + NewTestPoint("cpu", + models.NewTags(map[string]string{ + "region,zone": "east", // comma in the tag key + }), + models.Fields{ + "value": 1.0, + }, + time.Unix(0, 0))) + + // spaces in tag name + test(t, `cpu,region\ zone=east value=1.0`, + NewTestPoint("cpu", + models.NewTags(map[string]string{ + "region zone": "east", // space in the tag name + }), + models.Fields{ + "value": 1.0, + }, + time.Unix(0, 0))) + + // backslash with escaped equals in tag name + test(t, `cpu,reg\\=ion=east value=1.0`, + NewTestPoint("cpu", + models.NewTags(map[string]string{ + `reg\=ion`: "east", + }), + models.Fields{ + "value": 1.0, + }, + time.Unix(0, 0))) + + // space is tag name + test(t, `cpu,\ =east value=1.0`, + NewTestPoint("cpu", + models.NewTags(map[string]string{ + " ": "east", // tag name is single space + }), + models.Fields{ + "value": 1.0, + }, + time.Unix(0, 0))) + + // commas in tag values + test(t, `cpu,regions=east\,west value=1.0`, + NewTestPoint("cpu", + models.NewTags(map[string]string{ + "regions": "east,west", // comma in the tag value + }), + models.Fields{ + "value": 1.0, + }, + time.Unix(0, 0))) + + // backslash literal followed by escaped space + test(t, `cpu,regions=\\ east value=1.0`, + NewTestPoint( + "cpu", + models.NewTags(map[string]string{ + "regions": `\ east`, + }), + models.Fields{ + "value": 1.0, + }, + time.Unix(0, 0))) + + // backslash literal followed by escaped space + test(t, `cpu,regions=eas\\ t value=1.0`, + NewTestPoint( + "cpu", + models.NewTags(map[string]string{ + "regions": `eas\ t`, + }), + models.Fields{ + "value": 1.0, + }, + time.Unix(0, 0))) + + // backslash literal followed by trailing space + test(t, `cpu,regions=east\\ value=1.0`, + NewTestPoint( + "cpu", + models.NewTags(map[string]string{ + "regions": `east\ `, + }), + models.Fields{ + "value": 1.0, + }, + time.Unix(0, 0))) + + // spaces in tag values + test(t, `cpu,regions=east\ west value=1.0`, + NewTestPoint("cpu", + models.NewTags(map[string]string{ + "regions": "east west", // comma in the tag value + }), + models.Fields{ + "value": 1.0, + }, + time.Unix(0, 0))) + + // commas in field keys + test(t, `cpu,regions=east value\,ms=1.0`, + NewTestPoint("cpu", + models.NewTags(map[string]string{ + "regions": "east", + }), + models.Fields{ + "value,ms": 1.0, // comma in the field keys + }, + time.Unix(0, 0))) + + // spaces in field keys + test(t, `cpu,regions=east value\ ms=1.0`, + NewTestPoint("cpu", + models.NewTags(map[string]string{ + "regions": "east", + }), + models.Fields{ + "value ms": 1.0, // comma in the field keys + }, + time.Unix(0, 0))) + + // tag with no value + test(t, `cpu,regions=east value="1"`, + NewTestPoint("cpu", + models.NewTags(map[string]string{ + "regions": "east", + "foobar": "", + }), + models.Fields{ + "value": "1", + }, + time.Unix(0, 0))) + + // commas in field values + test(t, `cpu,regions=east value="1,0"`, + NewTestPoint("cpu", + models.NewTags(map[string]string{ + "regions": "east", + }), + models.Fields{ + "value": "1,0", // comma in the field value + }, + time.Unix(0, 0))) + + // random character escaped + test(t, `cpu,regions=eas\t value=1.0`, + NewTestPoint( + "cpu", + models.NewTags(map[string]string{ + "regions": "eas\\t", + }), + models.Fields{ + "value": 1.0, + }, + time.Unix(0, 0))) + + // backslash literal followed by escaped characters + test(t, `cpu,regions=\\,\,\=east value=1.0`, + NewTestPoint( + "cpu", + models.NewTags(map[string]string{ + "regions": `\,,=east`, + }), + models.Fields{ + "value": 1.0, + }, + time.Unix(0, 0))) + + // field keys using escape char. + test(t, `cpu \a=1i`, + NewTestPoint( + "cpu", + models.NewTags(map[string]string{}), + models.Fields{ + "\\a": int64(1), // Left as parsed since it's not a known escape sequence. + }, + time.Unix(0, 0))) + + // measurement, tag and tag value with equals + test(t, `cpu=load,equals\=foo=tag\=value value=1i`, + NewTestPoint( + "cpu=load", // Not escaped + models.NewTags(map[string]string{ + "equals=foo": "tag=value", // Tag and value unescaped + }), + models.Fields{ + "value": int64(1), + }, + time.Unix(0, 0))) + +} + +func TestParsePointWithTags(t *testing.T) { + test(t, + "cpu,host=serverA,region=us-east value=1.0 1000000000", + NewTestPoint("cpu", + models.NewTags(map[string]string{"host": "serverA", "region": "us-east"}), + models.Fields{"value": 1.0}, time.Unix(1, 0))) +} + +func TestParsePointWithDuplicateTags(t *testing.T) { + for i, tt := range []struct { + line string + err string + }{ + { + line: `cpu,host=serverA,host=serverB value=1i 1000000000`, + err: `unable to parse 'cpu,host=serverA,host=serverB value=1i 1000000000': duplicate tags`, + }, + { + line: `cpu,b=2,b=1,c=3 value=1i 1000000000`, + err: `unable to parse 'cpu,b=2,b=1,c=3 value=1i 1000000000': duplicate tags`, + }, + { + line: `cpu,b=2,c=3,b=1 value=1i 1000000000`, + err: `unable to parse 'cpu,b=2,c=3,b=1 value=1i 1000000000': duplicate tags`, + }, + } { + _, err := models.ParsePointsString(tt.line) + if err == nil || tt.err != err.Error() { + t.Errorf("%d. ParsePoint() expected error '%s'. got '%s'", i, tt.err, err) + } + } +} + +func TestParsePointWithStringField(t *testing.T) { + test(t, `cpu,host=serverA,region=us-east value=1.0,str="foo",str2="bar" 1000000000`, + NewTestPoint("cpu", + models.NewTags(map[string]string{ + "host": "serverA", + "region": "us-east", + }), + models.Fields{ + "value": 1.0, + "str": "foo", + "str2": "bar", + }, + time.Unix(1, 0)), + ) + + test(t, `cpu,host=serverA,region=us-east str="foo \" bar" 1000000000`, + NewTestPoint("cpu", + models.NewTags(map[string]string{ + "host": "serverA", + "region": "us-east", + }), + models.Fields{ + "str": `foo " bar`, + }, + time.Unix(1, 0)), + ) + +} + +func TestParsePointWithStringWithSpaces(t *testing.T) { + test(t, `cpu,host=serverA,region=us-east value=1.0,str="foo bar" 1000000000`, + NewTestPoint( + "cpu", + models.NewTags(map[string]string{ + "host": "serverA", + "region": "us-east", + }), + models.Fields{ + "value": 1.0, + "str": "foo bar", // spaces in string value + }, + time.Unix(1, 0)), + ) +} + +func TestParsePointWithStringWithNewline(t *testing.T) { + test(t, "cpu,host=serverA,region=us-east value=1.0,str=\"foo\nbar\" 1000000000", + NewTestPoint( + "cpu", + models.NewTags(map[string]string{ + "host": "serverA", + "region": "us-east", + }), + models.Fields{ + "value": 1.0, + "str": "foo\nbar", // newline in string value + }, + time.Unix(1, 0)), + ) +} + +func TestParsePointWithStringWithCommas(t *testing.T) { + // escaped comma + test(t, `cpu,host=serverA,region=us-east value=1.0,str="foo\,bar" 1000000000`, + NewTestPoint( + "cpu", + models.NewTags(map[string]string{ + "host": "serverA", + "region": "us-east", + }), + models.Fields{ + "value": 1.0, + "str": `foo\,bar`, // commas in string value + }, + time.Unix(1, 0)), + ) + + // non-escaped comma + test(t, `cpu,host=serverA,region=us-east value=1.0,str="foo,bar" 1000000000`, + NewTestPoint( + "cpu", + models.NewTags(map[string]string{ + "host": "serverA", + "region": "us-east", + }), + models.Fields{ + "value": 1.0, + "str": "foo,bar", // commas in string value + }, + time.Unix(1, 0)), + ) + + // string w/ trailing escape chars + test(t, `cpu,host=serverA,region=us-east str="foo\\",str2="bar" 1000000000`, + NewTestPoint( + "cpu", + models.NewTags(map[string]string{ + "host": "serverA", + "region": "us-east", + }), + models.Fields{ + "str": "foo\\", // trailing escape char + "str2": "bar", + }, + time.Unix(1, 0)), + ) +} + +func TestParsePointQuotedMeasurement(t *testing.T) { + // non-escaped comma + test(t, `"cpu",host=serverA,region=us-east value=1.0 1000000000`, + NewTestPoint( + `"cpu"`, + models.NewTags(map[string]string{ + "host": "serverA", + "region": "us-east", + }), + models.Fields{ + "value": 1.0, + }, + time.Unix(1, 0)), + ) +} + +func TestParsePointQuotedTags(t *testing.T) { + test(t, `cpu,"host"="serverA",region=us-east value=1.0 1000000000`, + NewTestPoint( + "cpu", + models.NewTags(map[string]string{ + `"host"`: `"serverA"`, + "region": "us-east", + }), + models.Fields{ + "value": 1.0, + }, + time.Unix(1, 0)), + ) +} + +func TestParsePoint_TrailingSlash(t *testing.T) { + _, err := models.ParsePointsString(`a v=1 0\`) + if err == nil { + t.Fatalf("ParsePoints failed: %v", err) + } else if !strings.Contains(err.Error(), "bad timestamp") { + t.Fatalf("ParsePoints unexpected error: %v", err) + } +} + +func TestParsePointsUnbalancedQuotedTags(t *testing.T) { + pts, err := models.ParsePointsString("baz,mytag=\"a x=1 1441103862125\nbaz,mytag=a z=1 1441103862126") + if err != nil { + t.Fatalf("ParsePoints failed: %v", err) + } + + if exp := 2; len(pts) != exp { + t.Fatalf("ParsePoints count mismatch. got %v, exp %v", len(pts), exp) + } + + // Expected " in the tag value + exp := models.MustNewPoint("baz", models.NewTags(map[string]string{"mytag": `"a`}), + models.Fields{"x": float64(1)}, time.Unix(0, 1441103862125)) + + if pts[0].String() != exp.String() { + t.Errorf("Point mismatch:\ngot: %v\nexp: %v", pts[0].String(), exp.String()) + } + + // Expected two points to ensure we did not overscan the line + exp = models.MustNewPoint("baz", models.NewTags(map[string]string{"mytag": `a`}), + models.Fields{"z": float64(1)}, time.Unix(0, 1441103862126)) + + if pts[1].String() != exp.String() { + t.Errorf("Point mismatch:\ngot: %v\nexp: %v", pts[1].String(), exp.String()) + } + +} + +func TestParsePointEscapedStringsAndCommas(t *testing.T) { + // non-escaped comma and quotes + test(t, `cpu,host=serverA,region=us-east value="{Hello\"{,}\" World}" 1000000000`, + NewTestPoint( + "cpu", + models.NewTags(map[string]string{ + "host": "serverA", + "region": "us-east", + }), + models.Fields{ + "value": `{Hello"{,}" World}`, + }, + time.Unix(1, 0)), + ) + + // escaped comma and quotes + test(t, `cpu,host=serverA,region=us-east value="{Hello\"{\,}\" World}" 1000000000`, + NewTestPoint( + "cpu", + models.NewTags(map[string]string{ + "host": "serverA", + "region": "us-east", + }), + models.Fields{ + "value": `{Hello"{\,}" World}`, + }, + time.Unix(1, 0)), + ) +} + +func TestParsePointWithStringWithEquals(t *testing.T) { + test(t, `cpu,host=serverA,region=us-east str="foo=bar",value=1.0 1000000000`, + NewTestPoint( + "cpu", + models.NewTags(map[string]string{ + "host": "serverA", + "region": "us-east", + }), + models.Fields{ + "value": 1.0, + "str": "foo=bar", // spaces in string value + }, + time.Unix(1, 0)), + ) +} + +func TestParsePointWithStringWithBackslash(t *testing.T) { + test(t, `cpu value="test\\\"" 1000000000`, + NewTestPoint( + "cpu", + models.NewTags(map[string]string{}), + models.Fields{ + "value": `test\"`, + }, + time.Unix(1, 0)), + ) + + test(t, `cpu value="test\\" 1000000000`, + NewTestPoint( + "cpu", + models.NewTags(map[string]string{}), + models.Fields{ + "value": `test\`, + }, + time.Unix(1, 0)), + ) + + test(t, `cpu value="test\\\"" 1000000000`, + NewTestPoint( + "cpu", + models.NewTags(map[string]string{}), + models.Fields{ + "value": `test\"`, + }, + time.Unix(1, 0)), + ) + + test(t, `cpu value="test\"" 1000000000`, + NewTestPoint( + "cpu", + models.NewTags(map[string]string{}), + models.Fields{ + "value": `test"`, + }, + time.Unix(1, 0)), + ) +} + +func TestParsePointWithBoolField(t *testing.T) { + test(t, `cpu,host=serverA,region=us-east true=true,t=t,T=T,TRUE=TRUE,True=True,false=false,f=f,F=F,FALSE=FALSE,False=False 1000000000`, + NewTestPoint( + "cpu", + models.NewTags(map[string]string{ + "host": "serverA", + "region": "us-east", + }), + models.Fields{ + "t": true, + "T": true, + "true": true, + "True": true, + "TRUE": true, + "f": false, + "F": false, + "false": false, + "False": false, + "FALSE": false, + }, + time.Unix(1, 0)), + ) +} + +func TestParsePointUnicodeString(t *testing.T) { + test(t, `cpu,host=serverA,region=us-east value="wè" 1000000000`, + NewTestPoint( + "cpu", + models.NewTags(map[string]string{ + "host": "serverA", + "region": "us-east", + }), + models.Fields{ + "value": "wè", + }, + time.Unix(1, 0)), + ) +} + +func TestParsePointNegativeTimestamp(t *testing.T) { + test(t, `cpu value=1 -1`, + NewTestPoint( + "cpu", + models.NewTags(map[string]string{}), + models.Fields{ + "value": 1.0, + }, + time.Unix(0, -1)), + ) +} + +func TestParsePointMaxTimestamp(t *testing.T) { + test(t, fmt.Sprintf(`cpu value=1 %d`, models.MaxNanoTime), + NewTestPoint( + "cpu", + models.NewTags(map[string]string{}), + models.Fields{ + "value": 1.0, + }, + time.Unix(0, models.MaxNanoTime)), + ) +} + +func TestParsePointMinTimestamp(t *testing.T) { + test(t, `cpu value=1 -9223372036854775806`, + NewTestPoint( + "cpu", + models.NewTags(map[string]string{}), + models.Fields{ + "value": 1.0, + }, + time.Unix(0, models.MinNanoTime)), + ) +} + +func TestParsePointInvalidTimestamp(t *testing.T) { + examples := []string{ + "cpu value=1 9223372036854775808", + "cpu value=1 -92233720368547758078", + "cpu value=1 -", + "cpu value=1 -/", + "cpu value=1 -1?", + "cpu value=1 1-", + "cpu value=1 9223372036854775807 12", + } + + for i, example := range examples { + _, err := models.ParsePointsString(example) + if err == nil { + t.Fatalf("[Example %d] ParsePoints failed: %v", i, err) + } + } +} + +func TestNewPointFloatWithoutDecimal(t *testing.T) { + test(t, `cpu value=1 1000000000`, + NewTestPoint( + "cpu", + models.NewTags(map[string]string{}), + models.Fields{ + "value": 1.0, + }, + time.Unix(1, 0)), + ) +} +func TestNewPointNegativeFloat(t *testing.T) { + test(t, `cpu value=-0.64 1000000000`, + NewTestPoint( + "cpu", + models.NewTags(map[string]string{}), + models.Fields{ + "value": -0.64, + }, + time.Unix(1, 0)), + ) +} + +func TestNewPointFloatNoDecimal(t *testing.T) { + test(t, `cpu value=1. 1000000000`, + NewTestPoint( + "cpu", + models.NewTags(map[string]string{}), + models.Fields{ + "value": 1.0, + }, + time.Unix(1, 0)), + ) +} + +func TestNewPointFloatScientific(t *testing.T) { + test(t, `cpu value=6.632243e+06 1000000000`, + NewTestPoint( + "cpu", + models.NewTags(map[string]string{}), + models.Fields{ + "value": float64(6632243), + }, + time.Unix(1, 0)), + ) +} + +func TestNewPointLargeInteger(t *testing.T) { + test(t, `cpu value=6632243i 1000000000`, + NewTestPoint( + "cpu", + models.NewTags(map[string]string{}), + models.Fields{ + "value": int64(6632243), // if incorrectly encoded as a float, it would show up as 6.632243e+06 + }, + time.Unix(1, 0)), + ) +} + +func TestParsePointNaN(t *testing.T) { + _, err := models.ParsePointsString("cpu value=NaN 1000000000") + if err == nil { + t.Fatalf("ParsePoints expected error, got nil") + } + + _, err = models.ParsePointsString("cpu value=nAn 1000000000") + if err == nil { + t.Fatalf("ParsePoints expected error, got nil") + } + + _, err = models.ParsePointsString("cpu value=NaN") + if err == nil { + t.Fatalf("ParsePoints expected error, got nil") + } +} + +func TestNewPointLargeNumberOfTags(t *testing.T) { + tags := "" + for i := 0; i < 255; i++ { + tags += fmt.Sprintf(",tag%d=value%d", i, i) + } + + pt, err := models.ParsePointsString(fmt.Sprintf("cpu%s value=1", tags)) + if err != nil { + t.Fatalf("ParsePoints() with max tags failed: %v", err) + } + + if len(pt[0].Tags()) != 255 { + t.Fatalf("expected %d tags, got %d", 255, len(pt[0].Tags())) + } +} + +func TestParsePointIntsFloats(t *testing.T) { + pts, err := models.ParsePoints([]byte(`cpu,host=serverA,region=us-east int=10i,float=11.0,float2=12.1 1000000000`)) + if err != nil { + t.Fatalf(`ParsePoints() failed. got %s`, err) + } + + if exp := 1; len(pts) != exp { + t.Errorf("ParsePoint() len mismatch: got %v, exp %v", len(pts), exp) + } + pt := pts[0] + + fields, err := pt.Fields() + if err != nil { + t.Fatal(err) + } + if _, ok := fields["int"].(int64); !ok { + t.Errorf("ParsePoint() int field mismatch: got %T, exp %T", fields["int"], int64(10)) + } + + if _, ok := fields["float"].(float64); !ok { + t.Errorf("ParsePoint() float field mismatch: got %T, exp %T", fields["float64"], float64(11.0)) + } + + if _, ok := fields["float2"].(float64); !ok { + t.Errorf("ParsePoint() float field mismatch: got %T, exp %T", fields["float64"], float64(12.1)) + } +} + +func TestParsePointKeyUnsorted(t *testing.T) { + pts, err := models.ParsePoints([]byte("cpu,last=1,first=2 value=1i")) + if err != nil { + t.Fatalf(`ParsePoints() failed. got %s`, err) + } + + if exp := 1; len(pts) != exp { + t.Errorf("ParsePoint() len mismatch: got %v, exp %v", len(pts), exp) + } + pt := pts[0] + + if exp := "cpu,first=2,last=1"; string(pt.Key()) != exp { + t.Errorf("ParsePoint key not sorted. got %v, exp %v", string(pt.Key()), exp) + } +} + +func TestParsePointToString(t *testing.T) { + line := `cpu,host=serverA,region=us-east bool=false,float=11,float2=12.123,int=10i,str="string val" 1000000000` + pts, err := models.ParsePoints([]byte(line)) + if err != nil { + t.Fatalf(`ParsePoints() failed. got %s`, err) + } + if exp := 1; len(pts) != exp { + t.Errorf("ParsePoint() len mismatch: got %v, exp %v", len(pts), exp) + } + pt := pts[0] + + got := pt.String() + if line != got { + t.Errorf("ParsePoint() to string mismatch:\n got %v\n exp %v", got, line) + } + + pt = models.MustNewPoint("cpu", models.NewTags(map[string]string{"host": "serverA", "region": "us-east"}), + models.Fields{"int": 10, "float": float64(11.0), "float2": float64(12.123), "bool": false, "str": "string val"}, + time.Unix(1, 0)) + + got = pt.String() + if line != got { + t.Errorf("NewPoint() to string mismatch:\n got %v\n exp %v", got, line) + } +} + +func TestParsePointsWithPrecision(t *testing.T) { + tests := []struct { + name string + line string + precision string + exp string + }{ + { + name: "nanosecond by default", + line: `cpu,host=serverA,region=us-east value=1.0 946730096789012345`, + precision: "", + exp: "cpu,host=serverA,region=us-east value=1.0 946730096789012345", + }, + { + name: "nanosecond", + line: `cpu,host=serverA,region=us-east value=1.0 946730096789012345`, + precision: "n", + exp: "cpu,host=serverA,region=us-east value=1.0 946730096789012345", + }, + { + name: "microsecond", + line: `cpu,host=serverA,region=us-east value=1.0 946730096789012`, + precision: "u", + exp: "cpu,host=serverA,region=us-east value=1.0 946730096789012000", + }, + { + name: "millisecond", + line: `cpu,host=serverA,region=us-east value=1.0 946730096789`, + precision: "ms", + exp: "cpu,host=serverA,region=us-east value=1.0 946730096789000000", + }, + { + name: "second", + line: `cpu,host=serverA,region=us-east value=1.0 946730096`, + precision: "s", + exp: "cpu,host=serverA,region=us-east value=1.0 946730096000000000", + }, + { + name: "minute", + line: `cpu,host=serverA,region=us-east value=1.0 15778834`, + precision: "m", + exp: "cpu,host=serverA,region=us-east value=1.0 946730040000000000", + }, + { + name: "hour", + line: `cpu,host=serverA,region=us-east value=1.0 262980`, + precision: "h", + exp: "cpu,host=serverA,region=us-east value=1.0 946728000000000000", + }, + } + for _, test := range tests { + pts, err := models.ParsePointsWithPrecision([]byte(test.line), time.Now().UTC(), test.precision) + if err != nil { + t.Fatalf(`%s: ParsePoints() failed. got %s`, test.name, err) + } + if exp := 1; len(pts) != exp { + t.Errorf("%s: ParsePoint() len mismatch: got %v, exp %v", test.name, len(pts), exp) + } + pt := pts[0] + + got := pt.String() + if got != test.exp { + t.Errorf("%s: ParsePoint() to string mismatch:\n got %v\n exp %v", test.name, got, test.exp) + } + } +} + +func TestParsePointsWithPrecisionNoTime(t *testing.T) { + line := `cpu,host=serverA,region=us-east value=1.0` + tm, _ := time.Parse(time.RFC3339Nano, "2000-01-01T12:34:56.789012345Z") + tests := []struct { + name string + precision string + exp string + }{ + { + name: "no precision", + precision: "", + exp: "cpu,host=serverA,region=us-east value=1.0 946730096789012345", + }, + { + name: "nanosecond precision", + precision: "n", + exp: "cpu,host=serverA,region=us-east value=1.0 946730096789012345", + }, + { + name: "microsecond precision", + precision: "u", + exp: "cpu,host=serverA,region=us-east value=1.0 946730096789012000", + }, + { + name: "millisecond precision", + precision: "ms", + exp: "cpu,host=serverA,region=us-east value=1.0 946730096789000000", + }, + { + name: "second precision", + precision: "s", + exp: "cpu,host=serverA,region=us-east value=1.0 946730096000000000", + }, + { + name: "minute precision", + precision: "m", + exp: "cpu,host=serverA,region=us-east value=1.0 946730040000000000", + }, + { + name: "hour precision", + precision: "h", + exp: "cpu,host=serverA,region=us-east value=1.0 946728000000000000", + }, + } + + for _, test := range tests { + pts, err := models.ParsePointsWithPrecision([]byte(line), tm, test.precision) + if err != nil { + t.Fatalf(`%s: ParsePoints() failed. got %s`, test.name, err) + } + if exp := 1; len(pts) != exp { + t.Errorf("%s: ParsePoint() len mismatch: got %v, exp %v", test.name, len(pts), exp) + } + pt := pts[0] + + got := pt.String() + if got != test.exp { + t.Errorf("%s: ParsePoint() to string mismatch:\n got %v\n exp %v", test.name, got, test.exp) + } + } +} + +func TestParsePointsWithPrecisionComments(t *testing.T) { + tests := []struct { + name string + batch string + exp string + lenPoints int + }{ + { + name: "comment only", + batch: `# comment only`, + exp: "cpu,host=serverA,region=us-east value=1.0 946730096789012345", + lenPoints: 0, + }, + { + name: "point with comment above", + batch: `# a point is below +cpu,host=serverA,region=us-east value=1.0 946730096789012345`, + exp: "cpu,host=serverA,region=us-east value=1.0 946730096789012345", + lenPoints: 1, + }, + { + name: "point with comment below", + batch: `cpu,host=serverA,region=us-east value=1.0 946730096789012345 +# end of points`, + exp: "cpu,host=serverA,region=us-east value=1.0 946730096789012345", + lenPoints: 1, + }, + { + name: "indented comment", + batch: ` # a point is below +cpu,host=serverA,region=us-east value=1.0 946730096789012345`, + exp: "cpu,host=serverA,region=us-east value=1.0 946730096789012345", + lenPoints: 1, + }, + } + for _, test := range tests { + pts, err := models.ParsePointsWithPrecision([]byte(test.batch), time.Now().UTC(), "") + if err != nil { + t.Fatalf(`%s: ParsePoints() failed. got %s`, test.name, err) + } + pointsLength := len(pts) + if exp := test.lenPoints; pointsLength != exp { + t.Errorf("%s: ParsePoint() len mismatch: got %v, exp %v", test.name, pointsLength, exp) + } + + if pointsLength > 0 { + pt := pts[0] + + got := pt.String() + if got != test.exp { + t.Errorf("%s: ParsePoint() to string mismatch:\n got %v\n exp %v", test.name, got, test.exp) + } + } + } +} + +func TestNewPointEscaped(t *testing.T) { + // commas + pt := models.MustNewPoint("cpu,main", models.NewTags(map[string]string{"tag,bar": "value"}), models.Fields{"name,bar": 1.0}, time.Unix(0, 0)) + if exp := `cpu\,main,tag\,bar=value name\,bar=1 0`; pt.String() != exp { + t.Errorf("NewPoint().String() mismatch.\ngot %v\nexp %v", pt.String(), exp) + } + + // spaces + pt = models.MustNewPoint("cpu main", models.NewTags(map[string]string{"tag bar": "value"}), models.Fields{"name bar": 1.0}, time.Unix(0, 0)) + if exp := `cpu\ main,tag\ bar=value name\ bar=1 0`; pt.String() != exp { + t.Errorf("NewPoint().String() mismatch.\ngot %v\nexp %v", pt.String(), exp) + } + + // equals + pt = models.MustNewPoint("cpu=main", models.NewTags(map[string]string{"tag=bar": "value=foo"}), models.Fields{"name=bar": 1.0}, time.Unix(0, 0)) + if exp := `cpu=main,tag\=bar=value\=foo name\=bar=1 0`; pt.String() != exp { + t.Errorf("NewPoint().String() mismatch.\ngot %v\nexp %v", pt.String(), exp) + } +} + +func TestNewPointWithoutField(t *testing.T) { + _, err := models.NewPoint("cpu", models.NewTags(map[string]string{"tag": "bar"}), models.Fields{}, time.Unix(0, 0)) + if err == nil { + t.Fatalf(`NewPoint() expected error. got nil`) + } +} + +func TestNewPointUnhandledType(t *testing.T) { + // nil value + pt := models.MustNewPoint("cpu", nil, models.Fields{"value": nil}, time.Unix(0, 0)) + if exp := `cpu value= 0`; pt.String() != exp { + t.Errorf("NewPoint().String() mismatch.\ngot %v\nexp %v", pt.String(), exp) + } + + // unsupported type gets stored as string + now := time.Unix(0, 0).UTC() + pt = models.MustNewPoint("cpu", nil, models.Fields{"value": now}, time.Unix(0, 0)) + if exp := `cpu value="1970-01-01 00:00:00 +0000 UTC" 0`; pt.String() != exp { + t.Errorf("NewPoint().String() mismatch.\ngot %v\nexp %v", pt.String(), exp) + } + + fields, err := pt.Fields() + if err != nil { + t.Fatal(err) + } + if exp := "1970-01-01 00:00:00 +0000 UTC"; fields["value"] != exp { + t.Errorf("NewPoint().String() mismatch.\ngot %v\nexp %v", pt.String(), exp) + } +} + +func TestMakeKeyEscaped(t *testing.T) { + if exp, got := `cpu\ load`, models.MakeKey([]byte(`cpu\ load`), models.NewTags(map[string]string{})); string(got) != exp { + t.Errorf("MakeKey() mismatch.\ngot %v\nexp %v", got, exp) + } + + if exp, got := `cpu\ load`, models.MakeKey([]byte(`cpu load`), models.NewTags(map[string]string{})); string(got) != exp { + t.Errorf("MakeKey() mismatch.\ngot %v\nexp %v", got, exp) + } + + if exp, got := `cpu\,load`, models.MakeKey([]byte(`cpu\,load`), models.NewTags(map[string]string{})); string(got) != exp { + t.Errorf("MakeKey() mismatch.\ngot %v\nexp %v", got, exp) + } + + if exp, got := `cpu\,load`, models.MakeKey([]byte(`cpu,load`), models.NewTags(map[string]string{})); string(got) != exp { + t.Errorf("MakeKey() mismatch.\ngot %v\nexp %v", got, exp) + } + +} + +func TestPrecisionString(t *testing.T) { + tags := map[string]interface{}{"value": float64(1)} + tm, _ := time.Parse(time.RFC3339Nano, "2000-01-01T12:34:56.789012345Z") + tests := []struct { + name string + precision string + exp string + }{ + { + name: "no precision", + precision: "", + exp: "cpu value=1 946730096789012345", + }, + { + name: "nanosecond precision", + precision: "ns", + exp: "cpu value=1 946730096789012345", + }, + { + name: "microsecond precision", + precision: "u", + exp: "cpu value=1 946730096789012", + }, + { + name: "millisecond precision", + precision: "ms", + exp: "cpu value=1 946730096789", + }, + { + name: "second precision", + precision: "s", + exp: "cpu value=1 946730096", + }, + { + name: "minute precision", + precision: "m", + exp: "cpu value=1 15778834", + }, + { + name: "hour precision", + precision: "h", + exp: "cpu value=1 262980", + }, + } + + for _, test := range tests { + pt := models.MustNewPoint("cpu", nil, tags, tm) + act := pt.PrecisionString(test.precision) + + if act != test.exp { + t.Errorf("%s: PrecisionString() mismatch:\n actual: %v\n exp: %v", + test.name, act, test.exp) + } + } +} + +func TestRoundedString(t *testing.T) { + tags := map[string]interface{}{"value": float64(1)} + tm, _ := time.Parse(time.RFC3339Nano, "2000-01-01T12:34:56.789012345Z") + tests := []struct { + name string + precision time.Duration + exp string + }{ + { + name: "no precision", + precision: time.Duration(0), + exp: "cpu value=1 946730096789012345", + }, + { + name: "nanosecond precision", + precision: time.Nanosecond, + exp: "cpu value=1 946730096789012345", + }, + { + name: "microsecond precision", + precision: time.Microsecond, + exp: "cpu value=1 946730096789012000", + }, + { + name: "millisecond precision", + precision: time.Millisecond, + exp: "cpu value=1 946730096789000000", + }, + { + name: "second precision", + precision: time.Second, + exp: "cpu value=1 946730097000000000", + }, + { + name: "minute precision", + precision: time.Minute, + exp: "cpu value=1 946730100000000000", + }, + { + name: "hour precision", + precision: time.Hour, + exp: "cpu value=1 946731600000000000", + }, + } + + for _, test := range tests { + pt := models.MustNewPoint("cpu", nil, tags, tm) + act := pt.RoundedString(test.precision) + + if act != test.exp { + t.Errorf("%s: RoundedString() mismatch:\n actual: %v\n exp: %v", + test.name, act, test.exp) + } + } +} + +func TestParsePointsStringWithExtraBuffer(t *testing.T) { + b := make([]byte, 70*5000) + buf := bytes.NewBuffer(b) + key := "cpu,host=A,region=uswest" + buf.WriteString(fmt.Sprintf("%s value=%.3f 1\n", key, rand.Float64())) + + points, err := models.ParsePointsString(buf.String()) + if err != nil { + t.Fatalf("failed to write points: %s", err.Error()) + } + + pointKey := string(points[0].Key()) + + if len(key) != len(pointKey) { + t.Fatalf("expected length of both keys are same but got %d and %d", len(key), len(pointKey)) + } + + if key != pointKey { + t.Fatalf("expected both keys are same but got %s and %s", key, pointKey) + } +} + +func TestParsePointsQuotesInFieldKey(t *testing.T) { + buf := `cpu "a=1 +cpu value=2 1` + points, err := models.ParsePointsString(buf) + if err != nil { + t.Fatalf("failed to write points: %s", err.Error()) + } + + fields, err := points[0].Fields() + if err != nil { + t.Fatal(err) + } + value, ok := fields["\"a"] + if !ok { + t.Fatalf("expected to parse field '\"a'") + } + + if value != float64(1) { + t.Fatalf("expected field value to be 1, got %v", value) + } + + // The following input should not parse + buf = `cpu "\, '= "\ v=1.0` + _, err = models.ParsePointsString(buf) + if err == nil { + t.Fatalf("expected parsing failure but got no error") + } +} + +func TestParsePointsQuotesInTags(t *testing.T) { + buf := `t159,label=hey\ "ya a=1i,value=0i +t159,label=another a=2i,value=1i 1` + points, err := models.ParsePointsString(buf) + if err != nil { + t.Fatalf("failed to write points: %s", err.Error()) + } + + if len(points) != 2 { + t.Fatalf("expected 2 points, got %d", len(points)) + } +} + +func TestParsePointsBlankLine(t *testing.T) { + buf := `cpu value=1i 1000000000 + +cpu value=2i 2000000000` + points, err := models.ParsePointsString(buf) + if err != nil { + t.Fatalf("failed to write points: %s", err.Error()) + } + + if len(points) != 2 { + t.Fatalf("expected 2 points, got %d", len(points)) + } +} + +func TestNewPointsWithBytesWithCorruptData(t *testing.T) { + corrupted := []byte{0, 0, 0, 3, 102, 111, 111, 0, 0, 0, 4, 61, 34, 65, 34, 1, 0, 0, 0, 14, 206, 86, 119, 24, 32, 72, 233, 168, 2, 148} + p, err := models.NewPointFromBytes(corrupted) + if p != nil || err == nil { + t.Fatalf("NewPointFromBytes: got: (%v, %v), expected: (nil, error)", p, err) + } +} + +func TestNewPointsWithShortBuffer(t *testing.T) { + _, err := models.NewPointFromBytes([]byte{0, 0, 0, 3, 4}) + if err != io.ErrShortBuffer { + t.Fatalf("NewPointFromBytes: got: (%v, %v), expected: (nil, error)", p, err) + } +} + +func TestNewPointsRejectsEmptyFieldNames(t *testing.T) { + if _, err := models.NewPoint("foo", nil, models.Fields{"": 1}, time.Now()); err == nil { + t.Fatalf("new point with empty field name. got: nil, expected: error") + } +} + +func TestNewPointsRejectsMaxKey(t *testing.T) { + var key string + // tsm field key is point key, separator (4 bytes) and field + for i := 0; i < models.MaxKeyLength-len("value")-4; i++ { + key += "a" + } + + // Test max key len + if _, err := models.NewPoint(key, nil, models.Fields{"value": 1, "ok": 2.0}, time.Now()); err != nil { + t.Fatalf("new point with max key. got: %v, expected: nil", err) + } + + if _, err := models.ParsePointsString(fmt.Sprintf("%v value=1,ok=2.0", key)); err != nil { + t.Fatalf("parse point with max key. got: %v, expected: nil", err) + } + + // Test 1 byte over max key len + key += "a" + if _, err := models.NewPoint(key, nil, models.Fields{"value": 1, "ok": 2.0}, time.Now()); err == nil { + t.Fatalf("new point with max key. got: nil, expected: error") + } + + if _, err := models.ParsePointsString(fmt.Sprintf("%v value=1,ok=2.0", key)); err == nil { + t.Fatalf("parse point with max key. got: nil, expected: error") + } + +} + +func TestPoint_FieldIterator_Simple(t *testing.T) { + + p, err := models.ParsePoints([]byte(`m v=42i,f=42 36`)) + if err != nil { + t.Fatal(err) + } + + if len(p) != 1 { + t.Fatalf("wrong number of points, got %d, exp %d", len(p), 1) + } + + fi := p[0].FieldIterator() + + if !fi.Next() { + t.Fatal("field iterator terminated before first field") + } + + if fi.Type() != models.Integer { + t.Fatalf("'42i' should be an Integer, got %v", fi.Type()) + } + + iv, err := fi.IntegerValue() + if err != nil { + t.Fatal(err) + } + if exp, got := int64(42), iv; exp != got { + t.Fatalf("'42i' should be %d, got %d", exp, got) + } + + if !fi.Next() { + t.Fatalf("field iterator terminated before second field") + } + + if fi.Type() != models.Float { + t.Fatalf("'42' should be a Float, got %v", fi.Type()) + } + + fv, err := fi.FloatValue() + if err != nil { + t.Fatal(err) + } + if exp, got := 42.0, fv; exp != got { + t.Fatalf("'42' should be %f, got %f", exp, got) + } + + if fi.Next() { + t.Fatal("field iterator didn't terminate") + } +} + +func toFields(fi models.FieldIterator) models.Fields { + m := make(models.Fields) + for fi.Next() { + var v interface{} + var err error + switch fi.Type() { + case models.Float: + v, err = fi.FloatValue() + case models.Integer: + v, err = fi.IntegerValue() + case models.Unsigned: + v, err = fi.UnsignedValue() + case models.String: + v = fi.StringValue() + case models.Boolean: + v, err = fi.BooleanValue() + case models.Empty: + v = nil + default: + panic("unknown type") + } + if err != nil { + panic(err) + } + m[string(fi.FieldKey())] = v + } + return m +} + +func TestPoint_FieldIterator_FieldMap(t *testing.T) { + + points, err := models.ParsePointsString(` +m v=42 +m v=42i +m v="string" +m v=true +m v="string\"with\"escapes" +m v=42i,f=42,g=42.314,u=123u +m a=2i,b=3i,c=true,d="stuff",e=-0.23,f=123.456 +`) + + if err != nil { + t.Fatal("failed to parse test points:", err) + } + + for _, p := range points { + exp, err := p.Fields() + if err != nil { + t.Fatal(err) + } + got := toFields(p.FieldIterator()) + + if !reflect.DeepEqual(got, exp) { + t.Errorf("FieldIterator failed for %#q: got %#v, exp %#v", p.String(), got, exp) + } + } +} + +func TestEscapeStringField(t *testing.T) { + cases := []struct { + in string + expOut string + }{ + {in: "abcdefg", expOut: "abcdefg"}, + {in: `one double quote " .`, expOut: `one double quote \" .`}, + {in: `quote " then backslash \ .`, expOut: `quote \" then backslash \\ .`}, + {in: `backslash \ then quote " .`, expOut: `backslash \\ then quote \" .`}, + } + + for _, c := range cases { + // Unescapes as expected. + got := models.EscapeStringField(c.in) + if got != c.expOut { + t.Errorf("unexpected result from EscapeStringField(%s)\ngot [%s]\nexp [%s]\n", c.in, got, c.expOut) + continue + } + + pointLine := fmt.Sprintf(`t s="%s"`, got) + test(t, pointLine, NewTestPoint( + "t", + models.NewTags(nil), + models.Fields{"s": c.in}, + time.Unix(0, 0), + )) + } +} + +func TestParseKeyBytes(t *testing.T) { + testCases := []struct { + input string + expectedName string + expectedTags map[string]string + }{ + {input: "m,k=v", expectedName: "m", expectedTags: map[string]string{"k": "v"}}, + {input: "m\\ q,k=v", expectedName: "m q", expectedTags: map[string]string{"k": "v"}}, + {input: "m,k\\ q=v", expectedName: "m", expectedTags: map[string]string{"k q": "v"}}, + {input: "m\\ q,k\\ q=v", expectedName: "m q", expectedTags: map[string]string{"k q": "v"}}, + } + + for _, testCase := range testCases { + t.Run(testCase.input, func(t *testing.T) { + name, tags := models.ParseKeyBytes([]byte(testCase.input)) + if !bytes.Equal([]byte(testCase.expectedName), name) { + t.Errorf("%s produced measurement %s but expected %s", testCase.input, string(name), testCase.expectedName) + } + if !tags.Equal(models.NewTags(testCase.expectedTags)) { + t.Errorf("%s produced tags %s but expected %s", testCase.input, tags.String(), models.NewTags(testCase.expectedTags).String()) + } + }) + } +} + +func TestParseName(t *testing.T) { + testCases := []struct { + input string + expectedName string + }{ + {input: "m,k=v", expectedName: "m"}, + {input: "m\\ q,k=v", expectedName: "m q"}, + } + + for _, testCase := range testCases { + t.Run(testCase.input, func(t *testing.T) { + name := models.ParseName([]byte(testCase.input)) + if !bytes.Equal([]byte(testCase.expectedName), name) { + t.Errorf("%s produced measurement %s but expected %s", testCase.input, string(name), testCase.expectedName) + } + }) + } +} + +func BenchmarkEscapeStringField_Plain(b *testing.B) { + s := "nothing special" + for i := 0; i < b.N; i++ { + sink = models.EscapeStringField(s) + } +} + +func BenchmarkEscapeString_Quotes(b *testing.B) { + s := `Hello, "world"` + for i := 0; i < b.N; i++ { + sink = models.EscapeStringField(s) + } +} + +func BenchmarkEscapeString_Backslashes(b *testing.B) { + s := `C:\windows\system32` + for i := 0; i < b.N; i++ { + sink = models.EscapeStringField(s) + } +} + +func BenchmarkEscapeString_QuotesAndBackslashes(b *testing.B) { + s1 := `a quote " then backslash \ .` + s2 := `a backslash \ then quote " .` + for i := 0; i < b.N; i++ { + sink = [...]string{models.EscapeStringField(s1), models.EscapeStringField(s2)} + } +} + +func BenchmarkParseTags(b *testing.B) { + tags := []byte("cpu,tag0=value0,tag1=value1,tag2=value2,tag3=value3,tag4=value4,tag5=value5") + for i := 0; i < b.N; i++ { + models.ParseTags(tags) + } +} + +func BenchmarkEscapeMeasurement(b *testing.B) { + benchmarks := []struct { + m []byte + }{ + {[]byte("this_is_a_test")}, + {[]byte("this,is,a,test")}, + } + + for _, bm := range benchmarks { + b.Run(string(bm.m), func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + models.EscapeMeasurement(bm.m) + } + }) + } +} + +func makeTags(key, val string, n int) models.Tags { + tags := make(models.Tags, n) + for i := range tags { + tags[i].Key = []byte(fmt.Sprintf("%s%03d", key, i)) + tags[i].Value = []byte(fmt.Sprintf("%s%03d", val, i)) + } + return tags +} + +func BenchmarkTags_HashKey(b *testing.B) { + benchmarks := []struct { + name string + t models.Tags + }{ + {"5 tags-no esc", makeTags("tag_foo", "val_bar", 5)}, + {"25 tags-no esc", makeTags("tag_foo", "val_bar", 25)}, + {"5 tags-esc", makeTags("tag foo", "val bar", 5)}, + {"25 tags-esc", makeTags("tag foo", "val bar", 25)}, + } + for _, bm := range benchmarks { + b.Run(bm.name, func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + bm.t.HashKey() + } + }) + } +} + +func BenchmarkMakeKey(b *testing.B) { + benchmarks := []struct { + m []byte + t models.Tags + }{ + {[]byte("this_is_a_test"), nil}, + {[]byte("this,is,a,test"), nil}, + {[]byte(`this\ is\ a\ test`), nil}, + + {[]byte("this_is_a_test"), makeTags("tag_foo", "val_bar", 8)}, + {[]byte("this,is,a,test"), makeTags("tag_foo", "val_bar", 8)}, + {[]byte("this_is_a_test"), makeTags("tag_foo", "val bar", 8)}, + {[]byte("this,is,a,test"), makeTags("tag_foo", "val bar", 8)}, + } + + for _, bm := range benchmarks { + b.Run(string(bm.m), func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + models.MakeKey(bm.m, bm.t) + } + }) + } +} + +func init() { + // Force uint support to be enabled for testing. + models.EnableUintSupport() +} diff --git a/models/rows.go b/models/rows.go new file mode 100644 index 0000000000..c087a4882d --- /dev/null +++ b/models/rows.go @@ -0,0 +1,62 @@ +package models + +import ( + "sort" +) + +// Row represents a single row returned from the execution of a statement. +type Row struct { + Name string `json:"name,omitempty"` + Tags map[string]string `json:"tags,omitempty"` + Columns []string `json:"columns,omitempty"` + Values [][]interface{} `json:"values,omitempty"` + Partial bool `json:"partial,omitempty"` +} + +// SameSeries returns true if r contains values for the same series as o. +func (r *Row) SameSeries(o *Row) bool { + return r.tagsHash() == o.tagsHash() && r.Name == o.Name +} + +// tagsHash returns a hash of tag key/value pairs. +func (r *Row) tagsHash() uint64 { + h := NewInlineFNV64a() + keys := r.tagsKeys() + for _, k := range keys { + h.Write([]byte(k)) + h.Write([]byte(r.Tags[k])) + } + return h.Sum64() +} + +// tagKeys returns a sorted list of tag keys. +func (r *Row) tagsKeys() []string { + a := make([]string, 0, len(r.Tags)) + for k := range r.Tags { + a = append(a, k) + } + sort.Strings(a) + return a +} + +// Rows represents a collection of rows. Rows implements sort.Interface. +type Rows []*Row + +// Len implements sort.Interface. +func (p Rows) Len() int { return len(p) } + +// Less implements sort.Interface. +func (p Rows) Less(i, j int) bool { + // Sort by name first. + if p[i].Name != p[j].Name { + return p[i].Name < p[j].Name + } + + // Sort by tag set hash. Tags don't have a meaningful sort order so we + // just compute a hash and sort by that instead. This allows the tests + // to receive rows in a predictable order every time. + return p[i].tagsHash() < p[j].tagsHash() +} + +// Swap implements sort.Interface. +func (p Rows) Swap(i, j int) { p[i], p[j] = p[j], p[i] } diff --git a/models/statistic.go b/models/statistic.go new file mode 100644 index 0000000000..553e9d09fb --- /dev/null +++ b/models/statistic.go @@ -0,0 +1,42 @@ +package models + +// Statistic is the representation of a statistic used by the monitoring service. +type Statistic struct { + Name string `json:"name"` + Tags map[string]string `json:"tags"` + Values map[string]interface{} `json:"values"` +} + +// NewStatistic returns an initialized Statistic. +func NewStatistic(name string) Statistic { + return Statistic{ + Name: name, + Tags: make(map[string]string), + Values: make(map[string]interface{}), + } +} + +// StatisticTags is a map that can be merged with others without causing +// mutations to either map. +type StatisticTags map[string]string + +// Merge creates a new map containing the merged contents of tags and t. +// If both tags and the receiver map contain the same key, the value in tags +// is used in the resulting map. +// +// Merge always returns a usable map. +func (t StatisticTags) Merge(tags map[string]string) map[string]string { + // Add everything in tags to the result. + out := make(map[string]string, len(tags)) + for k, v := range tags { + out[k] = v + } + + // Only add values from t that don't appear in tags. + for k, v := range t { + if _, ok := tags[k]; !ok { + out[k] = v + } + } + return out +} diff --git a/models/statistic_test.go b/models/statistic_test.go new file mode 100644 index 0000000000..990cef0547 --- /dev/null +++ b/models/statistic_test.go @@ -0,0 +1,55 @@ +package models_test + +import ( + "reflect" + "testing" + + "github.com/influxdata/platform/models" +) + +func TestTags_Merge(t *testing.T) { + examples := []struct { + Base map[string]string + Arg map[string]string + Result map[string]string + }{ + { + Base: nil, + Arg: nil, + Result: map[string]string{}, + }, + { + Base: nil, + Arg: map[string]string{"foo": "foo"}, + Result: map[string]string{"foo": "foo"}, + }, + { + Base: map[string]string{"foo": "foo"}, + Arg: nil, + Result: map[string]string{"foo": "foo"}, + }, + { + Base: map[string]string{"foo": "foo"}, + Arg: map[string]string{"bar": "bar"}, + Result: map[string]string{"foo": "foo", "bar": "bar"}, + }, + { + Base: map[string]string{"foo": "foo", "bar": "bar"}, + Arg: map[string]string{"zoo": "zoo"}, + Result: map[string]string{"foo": "foo", "bar": "bar", "zoo": "zoo"}, + }, + { + Base: map[string]string{"foo": "foo", "bar": "bar"}, + Arg: map[string]string{"bar": "newbar"}, + Result: map[string]string{"foo": "foo", "bar": "newbar"}, + }, + } + + for i, example := range examples { + i++ + result := models.StatisticTags(example.Base).Merge(example.Arg) + if got, exp := result, example.Result; !reflect.DeepEqual(got, exp) { + t.Errorf("[Example %d] got %#v, expected %#v", i, got, exp) + } + } +} diff --git a/models/time.go b/models/time.go new file mode 100644 index 0000000000..e98f2cb336 --- /dev/null +++ b/models/time.go @@ -0,0 +1,74 @@ +package models + +// Helper time methods since parsing time can easily overflow and we only support a +// specific time range. + +import ( + "fmt" + "math" + "time" +) + +const ( + // MinNanoTime is the minumum time that can be represented. + // + // 1677-09-21 00:12:43.145224194 +0000 UTC + // + // The two lowest minimum integers are used as sentinel values. The + // minimum value needs to be used as a value lower than any other value for + // comparisons and another separate value is needed to act as a sentinel + // default value that is unusable by the user, but usable internally. + // Because these two values need to be used for a special purpose, we do + // not allow users to write points at these two times. + MinNanoTime = int64(math.MinInt64) + 2 + + // MaxNanoTime is the maximum time that can be represented. + // + // 2262-04-11 23:47:16.854775806 +0000 UTC + // + // The highest time represented by a nanosecond needs to be used for an + // exclusive range in the shard group, so the maximum time needs to be one + // less than the possible maximum number of nanoseconds representable by an + // int64 so that we don't lose a point at that one time. + MaxNanoTime = int64(math.MaxInt64) - 1 +) + +var ( + minNanoTime = time.Unix(0, MinNanoTime).UTC() + maxNanoTime = time.Unix(0, MaxNanoTime).UTC() + + // ErrTimeOutOfRange gets returned when time is out of the representable range using int64 nanoseconds since the epoch. + ErrTimeOutOfRange = fmt.Errorf("time outside range %d - %d", MinNanoTime, MaxNanoTime) +) + +// SafeCalcTime safely calculates the time given. Will return error if the time is outside the +// supported range. +func SafeCalcTime(timestamp int64, precision string) (time.Time, error) { + mult := GetPrecisionMultiplier(precision) + if t, ok := safeSignedMult(timestamp, mult); ok { + tme := time.Unix(0, t).UTC() + return tme, CheckTime(tme) + } + + return time.Time{}, ErrTimeOutOfRange +} + +// CheckTime checks that a time is within the safe range. +func CheckTime(t time.Time) error { + if t.Before(minNanoTime) || t.After(maxNanoTime) { + return ErrTimeOutOfRange + } + return nil +} + +// Perform the multiplication and check to make sure it didn't overflow. +func safeSignedMult(a, b int64) (int64, bool) { + if a == 0 || b == 0 || a == 1 || b == 1 { + return a * b, true + } + if a == MinNanoTime || b == MaxNanoTime { + return 0, false + } + c := a * b + return c, c/b == a +} diff --git a/models/uint_support.go b/models/uint_support.go new file mode 100644 index 0000000000..18d1ca06e2 --- /dev/null +++ b/models/uint_support.go @@ -0,0 +1,7 @@ +// +build uint uint64 + +package models + +func init() { + EnableUintSupport() +} diff --git a/tsdb/cursor.go b/tsdb/cursor.go index a445d50097..97a932ff62 100644 --- a/tsdb/cursor.go +++ b/tsdb/cursor.go @@ -3,8 +3,8 @@ package tsdb import ( "context" - "github.com/influxdata/influxdb/models" "github.com/influxdata/influxdb/query" + "github.com/influxdata/platform/models" ) // EOF represents a "not found" key returned by a Cursor. diff --git a/tsdb/engine.go b/tsdb/engine.go index ff097732d0..24350c4931 100644 --- a/tsdb/engine.go +++ b/tsdb/engine.go @@ -11,11 +11,11 @@ import ( "sort" "time" - "github.com/influxdata/influxdb/models" "github.com/influxdata/influxdb/pkg/estimator" "github.com/influxdata/influxdb/pkg/limiter" "github.com/influxdata/influxdb/query" "github.com/influxdata/influxql" + "github.com/influxdata/platform/models" "go.uber.org/zap" ) diff --git a/tsdb/field_validator.go b/tsdb/field_validator.go index 596a2c7448..29f62ae8d4 100644 --- a/tsdb/field_validator.go +++ b/tsdb/field_validator.go @@ -4,8 +4,8 @@ import ( "bytes" "fmt" - "github.com/influxdata/influxdb/models" "github.com/influxdata/influxql" + "github.com/influxdata/platform/models" ) // FieldValidator should return a PartialWriteError if the point should not be written. diff --git a/tsdb/index.go b/tsdb/index.go index f039af73d5..b4b5adafa1 100644 --- a/tsdb/index.go +++ b/tsdb/index.go @@ -7,10 +7,10 @@ import ( "regexp" "sort" - "github.com/influxdata/influxdb/models" "github.com/influxdata/influxdb/pkg/estimator" "github.com/influxdata/influxdb/query" "github.com/influxdata/influxql" + "github.com/influxdata/platform/models" "go.uber.org/zap" ) diff --git a/tsdb/meta.go b/tsdb/meta.go index 21a06d55f4..0cbff5a2f0 100644 --- a/tsdb/meta.go +++ b/tsdb/meta.go @@ -3,7 +3,7 @@ package tsdb //go:generate protoc --gogo_out=. internal/meta.proto import ( - "github.com/influxdata/influxdb/models" + "github.com/influxdata/platform/models" ) // MakeTagsKey converts a tag set to bytes for use as a lookup key. diff --git a/tsdb/series_collection.go b/tsdb/series_collection.go index 371ebef819..2bd5c0ddf8 100644 --- a/tsdb/series_collection.go +++ b/tsdb/series_collection.go @@ -5,8 +5,8 @@ import ( "sync/atomic" "unsafe" - "github.com/influxdata/influxdb/models" "github.com/influxdata/influxdb/pkg/bytesutil" + "github.com/influxdata/platform/models" ) // SeriesCollection is a struct of arrays representation of a collection of series that allows diff --git a/tsdb/series_cursor.go b/tsdb/series_cursor.go index 6a7c722535..c06a257c4f 100644 --- a/tsdb/series_cursor.go +++ b/tsdb/series_cursor.go @@ -5,8 +5,8 @@ import ( "sort" "sync" - "github.com/influxdata/influxdb/models" "github.com/influxdata/influxql" + "github.com/influxdata/platform/models" ) type SeriesCursor interface { diff --git a/tsdb/series_file.go b/tsdb/series_file.go index 7bb2796435..0303e19bea 100644 --- a/tsdb/series_file.go +++ b/tsdb/series_file.go @@ -10,8 +10,8 @@ import ( "sync" "github.com/cespare/xxhash" - "github.com/influxdata/influxdb/models" "github.com/influxdata/influxdb/pkg/binaryutil" + "github.com/influxdata/platform/models" "go.uber.org/zap" "golang.org/x/sync/errgroup" ) diff --git a/tsdb/series_id.go b/tsdb/series_id.go index 90632e9752..b85e67d1d8 100644 --- a/tsdb/series_id.go +++ b/tsdb/series_id.go @@ -3,7 +3,7 @@ package tsdb import ( "unsafe" - "github.com/influxdata/influxdb/models" + "github.com/influxdata/platform/models" ) const ( diff --git a/tsdb/series_partition.go b/tsdb/series_partition.go index 25c639cdf5..e1ec097724 100644 --- a/tsdb/series_partition.go +++ b/tsdb/series_partition.go @@ -10,8 +10,8 @@ import ( "sync" "github.com/influxdata/influxdb/logger" - "github.com/influxdata/influxdb/models" "github.com/influxdata/influxdb/pkg/rhh" + "github.com/influxdata/platform/models" "go.uber.org/zap" ) diff --git a/tsdb/shard.go b/tsdb/shard.go index 9e6f7829b1..3caa6a4967 100644 --- a/tsdb/shard.go +++ b/tsdb/shard.go @@ -6,8 +6,8 @@ import ( "path/filepath" "sync" - "github.com/influxdata/influxdb/models" "github.com/influxdata/influxql" + "github.com/influxdata/platform/models" "go.uber.org/zap" )