diff --git a/models/points.go b/models/points.go index 3d0c61e0b8..a8098a1cb1 100644 --- a/models/points.go +++ b/models/points.go @@ -43,6 +43,16 @@ const ( MaxKeyLength = 65535 ) +// enableUint64Support will enable uint64 support if set to true. +var enableUint64Support = false + +// EnableUintSupport manually enables uint support for the point parser. +// This function will be removed in the future and only exists for unit tests during the +// transition. +func EnableUintSupport() { + enableUint64Support = true +} + // Point defines the values that will be written to the database. type Point interface { // Name return the measurement name for the point. @@ -224,6 +234,9 @@ const ( // the number of characters for the smallest possible int64 (-9223372036854775808) minInt64Digits = 20 + // the number of characters for the largest possible uint64 (18446744073709551615) + maxUint64Digits = 20 + // the number of characters required for the largest float64 before a range check // would occur during parsing maxFloat64Digits = 25 @@ -827,7 +840,7 @@ func isNumeric(b byte) bool { // error if a invalid number is scanned. func scanNumber(buf []byte, i int) (int, error) { start := i - var isInt bool + var isInt, isUnsigned bool // Is negative number? if i < len(buf) && buf[i] == '-' { @@ -853,10 +866,14 @@ func scanNumber(buf []byte, i int) (int, error) { break } - if buf[i] == 'i' && i > start && !isInt { + if buf[i] == 'i' && i > start && !(isInt || isUnsigned) { isInt = true i++ continue + } else if buf[i] == 'u' && i > start && !(isInt || isUnsigned) { + isUnsigned = true + i++ + continue } if buf[i] == '.' { @@ -891,7 +908,7 @@ func scanNumber(buf []byte, i int) (int, error) { i++ } - if isInt && (decimal || scientific) { + if (isInt || isUnsigned) && (decimal || scientific) { return i, ErrInvalidNumber } @@ -926,6 +943,26 @@ func scanNumber(buf []byte, i int) (int, error) { return i, fmt.Errorf("unable to parse integer %s: %s", buf[start:i-1], err) } } + } else if isUnsigned { + // Return an error if uint64 support has not been enabled. + if !enableUint64Support { + return i, ErrInvalidNumber + } + // Make sure the last char is a 'u' for unsigned + if buf[i-1] != 'u' { + return i, ErrInvalidNumber + } + // Make sure the first char is not a '-' for unsigned + if buf[start] == '-' { + return i, ErrInvalidNumber + } + // Parse the uint to check bounds the number of digits could be larger than the max range + // We subtract 1 from the index to remove the `u` from our tests + if len(buf[start:i-1]) >= maxUint64Digits { + if _, err := parseUintBytes(buf[start:i-1], 10, 64); err != nil { + return i, fmt.Errorf("unable to parse unsigned %s: %s", buf[start:i-1], err) + } + } } else { // Parse the float to check bounds if it's scientific or the number of digits could be larger than the max range if scientific || len(buf[start:i]) >= maxFloat64Digits || len(buf[start:i]) >= minFloat64Digits { @@ -2118,10 +2155,13 @@ func (p *point) Next() bool { return true } - if strings.IndexByte(`0123456789-.nNiI`, c) >= 0 { + if strings.IndexByte(`0123456789-.nNiIu`, c) >= 0 { if p.it.valueBuf[len(p.it.valueBuf)-1] == 'i' { p.it.fieldType = Integer p.it.valueBuf = p.it.valueBuf[:len(p.it.valueBuf)-1] + } else if p.it.valueBuf[len(p.it.valueBuf)-1] == 'u' { + p.it.fieldType = Unsigned + p.it.valueBuf = p.it.valueBuf[:len(p.it.valueBuf)-1] } else { p.it.fieldType = Float } diff --git a/models/points_test.go b/models/points_test.go index f1deb3bd65..e2e25a1c92 100644 --- a/models/points_test.go +++ b/models/points_test.go @@ -564,6 +564,7 @@ func TestParsePointBadNumber(t *testing.T) { "cpu v=-e-e-e ", "cpu v=42+3 ", "cpu v= ", + "cpu v=-123u", } { _, err := models.ParsePointsString(tt) if err == nil { @@ -608,10 +609,17 @@ func TestParsePointMinInt64(t *testing.T) { } // min int - _, err = models.ParsePointsString(`cpu,host=serverA,region=us-west value=-9223372036854775808i`) + p, err := models.ParsePointsString(`cpu,host=serverA,region=us-west value=-9223372036854775808i`) if err != nil { t.Errorf(`ParsePoints("%s") mismatch. got %v, exp nil`, `cpu,host=serverA,region=us-west value=-9223372036854775808i`, err) } + fields, err := p[0].Fields() + if err != nil { + t.Fatal(err) + } + if exp, got := int64(-9223372036854775808), fields["value"].(int64); exp != got { + t.Fatalf("ParsePoints Value mismatch. \nexp: %v\ngot: %v", exp, got) + } // leading zeros _, err = models.ParsePointsString(`cpu,host=serverA,region=us-west value=-0009223372036854775808i`) @@ -628,10 +636,17 @@ func TestParsePointMaxFloat64(t *testing.T) { } // max float - _, err = models.ParsePointsString(fmt.Sprintf(`cpu,host=serverA,region=us-west value=%s`, string(maxFloat64))) + p, err := models.ParsePointsString(fmt.Sprintf(`cpu,host=serverA,region=us-west value=%s`, string(maxFloat64))) if err != nil { t.Errorf(`ParsePoints("%s") mismatch. got %v, exp nil`, `cpu,host=serverA,region=us-west value=9223372036854775807`, err) } + fields, err := p[0].Fields() + if err != nil { + t.Fatal(err) + } + if exp, got := math.MaxFloat64, fields["value"].(float64); exp != got { + t.Fatalf("ParsePoints Value mismatch. \nexp: %v\ngot: %v", exp, got) + } // leading zeros _, err = models.ParsePointsString(fmt.Sprintf(`cpu,host=serverA,region=us-west value=%s`, "0000"+string(maxFloat64))) @@ -648,10 +663,17 @@ func TestParsePointMinFloat64(t *testing.T) { } // min float - _, err = models.ParsePointsString(fmt.Sprintf(`cpu,host=serverA,region=us-west value=%s`, string(minFloat64))) + p, err := models.ParsePointsString(fmt.Sprintf(`cpu,host=serverA,region=us-west value=%s`, string(minFloat64))) if err != nil { t.Errorf(`ParsePoints("%s") mismatch. got %v, exp nil`, `cpu,host=serverA,region=us-west value=...`, err) } + fields, err := p[0].Fields() + if err != nil { + t.Fatal(err) + } + if exp, got := -math.MaxFloat64, fields["value"].(float64); exp != got { + t.Fatalf("ParsePoints Value mismatch. \nexp: %v\ngot: %v", exp, got) + } // leading zeros _, err = models.ParsePointsString(fmt.Sprintf(`cpu,host=serverA,region=us-west value=%s`, "-0000000"+string(minFloat64)[1:])) @@ -660,6 +682,61 @@ func TestParsePointMinFloat64(t *testing.T) { } } +func TestParsePointMaxUint64(t *testing.T) { + // out of range + _, err := models.ParsePointsString(`cpu,host=serverA,region=us-west value=18446744073709551616u`) + exp := `unable to parse 'cpu,host=serverA,region=us-west value=18446744073709551616u': unable to parse unsigned 18446744073709551616: strconv.ParseUint: parsing "18446744073709551616": value out of range` + if err == nil || (err != nil && err.Error() != exp) { + t.Fatalf("Error mismatch:\nexp: %s\ngot: %v", exp, err) + } + + // max int + p, err := models.ParsePointsString(`cpu,host=serverA,region=us-west value=18446744073709551615u`) + if err != nil { + t.Fatalf(`ParsePoints("%s") mismatch. got %v, exp nil`, `cpu,host=serverA,region=us-west value=18446744073709551615u`, err) + } + fields, err := p[0].Fields() + if err != nil { + t.Fatal(err) + } + if exp, got := uint64(18446744073709551615), fields["value"].(uint64); exp != got { + t.Fatalf("ParsePoints Value mismatch. \nexp: %v\ngot: %v", exp, got) + } + + // leading zeros + _, err = models.ParsePointsString(`cpu,host=serverA,region=us-west value=00018446744073709551615u`) + if err != nil { + t.Fatalf(`ParsePoints("%s") mismatch. got %v, exp nil`, `cpu,host=serverA,region=us-west value=00018446744073709551615u`, err) + } +} + +func TestParsePointMinUint64(t *testing.T) { + // out of range + _, err := models.ParsePointsString(`cpu,host=serverA,region=us-west value=--1u`) + if err == nil { + t.Errorf(`ParsePoints("%s") mismatch. got nil, exp error`, `cpu,host=serverA,region=us-west value=-1u`) + } + + // min int + p, err := models.ParsePointsString(`cpu,host=serverA,region=us-west value=0u`) + if err != nil { + t.Errorf(`ParsePoints("%s") mismatch. got %v, exp nil`, `cpu,host=serverA,region=us-west value=0u`, err) + } + fields, err := p[0].Fields() + if err != nil { + t.Fatal(err) + } + if exp, got := uint64(0), fields["value"].(uint64); exp != got { + t.Fatalf("ParsePoints Value mismatch. \nexp: %v\ngot: %v", exp, got) + } + + // leading zeros + _, err = models.ParsePointsString(`cpu,host=serverA,region=us-west value=0000u`) + if err != nil { + t.Errorf(`ParsePoints("%s") mismatch. got %v, exp nil`, `cpu,host=serverA,region=us-west value=0000u`, err) + } +} + func TestParsePointNumberNonNumeric(t *testing.T) { _, err := models.ParsePointsString(`cpu,host=serverA,region=us-west value=.1a`) if err == nil { @@ -2189,6 +2266,8 @@ func toFields(fi models.FieldIterator) models.Fields { v, err = fi.FloatValue() case models.Integer: v, err = fi.IntegerValue() + case models.Unsigned: + v, err = fi.UnsignedValue() case models.String: v = fi.StringValue() case models.Boolean: @@ -2214,7 +2293,7 @@ m v=42i m v="string" m v=true m v="string\"with\"escapes" -m v=42i,f=42,g=42.314 +m v=42i,f=42,g=42.314,u=123u m a=2i,b=3i,c=true,d="stuff",e=-0.23,f=123.456 `) @@ -2292,3 +2371,8 @@ func BenchmarkEscapeString_QuotesAndBackslashes(b *testing.B) { sink = [...]string{models.EscapeStringField(s1), models.EscapeStringField(s2)} } } + +func init() { + // Force uint support to be enabled for testing. + models.EnableUintSupport() +} diff --git a/models/uint_support.go b/models/uint_support.go new file mode 100644 index 0000000000..18d1ca06e2 --- /dev/null +++ b/models/uint_support.go @@ -0,0 +1,7 @@ +// +build uint uint64 + +package models + +func init() { + EnableUintSupport() +}