pull/5382/head
Edd Robinson 2016-01-17 19:31:01 +00:00
parent 0e0f85a0c1
commit 789aff9b23
3 changed files with 139 additions and 72 deletions

View File

@ -26,6 +26,7 @@
- [#5193](https://github.com/influxdata/influxdb/issues/5193): Missing data a minute before current time. Comes back later.
- [#5350](https://github.com/influxdata/influxdb/issues/5350): 'influxd backup' should create backup directory
- [#5262](https://github.com/influxdata/influxdb/issues/5262): Fix a panic when a tag value was empty.
- [#5382](https://github.com/influxdata/influxdb/pull/5382): Fixes some escaping bugs with tag keys and values.
## v0.9.6 [2015-12-09]

View File

@ -876,8 +876,10 @@ func scanLine(buf []byte, i int) (int, []byte) {
}
// scanTo returns the end position in buf and the next consecutive block
// of bytes, starting from i and ending with stop byte. If there are leading
// spaces or escaped chars, they are skipped.
// of bytes, starting from i and ending with stop byte, where stop byte
// has not been escaped.
//
// If there are leading spaces, they are skipped.
func scanTo(buf []byte, i int, stop byte) (int, []byte) {
start := i
for {
@ -886,13 +888,8 @@ func scanTo(buf []byte, i int, stop byte) (int, []byte) {
break
}
if buf[i] == '\\' {
i += 2
continue
}
// reached end of block?
if buf[i] == stop {
if buf[i] == stop && buf[i-1] != '\\' {
break
}
i++
@ -935,12 +932,7 @@ func scanTagValue(buf []byte, i int) (int, []byte) {
break
}
if buf[i] == '\\' {
i += 2
continue
}
if buf[i] == ',' {
if buf[i] == ',' && buf[i-1] != '\\' {
break
}
i++

View File

@ -112,7 +112,33 @@ func BenchmarkParsePointsTagsUnSorted10(b *testing.B) {
}
}
func test(t *testing.T, line string, point models.Point) {
// TestPoint wraps a models.Point but also makes available the raw
// arguments to the Point.
//
// This is useful for ensuring that comparisons between results of
// operations on Points match the expected input data to the Point,
// since models.Point does not expose the raw input data (e.g., tags)
// via its API.
type TestPoint struct {
RawFields models.Fields
RawTags models.Tags
RawTime time.Time
models.Point
}
// NewTestPoint returns a new TestPoint.
//
// NewTestPoint panics if it is not a valid models.Point.
func NewTestPoint(name string, tags models.Tags, fields models.Fields, time time.Time) TestPoint {
return TestPoint{
RawTags: tags,
RawFields: fields,
RawTime: time,
Point: models.MustNewPoint(name, tags, fields, time),
}
}
func test(t *testing.T, line string, point TestPoint) {
pts, err := models.ParsePointsWithPrecision([]byte(line), time.Unix(0, 0), "n")
if err != nil {
t.Fatalf(`ParsePoints("%s") mismatch. got %v, exp nil`, line, err)
@ -130,13 +156,13 @@ func test(t *testing.T, line string, point models.Point) {
t.Errorf(`ParsePoints("%s") tags mismatch. got %v, exp %v`, line, pts[0].Tags(), exp)
}
for tag, value := range point.Tags() {
if pts[0].Tags()[tag] != value {
t.Errorf(`ParsePoints("%s") tags mismatch. got %v, exp %v`, line, pts[0].Tags()[tag], value)
for tag, value := range pts[0].Tags() {
if value != point.RawTags[tag] {
t.Errorf(`ParsePoints("%s") tags mismatch. got %v, exp %v`, line, value, point.RawTags[tag])
}
}
for name, value := range point.Fields() {
for name, value := range point.RawFields {
val := pts[0].Fields()[name]
expfval, ok := val.(float64)
@ -146,7 +172,7 @@ func test(t *testing.T, line string, point models.Point) {
t.Errorf(`ParsePoints("%s") field '%s' mismatch. exp NaN`, line, name)
}
} else if !reflect.DeepEqual(pts[0].Fields()[name], value) {
t.Errorf(`ParsePoints("%s") field '%s' mismatch. got %v, exp %v`, line, name, pts[0].Fields()[name], value)
t.Errorf(`ParsePoints("%s") field '%s' mismatch. got %[3]v (%[3]T), exp %[4]v (%[4]T)`, line, name, pts[0].Fields()[name], value)
}
}
@ -201,7 +227,7 @@ func TestParsePointNoFields(t *testing.T) {
}
func TestParsePointNoTimestamp(t *testing.T) {
test(t, "cpu value=1", models.MustNewPoint("cpu", nil, models.Fields{"value": 1.0}, time.Unix(0, 0)))
test(t, "cpu value=1", NewTestPoint("cpu", nil, models.Fields{"value": 1.0}, time.Unix(0, 0)))
}
func TestParsePointMissingQuote(t *testing.T) {
@ -556,21 +582,22 @@ func TestParsePointScientificIntInvalid(t *testing.T) {
}
func TestParsePointUnescape(t *testing.T) {
// commas in measurement name
test(t, `foo\,bar value=1i`,
models.MustNewPoint(
NewTestPoint(
"foo,bar", // comma in the name
models.Tags{},
models.Fields{
"value": 1,
"value": int64(1),
},
time.Unix(0, 0)))
// commas in measurement name
test(t, `cpu\,main,regions=east\,west value=1.0`,
models.MustNewPoint(
// comma in measurement name with tags
test(t, `cpu\,main,regions=east value=1.0`,
NewTestPoint(
"cpu,main", // comma in the name
models.Tags{
"regions": "east,west",
"regions": "east",
},
models.Fields{
"value": 1.0,
@ -579,7 +606,7 @@ func TestParsePointUnescape(t *testing.T) {
// spaces in measurement name
test(t, `cpu\ load,region=east value=1.0`,
models.MustNewPoint(
NewTestPoint(
"cpu load", // space in the name
models.Tags{
"region": "east",
@ -591,7 +618,7 @@ func TestParsePointUnescape(t *testing.T) {
// equals in measurement name
test(t, `cpu\=load,region=east value=1.0`,
models.MustNewPoint(
NewTestPoint(
`cpu\=load`, // backslash is literal
models.Tags{
"region": "east",
@ -603,7 +630,7 @@ func TestParsePointUnescape(t *testing.T) {
// equals in measurement name
test(t, `cpu=load,region=east value=1.0`,
models.MustNewPoint(
NewTestPoint(
`cpu=load`, // literal equals is fine in measurement name
models.Tags{
"region": "east",
@ -615,7 +642,7 @@ func TestParsePointUnescape(t *testing.T) {
// commas in tag names
test(t, `cpu,region\,zone=east value=1.0`,
models.MustNewPoint("cpu",
NewTestPoint("cpu",
models.Tags{
"region,zone": "east", // comma in the tag key
},
@ -626,7 +653,7 @@ func TestParsePointUnescape(t *testing.T) {
// spaces in tag name
test(t, `cpu,region\ zone=east value=1.0`,
models.MustNewPoint("cpu",
NewTestPoint("cpu",
models.Tags{
"region zone": "east", // space in the tag name
},
@ -635,9 +662,20 @@ func TestParsePointUnescape(t *testing.T) {
},
time.Unix(0, 0)))
// backslash with escaped equals in tag name
test(t, `cpu,reg\\=ion=east value=1.0`,
NewTestPoint("cpu",
models.Tags{
`reg\=ion`: "east",
},
models.Fields{
"value": 1.0,
},
time.Unix(0, 0)))
// space is tag name
test(t, `cpu,\ =east value=1.0`,
models.MustNewPoint("cpu",
NewTestPoint("cpu",
models.Tags{
" ": "east", // tag name is single space
},
@ -648,7 +686,7 @@ func TestParsePointUnescape(t *testing.T) {
// commas in tag values
test(t, `cpu,regions=east\,west value=1.0`,
models.MustNewPoint("cpu",
NewTestPoint("cpu",
models.Tags{
"regions": "east,west", // comma in the tag value
},
@ -657,9 +695,45 @@ func TestParsePointUnescape(t *testing.T) {
},
time.Unix(0, 0)))
// backslash literal followed by escaped space
test(t, `cpu,regions=\\ east value=1.0`,
NewTestPoint(
"cpu",
models.Tags{
"regions": `\ east`,
},
models.Fields{
"value": 1.0,
},
time.Unix(0, 0)))
// backslash literal followed by escaped space
test(t, `cpu,regions=eas\\ t value=1.0`,
NewTestPoint(
"cpu",
models.Tags{
"regions": `eas\ t`,
},
models.Fields{
"value": 1.0,
},
time.Unix(0, 0)))
// backslash literal followed by trailing space
test(t, `cpu,regions=east\\ value=1.0`,
NewTestPoint(
"cpu",
models.Tags{
"regions": `east\ `,
},
models.Fields{
"value": 1.0,
},
time.Unix(0, 0)))
// spaces in tag values
test(t, `cpu,regions=east\ west value=1.0`,
models.MustNewPoint("cpu",
NewTestPoint("cpu",
models.Tags{
"regions": "east west", // comma in the tag value
},
@ -670,7 +744,7 @@ func TestParsePointUnescape(t *testing.T) {
// commas in field keys
test(t, `cpu,regions=east value\,ms=1.0`,
models.MustNewPoint("cpu",
NewTestPoint("cpu",
models.Tags{
"regions": "east",
},
@ -681,7 +755,7 @@ func TestParsePointUnescape(t *testing.T) {
// spaces in field keys
test(t, `cpu,regions=east value\ ms=1.0`,
models.MustNewPoint("cpu",
NewTestPoint("cpu",
models.Tags{
"regions": "east",
},
@ -692,7 +766,7 @@ func TestParsePointUnescape(t *testing.T) {
// tag with no value
test(t, `cpu,regions=east value="1"`,
models.MustNewPoint("cpu",
NewTestPoint("cpu",
models.Tags{
"regions": "east",
"foobar": "",
@ -704,7 +778,7 @@ func TestParsePointUnescape(t *testing.T) {
// commas in field values
test(t, `cpu,regions=east value="1,0"`,
models.MustNewPoint("cpu",
NewTestPoint("cpu",
models.Tags{
"regions": "east",
},
@ -715,7 +789,7 @@ func TestParsePointUnescape(t *testing.T) {
// random character escaped
test(t, `cpu,regions=eas\t value=1.0`,
models.MustNewPoint(
NewTestPoint(
"cpu",
models.Tags{
"regions": "eas\\t",
@ -727,7 +801,7 @@ func TestParsePointUnescape(t *testing.T) {
// backslash literal followed by escaped characters
test(t, `cpu,regions=\\,\,\=east value=1.0`,
models.MustNewPoint(
NewTestPoint(
"cpu",
models.Tags{
"regions": `\,,=east`,
@ -739,23 +813,23 @@ func TestParsePointUnescape(t *testing.T) {
// field keys using escape char.
test(t, `cpu \a=1i`,
models.MustNewPoint(
NewTestPoint(
"cpu",
models.Tags{},
models.Fields{
"\\a": 1, // Left as parsed since it's not a known escape sequence.
"\\a": int64(1), // Left as parsed since it's not a known escape sequence.
},
time.Unix(0, 0)))
// measurement, tag and tag value with equals
test(t, `cpu=load,equals\=foo=tag\=value value=1i`,
models.MustNewPoint(
NewTestPoint(
"cpu=load", // Not escaped
models.Tags{
"equals=foo": "tag=value", // Tag and value unescaped
},
models.Fields{
"value": 1,
"value": int64(1),
},
time.Unix(0, 0)))
@ -764,7 +838,7 @@ func TestParsePointUnescape(t *testing.T) {
func TestParsePointWithTags(t *testing.T) {
test(t,
"cpu,host=serverA,region=us-east value=1.0 1000000000",
models.MustNewPoint("cpu",
NewTestPoint("cpu",
models.Tags{"host": "serverA", "region": "us-east"},
models.Fields{"value": 1.0}, time.Unix(1, 0)))
}
@ -778,7 +852,7 @@ func TestParsPointWithDuplicateTags(t *testing.T) {
func TestParsePointWithStringField(t *testing.T) {
test(t, `cpu,host=serverA,region=us-east value=1.0,str="foo",str2="bar" 1000000000`,
models.MustNewPoint("cpu",
NewTestPoint("cpu",
models.Tags{
"host": "serverA",
"region": "us-east",
@ -792,7 +866,7 @@ func TestParsePointWithStringField(t *testing.T) {
)
test(t, `cpu,host=serverA,region=us-east str="foo \" bar" 1000000000`,
models.MustNewPoint("cpu",
NewTestPoint("cpu",
models.Tags{
"host": "serverA",
"region": "us-east",
@ -807,7 +881,7 @@ func TestParsePointWithStringField(t *testing.T) {
func TestParsePointWithStringWithSpaces(t *testing.T) {
test(t, `cpu,host=serverA,region=us-east value=1.0,str="foo bar" 1000000000`,
models.MustNewPoint(
NewTestPoint(
"cpu",
models.Tags{
"host": "serverA",
@ -823,7 +897,7 @@ func TestParsePointWithStringWithSpaces(t *testing.T) {
func TestParsePointWithStringWithNewline(t *testing.T) {
test(t, "cpu,host=serverA,region=us-east value=1.0,str=\"foo\nbar\" 1000000000",
models.MustNewPoint(
NewTestPoint(
"cpu",
models.Tags{
"host": "serverA",
@ -840,7 +914,7 @@ func TestParsePointWithStringWithNewline(t *testing.T) {
func TestParsePointWithStringWithCommas(t *testing.T) {
// escaped comma
test(t, `cpu,host=serverA,region=us-east value=1.0,str="foo\,bar" 1000000000`,
models.MustNewPoint(
NewTestPoint(
"cpu",
models.Tags{
"host": "serverA",
@ -855,7 +929,7 @@ func TestParsePointWithStringWithCommas(t *testing.T) {
// non-escaped comma
test(t, `cpu,host=serverA,region=us-east value=1.0,str="foo,bar" 1000000000`,
models.MustNewPoint(
NewTestPoint(
"cpu",
models.Tags{
"host": "serverA",
@ -872,7 +946,7 @@ func TestParsePointWithStringWithCommas(t *testing.T) {
func TestParsePointQuotedMeasurement(t *testing.T) {
// non-escaped comma
test(t, `"cpu",host=serverA,region=us-east value=1.0 1000000000`,
models.MustNewPoint(
NewTestPoint(
`"cpu"`,
models.Tags{
"host": "serverA",
@ -887,7 +961,7 @@ func TestParsePointQuotedMeasurement(t *testing.T) {
func TestParsePointQuotedTags(t *testing.T) {
test(t, `cpu,"host"="serverA",region=us-east value=1.0 1000000000`,
models.MustNewPoint(
NewTestPoint(
"cpu",
models.Tags{
`"host"`: `"serverA"`,
@ -931,7 +1005,7 @@ func TestParsePointsUnbalancedQuotedTags(t *testing.T) {
func TestParsePointEscapedStringsAndCommas(t *testing.T) {
// non-escaped comma and quotes
test(t, `cpu,host=serverA,region=us-east value="{Hello\"{,}\" World}" 1000000000`,
models.MustNewPoint(
NewTestPoint(
"cpu",
models.Tags{
"host": "serverA",
@ -945,7 +1019,7 @@ func TestParsePointEscapedStringsAndCommas(t *testing.T) {
// escaped comma and quotes
test(t, `cpu,host=serverA,region=us-east value="{Hello\"{\,}\" World}" 1000000000`,
models.MustNewPoint(
NewTestPoint(
"cpu",
models.Tags{
"host": "serverA",
@ -960,7 +1034,7 @@ func TestParsePointEscapedStringsAndCommas(t *testing.T) {
func TestParsePointWithStringWithEquals(t *testing.T) {
test(t, `cpu,host=serverA,region=us-east str="foo=bar",value=1.0 1000000000`,
models.MustNewPoint(
NewTestPoint(
"cpu",
models.Tags{
"host": "serverA",
@ -976,7 +1050,7 @@ func TestParsePointWithStringWithEquals(t *testing.T) {
func TestParsePointWithStringWithBackslash(t *testing.T) {
test(t, `cpu value="test\\\"" 1000000000`,
models.MustNewPoint(
NewTestPoint(
"cpu",
models.Tags{},
models.Fields{
@ -986,7 +1060,7 @@ func TestParsePointWithStringWithBackslash(t *testing.T) {
)
test(t, `cpu value="test\\" 1000000000`,
models.MustNewPoint(
NewTestPoint(
"cpu",
models.Tags{},
models.Fields{
@ -996,7 +1070,7 @@ func TestParsePointWithStringWithBackslash(t *testing.T) {
)
test(t, `cpu value="test\\\"" 1000000000`,
models.MustNewPoint(
NewTestPoint(
"cpu",
models.Tags{},
models.Fields{
@ -1006,7 +1080,7 @@ func TestParsePointWithStringWithBackslash(t *testing.T) {
)
test(t, `cpu value="test\"" 1000000000`,
models.MustNewPoint(
NewTestPoint(
"cpu",
models.Tags{},
models.Fields{
@ -1018,7 +1092,7 @@ func TestParsePointWithStringWithBackslash(t *testing.T) {
func TestParsePointWithBoolField(t *testing.T) {
test(t, `cpu,host=serverA,region=us-east true=true,t=t,T=T,TRUE=TRUE,True=True,false=false,f=f,F=F,FALSE=FALSE,False=False 1000000000`,
models.MustNewPoint(
NewTestPoint(
"cpu",
models.Tags{
"host": "serverA",
@ -1042,7 +1116,7 @@ func TestParsePointWithBoolField(t *testing.T) {
func TestParsePointUnicodeString(t *testing.T) {
test(t, `cpu,host=serverA,region=us-east value="wè" 1000000000`,
models.MustNewPoint(
NewTestPoint(
"cpu",
models.Tags{
"host": "serverA",
@ -1057,7 +1131,7 @@ func TestParsePointUnicodeString(t *testing.T) {
func TestParsePointNegativeTimestamp(t *testing.T) {
test(t, `cpu value=1 -1`,
models.MustNewPoint(
NewTestPoint(
"cpu",
models.Tags{},
models.Fields{
@ -1069,7 +1143,7 @@ func TestParsePointNegativeTimestamp(t *testing.T) {
func TestParsePointMaxTimestamp(t *testing.T) {
test(t, `cpu value=1 9223372036854775807`,
models.MustNewPoint(
NewTestPoint(
"cpu",
models.Tags{},
models.Fields{
@ -1081,7 +1155,7 @@ func TestParsePointMaxTimestamp(t *testing.T) {
func TestParsePointMinTimestamp(t *testing.T) {
test(t, `cpu value=1 -9223372036854775807`,
models.MustNewPoint(
NewTestPoint(
"cpu",
models.Tags{},
models.Fields{
@ -1120,7 +1194,7 @@ func TestParsePointInvalidTimestamp(t *testing.T) {
func TestNewPointFloatWithoutDecimal(t *testing.T) {
test(t, `cpu value=1 1000000000`,
models.MustNewPoint(
NewTestPoint(
"cpu",
models.Tags{},
models.Fields{
@ -1131,7 +1205,7 @@ func TestNewPointFloatWithoutDecimal(t *testing.T) {
}
func TestNewPointNegativeFloat(t *testing.T) {
test(t, `cpu value=-0.64 1000000000`,
models.MustNewPoint(
NewTestPoint(
"cpu",
models.Tags{},
models.Fields{
@ -1143,7 +1217,7 @@ func TestNewPointNegativeFloat(t *testing.T) {
func TestNewPointFloatNoDecimal(t *testing.T) {
test(t, `cpu value=1. 1000000000`,
models.MustNewPoint(
NewTestPoint(
"cpu",
models.Tags{},
models.Fields{
@ -1155,7 +1229,7 @@ func TestNewPointFloatNoDecimal(t *testing.T) {
func TestNewPointFloatScientific(t *testing.T) {
test(t, `cpu value=6.632243e+06 1000000000`,
models.MustNewPoint(
NewTestPoint(
"cpu",
models.Tags{},
models.Fields{
@ -1167,11 +1241,11 @@ func TestNewPointFloatScientific(t *testing.T) {
func TestNewPointLargeInteger(t *testing.T) {
test(t, `cpu value=6632243i 1000000000`,
models.MustNewPoint(
NewTestPoint(
"cpu",
models.Tags{},
models.Fields{
"value": 6632243, // if incorrectly encoded as a float, it would show up as 6.632243e+06
"value": int64(6632243), // if incorrectly encoded as a float, it would show up as 6.632243e+06
},
time.Unix(1, 0)),
)